diff --git a/.envrc b/.envrc index 4c2be53f6..b1c251a11 100644 --- a/.envrc +++ b/.envrc @@ -43,9 +43,22 @@ grep -qxF "$PATTERN" "$EXCLUDE_FILE" || echo "$PATTERN" >> "$EXCLUDE_FILE" # Needed for the Go extension in VS Code to find the right toolchain. export GOROOT="$(go env GOROOT)" +# Ensure git submodules are initialized, embedding model is downloaded, +# and llama.cpp libraries are built. These are idempotent (skip if already done). +mise run ensure-llama-libs +mise run ensure-model + +# CGO flags for llama.cpp - use source directory where mise builds the libraries. +export LIBRARY_PATH="$WORKSPACE/backend/util/llama-go" +export C_INCLUDE_PATH="$WORKSPACE/backend/util/llama-go" + # These variables are defined in a separate file to avoid having to invoke direnv allow # every time we change them. The file doesn't allow any scripting for security, only variables. dotenv .env.vars # Optional loading of local env vars. dotenv_if_exists .env.local + +# GPU acceleration is platform-dependent: +# - macOS: Metal (always enabled, via backend/util/llama-go/zgpu_darwin.go) +# - Linux: CPU-only for local dev (Vulkan used in CI/production only) \ No newline at end of file diff --git a/.github/actions/ci-setup/action.yml b/.github/actions/ci-setup/action.yml index baac778f4..ef037904b 100644 --- a/.github/actions/ci-setup/action.yml +++ b/.github/actions/ci-setup/action.yml @@ -33,13 +33,79 @@ runs: sudo apt-get install -y gcc-12 g++-12 sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 - sudo apt-get install -y libgtk-3-dev webkit2gtk-4.0 libayatana-appindicator3-dev librsvg2-dev patchelf rpm libc6 python3 build-essential sqlite3 libsqlite3-dev flatpak flatpak-builder elfutils libnss3 libnspr4 libasound2t64 libnotify4 libpcre3 libpulse0 libxss1 libxtst6 squashfs-tools + sudo apt-get install -y libgtk-3-dev webkit2gtk-4.0 libayatana-appindicator3-dev librsvg2-dev patchelf rpm libc6 python3 build-essential sqlite3 libsqlite3-dev flatpak flatpak-builder elfutils libnss3 libnspr4 libasound2t64 libnotify4 libpcre3 libpulse0 libxss1 libxtst6 squashfs-tools cmake libgomp1 # Snap-related packages temporarily disabled - focusing on flatpak # sudo apt-get install -y snapd # sudo snap install snapcraft --classic --channel=7.x/stable # sudo snap install multipass shell: bash + - name: "Install Vulkan dev packages (Linux)" + if: inputs.matrix-os == 'ubuntu-latest' + run: | + sudo apt-get install -y libvulkan-dev glslc + shell: bash + + - name: "Install Vulkan SDK (Windows)" + if: inputs.matrix-os == 'windows-2025' + shell: powershell + run: | + $vulkanVersion = "1.4.313.2" + curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/$vulkanVersion/windows/vulkansdk-windows-X64-$vulkanVersion.exe" + & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install + Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\$vulkanVersion" + Add-Content $env:GITHUB_PATH "C:\VulkanSDK\$vulkanVersion\Bin" + + - name: "Build llama.cpp (Linux)" + if: inputs.matrix-os == 'ubuntu-latest' + run: | + cd backend/util/llama-go + BUILD_TYPE=vulkan CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF" make libbinding.a + shell: bash + + - name: "Build llama.cpp (macOS)" + if: startsWith(inputs.matrix-os, 'macos') + run: | + cd backend/util/llama-go + BUILD_TYPE=metal CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF" make libbinding.a + shell: bash + + - name: "Build llama.cpp (Windows)" + if: inputs.matrix-os == 'windows-2025' + shell: bash + run: | + set -euo pipefail + cd backend/util/llama-go + + cmake -G "MinGW Makefiles" -B build -S llama.cpp \ + -DGGML_VULKAN=ON \ + -DGGML_OPENMP=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DLLAMA_CURL=OFF \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_TOOLS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DLLAMA_BUILD_SERVER=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ + + cmake --build build --config Release -j "$(nproc)" + + cp build/src/libllama.a ./libllama.a + cp build/ggml/src/ggml.a ./libggml.a + cp build/ggml/src/ggml-base.a ./libggml-base.a + cp build/ggml/src/ggml-cpu.a ./libggml-cpu.a + cp build/ggml/src/ggml-vulkan/ggml-vulkan.a ./libggml-vulkan.a + cp build/common/libcommon.a ./libcommon.a + cp "$VULKAN_SDK/Lib/vulkan-1.lib" ./libvulkan-1.a + + for lib in libllama.a libggml.a libggml-base.a libggml-cpu.a libggml-vulkan.a libcommon.a libvulkan-1.a; do + [ -f "$lib" ] || { echo "ERROR: Missing $lib"; exit 1; } + done + + echo "All llama.cpp libraries built successfully" + # Additional packages for Flatpak building - name: "Setup Flatpak" diff --git a/.github/workflows/desktop-performance.yml b/.github/workflows/desktop-performance.yml index 8d694836a..c78bae38a 100644 --- a/.github/workflows/desktop-performance.yml +++ b/.github/workflows/desktop-performance.yml @@ -72,6 +72,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: recursive - uses: ./.github/actions/ci-setup with: @@ -80,10 +82,13 @@ jobs: - name: Build Backend (Unix) run: | mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) go build -o plz-out/bin/backend/seed-daemon-x86_64-unknown-linux-gnu ./backend/cmd/seed-daemon env: GOARCH: amd64 CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go - name: Set temporal version in package.json run: | diff --git a/.github/workflows/desktop-smoke-test.yml b/.github/workflows/desktop-smoke-test.yml index bbebf25fc..0cca08cc4 100644 --- a/.github/workflows/desktop-smoke-test.yml +++ b/.github/workflows/desktop-smoke-test.yml @@ -33,6 +33,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: recursive - uses: ./.github/actions/ci-setup with: @@ -44,20 +46,26 @@ jobs: if: matrix.config.os != 'windows-2025' run: | mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) go build -o plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }} ./backend/cmd/seed-daemon env: GOARCH: ${{ matrix.config.goarch }} CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go - name: Build Backend (Windows) if: matrix.config.os == 'windows-2025' run: | mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) go build -o plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }}.exe ./backend/cmd/seed-daemon env: GOOS: "windows" GOARCH: ${{ matrix.config.goarch }} CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go - name: Set temporal version in package.json run: | diff --git a/.github/workflows/dev-desktop.yml b/.github/workflows/dev-desktop.yml index 31d04d4a4..2ff8ea364 100644 --- a/.github/workflows/dev-desktop.yml +++ b/.github/workflows/dev-desktop.yml @@ -71,10 +71,38 @@ jobs: - os: windows-2025 arch: x64 goarch: amd64 - daemon_name: x86_64-pc-windows-msvc + daemon_name: x86_64-pc-windows-gnu steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Cache GGUF model + uses: actions/cache@v4 + with: + path: backend/llm/backends/llamacpp/models/*.gguf + key: gguf-model-granite-v2 + enableCrossOsArchive: true + + - name: Download GGUF model (Unix) + if: matrix.config.os != 'windows-2025' + run: | + if [ ! -f backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf ]; then + mkdir -p backend/llm/backends/llamacpp/models + curl -fSL -o backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf \ + "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" + fi + + - name: Download GGUF model (Windows) + if: startsWith(matrix.config.os, 'windows') + shell: pwsh + run: | + $modelPath = "backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf" + if (-not (Test-Path $modelPath)) { + New-Item -ItemType Directory -Force -Path "backend/llm/backends/llamacpp/models" + Invoke-WebRequest -Uri "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" -OutFile $modelPath + } - uses: ./.github/actions/ci-setup with: @@ -86,20 +114,73 @@ jobs: if: matrix.config.os != 'windows-2025' run: | mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) go build -o plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }} ./backend/cmd/seed-daemon env: GOARCH: ${{ matrix.config.goarch }} CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go - name: Build Backend (Windows) if: matrix.config.os == 'windows-2025' + shell: bash run: | mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) go build -o plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }}.exe ./backend/cmd/seed-daemon env: GOOS: "windows" GOARCH: ${{ matrix.config.goarch }} CGO_ENABLED: 1 + CGO_LDFLAGS: -static-libgcc -static-libstdc++ + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go + + - name: Stage Windows runtime DLL + if: matrix.config.os == 'windows-2025' + shell: bash + run: | + set -euo pipefail + DLL_PATH="$(gcc -print-file-name=libwinpthread-1.dll)" + + if [ ! -f "$DLL_PATH" ]; then + echo "ERROR: libwinpthread-1.dll not found in gcc toolchain" + exit 1 + fi + + cp "$DLL_PATH" plz-out/bin/backend/libwinpthread-1.dll + ls -la plz-out/bin/backend/libwinpthread-1.dll + + - name: Verify Windows daemon runtime deps + if: matrix.config.os == 'windows-2025' + shell: bash + run: | + set -euo pipefail + BIN="plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }}.exe" + + if ! command -v objdump >/dev/null 2>&1; then + echo "objdump not available on runner; skipping dependency check" + exit 0 + fi + + DLLS="$(objdump -p "$BIN" | awk '/DLL Name:/ {print $3}')" + echo "Windows DLL imports:" + echo "$DLLS" + + if echo "$DLLS" | grep -Eiq '^(libstdc\+\+-6\.dll|libgcc_s_seh-1\.dll|libgomp-1\.dll)$'; then + echo "ERROR: MinGW runtime DLL dependency is still present" + exit 1 + fi + + if echo "$DLLS" | grep -Eiq '^libwinpthread-1\.dll$'; then + if [ ! -f "plz-out/bin/backend/libwinpthread-1.dll" ]; then + echo "ERROR: daemon imports libwinpthread-1.dll but runtime DLL is not staged" + exit 1 + fi + + echo "libwinpthread-1.dll import detected and staged correctly" + fi - name: Set MacOS signing certs if: startsWith(matrix.config.os, 'macos') diff --git a/.github/workflows/dev-docker-images.yml b/.github/workflows/dev-docker-images.yml index c5500f645..5bc7ac7da 100644 --- a/.github/workflows/dev-docker-images.yml +++ b/.github/workflows/dev-docker-images.yml @@ -28,14 +28,56 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Cache GGUF model + uses: actions/cache@v4 + with: + path: backend/llm/backends/llamacpp/models/*.gguf + key: gguf-model-granite-v2 + enableCrossOsArchive: true + + - name: Download GGUF model + run: | + if [ ! -f backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf ]; then + mkdir -p backend/llm/backends/llamacpp/models + curl -fSL -o backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf \ + "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" + fi + - name: Set up Go uses: actions/setup-go@v5 with: go-version: "1.25.4" - - run: go test --count 1 ./backend/... + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake g++ libvulkan-dev glslc + + - name: Build llama.cpp (with Vulkan GPU support) + run: | + cd backend/util/llama-go + BUILD_TYPE=vulkan CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF" make libbinding.a + + - name: Run tests + run: go test --count 1 ./backend/... + env: + CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go + LLAMA_LOG: error + # Run tests again with the race-detector. # Using the same job to reuse the build cache. - - run: go test --count 1 -race ./backend/... + - name: Run tests with race detector + run: go test --count 1 -race ./backend/... + env: + CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go + LLAMA_LOG: error generate-docker-images: runs-on: ubuntu-latest needs: [frontend-tests, backend-tests] @@ -51,6 +93,8 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Get commit date for the triggering commit run: | diff --git a/.github/workflows/lint-go.yml b/.github/workflows/lint-go.yml index 05f7cb6ff..6e28fd9a5 100644 --- a/.github/workflows/lint-go.yml +++ b/.github/workflows/lint-go.yml @@ -25,6 +25,22 @@ jobs: with: go-version: "1.25.4" - uses: actions/checkout@v4 + + - name: Cache GGUF model + uses: actions/cache@v4 + with: + path: backend/llm/backends/llamacpp/models/*.gguf + key: gguf-model-granite-v2 + enableCrossOsArchive: true + + - name: Download GGUF model + run: | + if [ ! -f backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf ]; then + mkdir -p backend/llm/backends/llamacpp/models + curl -fSL -o backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf \ + "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" + fi + - uses: golangci/golangci-lint-action@v8 with: version: latest diff --git a/.github/workflows/release-desktop.yml b/.github/workflows/release-desktop.yml index 8e6f01656..ce586ac16 100644 --- a/.github/workflows/release-desktop.yml +++ b/.github/workflows/release-desktop.yml @@ -66,10 +66,38 @@ jobs: - os: windows-2025 arch: x64 goarch: amd64 - daemon_name: x86_64-pc-windows-msvc + daemon_name: x86_64-pc-windows-gnu steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Cache GGUF model + uses: actions/cache@v4 + with: + path: backend/llm/backends/llamacpp/models/*.gguf + key: gguf-model-granite-v2 + enableCrossOsArchive: true + + - name: Download GGUF model (Unix) + if: matrix.config.os != 'windows-2025' + run: | + if [ ! -f backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf ]; then + mkdir -p backend/llm/backends/llamacpp/models + curl -fSL -o backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf \ + "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" + fi + + - name: Download GGUF model (Windows) + if: startsWith(matrix.config.os, 'windows') + shell: pwsh + run: | + $modelPath = "backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf" + if (-not (Test-Path $modelPath)) { + New-Item -ItemType Directory -Force -Path "backend/llm/backends/llamacpp/models" + Invoke-WebRequest -Uri "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" -OutFile $modelPath + } - uses: ./.github/actions/ci-setup with: @@ -77,24 +105,89 @@ jobs: # matrix-target: ${{ matrix.config.daemon_name }} # matrix-arch: ${{ matrix.config.arch }} - - name: Build Backend (Unix) - if: matrix.config.os != 'windows-2025' + - name: Build Backend (Linux) + if: matrix.config.os == 'ubuntu-latest' + run: | + mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) + go build -o plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }} ./backend/cmd/seed-daemon + env: + GOARCH: ${{ matrix.config.goarch }} + CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go + + - name: Build Backend (macOS) + if: startsWith(matrix.config.os, 'macos') run: | mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) go build -o plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }} ./backend/cmd/seed-daemon env: GOARCH: ${{ matrix.config.goarch }} CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go - name: Build Backend (Windows) if: startsWith(matrix.config.os, 'windows') + shell: bash run: | mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) go build -o plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }}.exe ./backend/cmd/seed-daemon env: GOOS: "windows" GOARCH: ${{ matrix.config.goarch }} CGO_ENABLED: 1 + CGO_LDFLAGS: -static-libgcc -static-libstdc++ + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go + + - name: Stage Windows runtime DLL + if: startsWith(matrix.config.os, 'windows') + shell: bash + run: | + set -euo pipefail + DLL_PATH="$(gcc -print-file-name=libwinpthread-1.dll)" + + if [ ! -f "$DLL_PATH" ]; then + echo "ERROR: libwinpthread-1.dll not found in gcc toolchain" + exit 1 + fi + + cp "$DLL_PATH" plz-out/bin/backend/libwinpthread-1.dll + ls -la plz-out/bin/backend/libwinpthread-1.dll + + - name: Verify Windows daemon runtime deps + if: startsWith(matrix.config.os, 'windows') + shell: bash + run: | + set -euo pipefail + BIN="plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }}.exe" + + if ! command -v objdump >/dev/null 2>&1; then + echo "objdump not available on runner; skipping dependency check" + exit 0 + fi + + DLLS="$(objdump -p "$BIN" | awk '/DLL Name:/ {print $3}')" + echo "Windows DLL imports:" + echo "$DLLS" + + if echo "$DLLS" | grep -Eiq '^(libstdc\+\+-6\.dll|libgcc_s_seh-1\.dll|libgomp-1\.dll)$'; then + echo "ERROR: MinGW runtime DLL dependency is still present" + exit 1 + fi + + if echo "$DLLS" | grep -Eiq '^libwinpthread-1\.dll$'; then + if [ ! -f "plz-out/bin/backend/libwinpthread-1.dll" ]; then + echo "ERROR: daemon imports libwinpthread-1.dll but runtime DLL is not staged" + exit 1 + fi + + echo "libwinpthread-1.dll import detected and staged correctly" + fi - name: Set MacOS signing certs if: startsWith(matrix.config.os, 'macos') diff --git a/.github/workflows/release-docker-images.yml b/.github/workflows/release-docker-images.yml index 25afcec1d..8eb8fc9c6 100644 --- a/.github/workflows/release-docker-images.yml +++ b/.github/workflows/release-docker-images.yml @@ -17,19 +17,61 @@ jobs: # Use the reusable parallel test workflow frontend-tests: uses: ./.github/workflows/test-frontend-parallel.yml - + backend-tests: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Cache GGUF model + uses: actions/cache@v4 + with: + path: backend/llm/backends/llamacpp/models/*.gguf + key: gguf-model-granite-v2 + enableCrossOsArchive: true + + - name: Download GGUF model + run: | + if [ ! -f backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf ]; then + mkdir -p backend/llm/backends/llamacpp/models + curl -fSL -o backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf \ + "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" + fi + - name: Set up Go uses: actions/setup-go@v5 with: go-version: "1.25.4" - - run: go test --count 1 ./backend/... + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake g++ libvulkan-dev glslc + + - name: Build llama.cpp (with Vulkan GPU support) + run: | + cd backend/util/llama-go + BUILD_TYPE=vulkan CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF" make libbinding.a + + - name: Run tests + run: go test --count 1 ./backend/... + env: + CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go + LLAMA_LOG: error + # Run tests again with the race-detector. # Using the same job to reuse the build cache. - - run: go test --count 1 -race ./backend/... + - name: Run tests with race detector + run: go test --count 1 -race ./backend/... + env: + CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go + LLAMA_LOG: error generate-docker-images: runs-on: ubuntu-latest needs: [frontend-tests, backend-tests] @@ -45,6 +87,8 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Get commit date for the triggering commit run: | diff --git a/.github/workflows/test-desktop.yml b/.github/workflows/test-desktop.yml index 10e9857e1..3ce46873a 100644 --- a/.github/workflows/test-desktop.yml +++ b/.github/workflows/test-desktop.yml @@ -49,10 +49,12 @@ jobs: - os: windows-2025 arch: x64 goarch: amd64 - daemon_name: x86_64-pc-windows-msvc + daemon_name: x86_64-pc-windows-gnu steps: - name: Checkout uses: actions/checkout@v4 + with: + submodules: recursive - uses: ./.github/actions/ci-setup with: @@ -64,20 +66,26 @@ jobs: if: matrix.config.os != 'windows-2025' run: | mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) go build -o plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }} ./backend/cmd/seed-daemon env: GOARCH: ${{ matrix.config.goarch }} CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go - name: Build Backend (Windows) if: matrix.config.os == 'windows-2025' run: | mkdir -p plz-out/bin/backend + # GPU is enabled by default (no -tags needed) go build -o plz-out/bin/backend/seed-daemon-${{ matrix.config.daemon_name }}.exe ./backend/cmd/seed-daemon env: GOOS: "windows" GOARCH: ${{ matrix.config.goarch }} CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go - name: Set temporal version in package.json run: | diff --git a/.github/workflows/test-frontend-parallel.yml b/.github/workflows/test-frontend-parallel.yml index 8103fc861..4fb62087f 100644 --- a/.github/workflows/test-frontend-parallel.yml +++ b/.github/workflows/test-frontend-parallel.yml @@ -4,7 +4,7 @@ on: workflow_call: inputs: run-integration-tests: - description: 'Whether to run integration tests (web-related)' + description: "Whether to run integration tests (web-related)" required: false type: boolean default: true @@ -56,6 +56,8 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + with: + submodules: recursive - name: Install pnpm uses: pnpm/action-setup@v4 - name: Install Node.js 22 @@ -69,10 +71,33 @@ jobs: go-version: "1.25.4" - name: Install dependencies run: pnpm install + - name: Install build dependencies + run: sudo apt-get install -y cmake + - name: Cache GGUF model + uses: actions/cache@v4 + with: + path: backend/llm/backends/llamacpp/models/*.gguf + key: gguf-model-granite-v2 + enableCrossOsArchive: true + - name: Download GGUF model + run: | + if [ ! -f backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf ]; then + mkdir -p backend/llm/backends/llamacpp/models + curl -fSL -o backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf \ + "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" + fi + - name: Build llama.cpp (CPU-only) + run: | + cd backend/util/llama-go + CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF -DGGML_VULKAN=OFF -DGGML_METAL=OFF -DGGML_CUDA=OFF -DGGML_HIP=OFF -DGGML_SYCL=OFF -DGGML_BLAS=OFF" make libbinding.a - name: Build daemon binary run: | mkdir -p plz-out/bin/backend - go build -o plz-out/bin/backend/seed-daemon-x86_64-unknown-linux-gnu ./backend/cmd/seed-daemon + go build -tags cpu -o plz-out/bin/backend/seed-daemon-x86_64-unknown-linux-gnu ./backend/cmd/seed-daemon + env: + CGO_ENABLED: "1" + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go - name: Build web app run: pnpm web:prod - name: Install Playwright browsers diff --git a/.github/workflows/test-go.yml b/.github/workflows/test-go.yml index 67b3484b5..e24b9bb67 100644 --- a/.github/workflows/test-go.yml +++ b/.github/workflows/test-go.yml @@ -26,11 +26,54 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Cache GGUF model + uses: actions/cache@v4 + with: + path: backend/llm/backends/llamacpp/models/*.gguf + key: gguf-model-granite-v2 + enableCrossOsArchive: true + + - name: Download GGUF model + run: | + if [ ! -f backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf ]; then + mkdir -p backend/llm/backends/llamacpp/models + curl -fSL -o backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf \ + "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" + fi + - name: Set up Go uses: actions/setup-go@v5 with: go-version: "1.25.4" - - run: go test --count 1 ./backend/... + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y cmake g++ + + - name: Build llama.cpp (CPU-only for tests) + run: | + cd backend/util/llama-go + # Tests run without GPU hardware, so we build CPU-only + CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF -DGGML_VULKAN=OFF -DGGML_METAL=OFF -DGGML_CUDA=OFF -DGGML_HIP=OFF -DGGML_SYCL=OFF -DGGML_BLAS=OFF" make libbinding.a + + - name: Run tests + run: go test -tags cpu --count 1 ./backend/... + env: + CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go + LLAMA_LOG: error + # Run tests again with the race-detector. # Using the same job to reuse the build cache. - - run: go test --count 1 -race ./backend/... + - name: Run tests with race detector + run: go test -tags cpu --count 1 -race ./backend/... + env: + CGO_ENABLED: 1 + LIBRARY_PATH: ${{ github.workspace }}/backend/util/llama-go + C_INCLUDE_PATH: ${{ github.workspace }}/backend/util/llama-go + LLAMA_LOG: error diff --git a/.gitignore b/.gitignore index 55e45b221..e315b72b5 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,12 @@ scratch.*.* data .yarn/cache + +# llama-go (submodule) build artifacts +backend/util/llama-go/build +backend/util/llama-go/**/*.a +backend/util/llama-go/**/*.o +.cache/ + +# GGUF models (downloaded at setup time) +*.gguf \ No newline at end of file diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..37adeb938 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "backend/util/llama-go"] + path = backend/util/llama-go + url = https://github.com/seed-hypermedia/llama-go.git diff --git a/backend/BUILD.plz b/backend/BUILD.plz index 9d9b8409e..8f44755cb 100644 --- a/backend/BUILD.plz +++ b/backend/BUILD.plz @@ -1,25 +1,131 @@ subinclude("//build/rules/go:defs", "//build/rules/codegen:defs") -# Builds the seed-daemon binary. It depends on all the non-test -# Go files inside the `backend` directory. -go_binary( +# Build llama.cpp bindings before compiling Go code. +# Builds in-place in $WORKSPACE to avoid copying ~2500 files from the +# llama-go submodule (which includes the full llama.cpp source tree) into +# the Please sandbox. Outputs are copied back to the sandbox for downstream +# targets. This matches the pattern used by the seed-daemon target. +genrule( + name = "llama-cpp", + srcs = ["util/llama-go/Makefile"], + outs = [ + "backend/util/llama-go/libbinding.a", + "backend/util/llama-go/libcommon.a", + "backend/util/llama-go/libllama.a", + "backend/util/llama-go/libggml.a", + "backend/util/llama-go/libggml-cpu.a", + "backend/util/llama-go/libggml-base.a", + "backend/util/llama-go/libggml-blas.a", + "backend/util/llama-go/libggml-vulkan.a", + "backend/util/llama-go/libggml-metal.a", + "backend/util/llama-go/ggml-metal.metal", + ], + cmd = """ +set -e +LLAMA_DIR="$WORKSPACE/backend/util/llama-go" +cd "$LLAMA_DIR" +export LIBRARY_PATH=$(pwd) +export C_INCLUDE_PATH=$(pwd) +export PATH="$(dirname $TOOLS_CMAKE):$PATH" +# macOS always builds with Metal, Linux/Windows builds CPU-only for local dev. +# CI handles Vulkan/Metal per-platform in ci-setup/action.yml. +if [ "$OS" = "darwin" ]; then + export BUILD_TYPE=metal + export CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF" + echo "Building llama.cpp with Metal..." + make libbinding.a || { echo "ERROR: llama.cpp Metal build failed"; exit 1; } + cp build/bin/ggml-metal.metal . 2>/dev/null || true + # Stub for Vulkan (not used on macOS) + touch libggml-vulkan.a +else + echo "Building llama.cpp (CPU-only)..." + export CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF -DGGML_VULKAN=OFF -DGGML_METAL=OFF -DGGML_CUDA=OFF -DGGML_HIP=OFF -DGGML_SYCL=OFF -DGGML_BLAS=OFF" + make libbinding.a || { echo "ERROR: llama.cpp CPU build failed"; exit 1; } + # Stubs for GPU libraries (not used in CPU-only build) + touch libggml-blas.a + touch libggml-vulkan.a + touch libggml-metal.a + touch ggml-metal.metal +fi +echo "llama.cpp build completed successfully" +# Copy outputs back to the Please sandbox where it expects them. +OUTDIR="$TMP_DIR/backend/util/llama-go" +mkdir -p "$OUTDIR" +cp libbinding.a libcommon.a libllama.a libggml.a libggml-cpu.a libggml-base.a "$OUTDIR/" +cp libggml-blas.a libggml-vulkan.a libggml-metal.a "$OUTDIR/" +cp ggml-metal.metal "$OUTDIR/" 2>/dev/null || touch "$OUTDIR/ggml-metal.metal" + """, + building_description = "Building llama.cpp bindings...", + tools = { + "cmake": ["//build/tools:cmake"], + }, + env = { + "OS": CONFIG.TARGET_OS, + }, + visibility = ["//backend/..."], +) + +# Builds the seed-daemon binary with llama.cpp CGO flags +genrule( name = "seed-daemon", srcs = glob( [ "**/*.go", "**/*.c", "**/*.h", + "**/*.cpp", + "**/*.hpp", + ], + exclude = [ + "**/*_test.go", + "util/llama-go/llama.cpp/**", ], - exclude = ["**/*_test.go"], ) + [ "//backend/lndhub/lndhubsql:go_library", "//backend/storage:go_library", "//backend/wallet/walletsql:go_library", + ":llama-cpp", + "//:gomod", ], - out = "seed-daemon-" + target_platform_triple(), - cgo = True, - gomod = "//:gomod", - package = "./cmd/seed-daemon", + outs = ["seed-daemon-" + target_platform_triple()], + cmd = """ +set -e +TMPDIR=/tmp +HOME=$(eval echo ~$(whoami)) + +# Work from the actual workspace, not the temp build directory +cd $WORKSPACE + +# Libraries from llama-cpp dependency are placed in TMP_DIR by Please +# The outs from llama-cpp are declared as "backend/util/llama-go/*.a" +# Since llama-cpp is in the backend package, outputs go to: +# $TMP_DIR/backend/backend/util/llama-go/ +LLAMA_GO_PATH=$TMP_DIR/backend/backend/util/llama-go + +export CGO_ENABLED=1 +export CGO_CXXFLAGS="-std=c++17" +export LIBRARY_PATH=$LLAMA_GO_PATH +export C_INCLUDE_PATH=$LLAMA_GO_PATH + +# macOS uses Metal (zgpu_darwin.go included), Linux uses CPU-only (-tags cpu excludes zgpu_*.go). +BUILD_TAGS="" +if [ "$OS" != "darwin" ]; then + BUILD_TAGS="-tags cpu" +fi + +echo "Looking for llama libraries in: $LLAMA_GO_PATH" +ls -la $LLAMA_GO_PATH/*.a || echo "No .a files found!" + +$TOOLS_GO build $BUILD_TAGS -trimpath -o $OUT ./backend/cmd/seed-daemon + """, + binary = True, + building_description = "Building seed-daemon with llama.cpp...", + tools = { + "go": [CONFIG.GO_TOOL], + }, + env = { + "OS": CONFIG.TARGET_OS, + }, visibility = ["PUBLIC"], ) diff --git a/backend/api/activity/v1alpha/activity.go b/backend/api/activity/v1alpha/activity.go index 2f6f51b7c..8ab97ea05 100644 --- a/backend/api/activity/v1alpha/activity.go +++ b/backend/api/activity/v1alpha/activity.go @@ -258,9 +258,7 @@ func (srv *Server) ListEvents(ctx context.Context, req *activity.ListEventsReque refsJson := "[" + strings.Join(refIDs, ",") + "]" var versions = map[int64]string{} if err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { - if err := sqlitex.ExecTransient(conn, qGetChangesFromRefs(), func(stmt *sqlite.Stmt) error { - mhBinary, err := hex.DecodeString(stmt.ColumnText(0)) if err != nil { return err diff --git a/backend/api/apis.go b/backend/api/apis.go index d82909506..201023f90 100644 --- a/backend/api/apis.go +++ b/backend/api/apis.go @@ -14,6 +14,7 @@ import ( p2p "seed/backend/genproto/p2p/v1alpha" "seed/backend/hmnet" "seed/backend/hmnet/syncing" + "seed/backend/llm" "seed/backend/logging" "seed/backend/storage" @@ -49,6 +50,7 @@ func New( isMainnet bool, dlink *devicelink.Service, taskMgr *taskmanager.TaskManager, + embedder llm.LightEmbedder, ) Server { db := repo.DB() proxy := &p2pProxy{node: node} @@ -56,7 +58,7 @@ func New( Activity: activity, Daemon: daemon.NewServer(repo, node, idx, dlink, taskMgr), Networking: networking.NewServer(node, db, logging.New("seed/networking", LogLevel)), - Entities: entities.NewServer(db, sync), + Entities: entities.NewServer(db, sync, embedder), DocumentsV3: documentsv3.NewServer(cfg, repo.KeyStore(), idx, db, logging.New("seed/documents", LogLevel), node), Syncing: sync, Payments: payments.NewServer(logging.New("seed/payments", LogLevel), db, node, repo.KeyStore(), isMainnet), diff --git a/backend/api/documents/v3alpha/dochistory.go b/backend/api/documents/v3alpha/dochistory.go index bba5e5158..de27a80d0 100644 --- a/backend/api/documents/v3alpha/dochistory.go +++ b/backend/api/documents/v3alpha/dochistory.go @@ -67,7 +67,7 @@ func (srv *Server) ListDocumentChanges(ctx context.Context, in *documents.ListDo StartFrom string } if in.PageToken != "" { - apiutil.DecodePageToken(in.PageToken, &cursor, nil) + _ = apiutil.DecodePageToken(in.PageToken, &cursor, nil) } out := &documents.ListDocumentChangesResponse{ diff --git a/backend/api/entities/v1alpha/entities.go b/backend/api/entities/v1alpha/entities.go index add74e8b2..95d887a7b 100644 --- a/backend/api/entities/v1alpha/entities.go +++ b/backend/api/entities/v1alpha/entities.go @@ -13,15 +13,17 @@ import ( "seed/backend/api/documents/v3alpha/docmodel" "seed/backend/blob" "seed/backend/core" - entities "seed/backend/genproto/entities/v1alpha" + entpb "seed/backend/genproto/entities/v1alpha" "seed/backend/hlc" "seed/backend/hmnet/syncing" + "seed/backend/llm" + "seed/backend/util/apiutil" "seed/backend/util/dqb" "seed/backend/util/errutil" "slices" - "sort" "strconv" "strings" + "sync" "time" "unicode/utf8" @@ -47,23 +49,32 @@ type Discoverer interface { // Server implements Entities API. type Server struct { - entities.UnimplementedEntitiesServer + entpb.UnimplementedEntitiesServer - db *sqlitex.Pool - disc Discoverer + db *sqlitex.Pool + disc Discoverer + embedder llm.LightEmbedder } // NewServer creates a new entities server. -func NewServer(db *sqlitex.Pool, disc Discoverer) *Server { +func NewServer(db *sqlitex.Pool, disc Discoverer, embedder llm.LightEmbedder) *Server { return &Server{ - db: db, - disc: disc, + db: db, + disc: disc, + embedder: embedder, } } // RegisterServer registers the server with the gRPC server. func (srv *Server) RegisterServer(rpc grpc.ServiceRegistrar) { - entities.RegisterEntitiesServer(rpc, srv) + entpb.RegisterEntitiesServer(rpc, srv) +} + +// validIriFilterRe validates iri_filter to prevent GLOB injection. +var validIriFilterRe = regexp.MustCompile(`^hm://[a-zA-Z0-9_\-./\*\?\[\]]*$`) + +func isValidIriFilter(s string) bool { + return validIriFilterRe.MatchString(s) } const ( @@ -72,8 +83,8 @@ const ( ) // DiscoverEntity implements the Entities server. -func (api *Server) DiscoverEntity(ctx context.Context, in *entities.DiscoverEntityRequest) (*entities.DiscoverEntityResponse, error) { - if api.disc == nil { +func (srv *Server) DiscoverEntity(_ context.Context, in *entpb.DiscoverEntityRequest) (*entpb.DiscoverEntityResponse, error) { + if srv.disc == nil { return nil, status.Errorf(codes.FailedPrecondition, "discovery is not enabled") } @@ -101,9 +112,9 @@ func (api *Server) DiscoverEntity(ctx context.Context, in *entities.DiscoverEnti v := blob.Version(in.Version) // Delegate to syncing service for task management. - info := api.disc.TouchHotTask(iri, v, in.Recursive) + info := srv.disc.TouchHotTask(iri, v, in.Recursive) - resp := &entities.DiscoverEntityResponse{ + resp := &entpb.DiscoverEntityResponse{ Version: info.Result.String(), State: stateToProto(info.State), Progress: progressToProto(info.Progress), @@ -121,24 +132,24 @@ func (api *Server) DiscoverEntity(ctx context.Context, in *entities.DiscoverEnti return resp, nil } -func stateToProto(state syncing.TaskState) entities.DiscoveryTaskState { +func stateToProto(state syncing.TaskState) entpb.DiscoveryTaskState { switch state { case syncing.TaskStateIdle: - return entities.DiscoveryTaskState_DISCOVERY_TASK_STARTED + return entpb.DiscoveryTaskState_DISCOVERY_TASK_STARTED case syncing.TaskStateInProgress: - return entities.DiscoveryTaskState_DISCOVERY_TASK_IN_PROGRESS + return entpb.DiscoveryTaskState_DISCOVERY_TASK_IN_PROGRESS case syncing.TaskStateCompleted: - return entities.DiscoveryTaskState_DISCOVERY_TASK_COMPLETED + return entpb.DiscoveryTaskState_DISCOVERY_TASK_COMPLETED default: - return entities.DiscoveryTaskState_DISCOVERY_TASK_STARTED + return entpb.DiscoveryTaskState_DISCOVERY_TASK_STARTED } } -func progressToProto(prog *syncing.Progress) *entities.DiscoveryProgress { +func progressToProto(prog *syncing.Progress) *entpb.DiscoveryProgress { if prog == nil { - return &entities.DiscoveryProgress{} + return &entpb.DiscoveryProgress{} } - return &entities.DiscoveryProgress{ + return &entpb.DiscoveryProgress{ PeersFound: prog.PeersFound.Load(), PeersSyncedOk: prog.PeersSyncedOK.Load(), PeersFailed: prog.PeersFailed.Load(), @@ -150,12 +161,15 @@ func progressToProto(prog *syncing.Progress) *entities.DiscoveryProgress { var qGetLatestBlockChange = dqb.Str(` SELECT - blob_id, + fts_index.blob_id, version, block_id, ts, - type - from fts_index + type, + b.codec, + b.multihash + FROM fts_index + JOIN blobs b ON b.id = fts_index.blob_id WHERE genesis_blob = :genesisBlobID AND ts >= :Ts AND type IN ('title', 'document', 'meta') @@ -187,8 +201,7 @@ SELECT AND sb.genesis_blob IN (SELECT value FROM json_each(:genesisBlobJson)); `) -// get the extra_attrs->>'redirect' != ” for the same genesis blob and if its not null then put that as a iri -var qGetFTS = dqb.Str(` +var qGetFTSByIDs = dqb.Str(` WITH fts_data AS ( SELECT fts.raw_content, @@ -198,7 +211,6 @@ WITH fts_data AS ( fts.blob_id, structural_blobs.genesis_blob, structural_blobs.extra_attrs->>'tsid' AS tsid, - fts.rank, fts.rowid FROM fts JOIN structural_blobs @@ -209,12 +221,8 @@ WITH fts_data AS ( ON public_keys.id = structural_blobs.author LEFT JOIN resources ON resources.id = structural_blobs.resource - WHERE fts.raw_content MATCH :ftsStr - AND fts.type IN (:entityTitle, :entityContact, :entityDoc, :entityComment) + WHERE fts.rowid IN (SELECT value FROM json_each(?)) AND blobs.size > 0 - ORDER BY - (fts.type = 'contact' || fts.type = 'title') ASC, -- prioritize contacts then titles, comments and documents are mixed based on rank - fts.rank ASC ) SELECT @@ -266,7 +274,7 @@ FROM fts_data AS f AND structural_blobs.type = 'Comment') OR (f.blob_id = structural_blobs.id AND structural_blobs.type = 'Contact' - AND structural_blobs.author = :loggedAccountID) + AND structural_blobs.author = ?) limit 1) JOIN document_generations @@ -278,14 +286,329 @@ FROM fts_data AS f LEFT JOIN public_keys pk_subject ON pk_subject.id = structural_blobs.extra_attrs->>'subject' -WHERE resources.iri IS NOT NULL AND resources.iri GLOB :iriGlob -AND document_generations.is_deleted = False +WHERE document_generations.is_deleted = False +`) + +var qKeywordSearch = dqb.Str(` +SELECT + fts.rowid, + fts.rank +FROM fts +JOIN fts_index fi ON fi.rowid = fts.rowid +JOIN structural_blobs sb ON sb.id = fts.blob_id +JOIN blobs ON blobs.id = fts.blob_id +LEFT JOIN resources r1 ON r1.id = sb.resource +LEFT JOIN blob_links bl ON bl.target = fts.blob_id AND bl.type = 'ref/head' +LEFT JOIN structural_blobs sb_ref ON sb_ref.id = bl.source +LEFT JOIN resources r2 ON r2.id = sb_ref.resource +WHERE fts.raw_content MATCH ? + AND fts.type IN (?, ?, ?, ?) + AND blobs.size > 0 + AND COALESCE(r1.iri, r2.iri) IS NOT NULL + AND COALESCE(r1.iri, r2.iri) GLOB ? ORDER BY - (f.type = 'contact' || f.type = 'title') ASC, -- prioritize contacts then titles, comments and documents are mixed based on rank - f.rank ASC -LIMIT :limit + (fts.type = 'contact' OR fts.type = 'title') DESC, + fts.rank ASC +LIMIT ? `) +// keywordSearch performs minimal FTS search returning SearchResultMap. +// This is a standalone function (not Server method) used for hybrid search. +func keywordSearch(conn *sqlite.Conn, query string, limit int, contentTypes map[string]bool, iriGlob string) (llm.SearchResultMap, error) { + results := make(llm.SearchResultMap) + var entityTypeTitle, entityTypeContact, entityTypeDoc, entityTypeComment interface{} + supportedType := false + if ok, val := contentTypes["title"]; ok && val { + entityTypeTitle = "title" + supportedType = true + } + if ok, val := contentTypes["contact"]; ok && val { + entityTypeContact = "contact" + supportedType = true + } + if ok, val := contentTypes["document"]; ok && val { + entityTypeDoc = "document" + supportedType = true + } + if ok, val := contentTypes["comment"]; ok && val { + entityTypeComment = "comment" + supportedType = true + } + if !supportedType { + return nil, fmt.Errorf("invalid content type filter: at least one of title, contact, document, comment must be specified") + } + if len(contentTypes) == 0 { + return nil, errors.New("at least one content type is required. Otherwise there is nothing to search :)") + } + score := float32(999999.9) + if err := sqlitex.Exec(conn, qKeywordSearch(), func(stmt *sqlite.Stmt) error { + // The query alredy handles proper ordering and limit. The order depends on type and rank. + // We assign scores in decreasing order to be consistent with other search methods. + results[stmt.ColumnInt64(0)] = score + score-- + return nil + }, query, entityTypeTitle, entityTypeContact, entityTypeDoc, entityTypeComment, iriGlob, limit); err != nil { + return nil, fmt.Errorf("keyword search failed: %w", err) + } + + return results, nil +} + +type blendedResult struct { + result llm.SearchResult + semanticRank *int + keywordRank *int +} + +// blendSearchResults uses RRF (Reciprocal Rank Fusion) to blend semantic and keyword results. +// For single-word queries, keyword results are weighted higher (60%) since semantic embeddings +// are less reliable for short queries. For multi-word queries, equal weights (50/50) are used. +func blendSearchResults(semanticResults, keywordResults llm.SearchResultMap, limit int, query string) llm.SearchResultMap { + const rrfK = 60 + + // Single-word queries: favor keyword (60%) over semantic (40%). + // Multi-word queries: equal weight (50/50). + wordCount := len(strings.Fields(query)) + semanticWeight := float32(0.5) + if wordCount <= 1 { + semanticWeight = 0.4 + } + + resultMap := make(map[int64]*blendedResult) + semanticResultsOrdered := semanticResults.ToList(true) + keywordResultsOrdered := keywordResults.ToList(true) + // Map semantic results + for rank, result := range semanticResultsOrdered { + r := rank + 1 + resultMap[result.RowID] = &blendedResult{ + result: result, + semanticRank: &r, + keywordRank: nil, + } + } + + // Map keyword results + for rank, result := range keywordResultsOrdered { + r := rank + 1 + if existing, ok := resultMap[result.RowID]; ok { + existing.keywordRank = &r + } else { + resultMap[result.RowID] = &blendedResult{ + result: result, + semanticRank: nil, + keywordRank: &r, + } + } + } + + resultList := make([]llm.SearchResult, 0, len(resultMap)) + // Calculate RRF combined scores + for _, br := range resultMap { + semanticRRF := float32(0.0) + keywordRRF := float32(0.0) + + if br.semanticRank != nil { + semanticRRF = 1.0 / float32(rrfK+*br.semanticRank) + } + if br.keywordRank != nil { + keywordRRF = 1.0 / float32(rrfK+*br.keywordRank) + } + + combinedScore := semanticWeight*semanticRRF + (1-semanticWeight)*keywordRRF + resultList = append(resultList, llm.SearchResult{Score: combinedScore, RowID: br.result.RowID}) + } + + // Sort by combined score with RowID as tie-breaker for deterministic ordering. + slices.SortFunc(resultList, func(a, b llm.SearchResult) int { + if a.Score < b.Score { + return 1 + } else if a.Score > b.Score { + return -1 + } + // Tie-breaker: sort by RowID for deterministic ordering. + if a.RowID < b.RowID { + return -1 + } else if a.RowID > b.RowID { + return 1 + } + return 0 + }) + + // Take top winners + winners := resultList[:min(limit, len(resultList))] + return llm.SearchResultList(winners).ToMap() +} + +// Document citation count: how many times each resource is linked to by others. +var qDocAuthority = dqb.Str(` +SELECT r.iri, COUNT(*) AS mention_count +FROM resource_links rl +JOIN resources r ON r.id = rl.target +WHERE r.iri IN (SELECT value FROM json_each(?)) +GROUP BY rl.target +`) + +// Author external citation count with self-citation filtering. +// Uses CTE to deduplicate authors, then counts external citations per author. +var qAuthorAuthority = dqb.Str(` +WITH doc_authors AS ( + SELECT DISTINCT doc.owner AS author_id + FROM json_each(?) je + JOIN resources doc ON doc.iri = je.value + WHERE doc.owner IS NOT NULL +), +author_scores AS ( + SELECT da.author_id, + COUNT(*) AS external_citations + FROM doc_authors da + JOIN resources r ON r.owner = da.author_id + JOIN resource_links rl ON rl.target = r.id + JOIN structural_blobs sb ON sb.id = rl.source + WHERE sb.author IS NULL OR sb.author <> da.author_id + GROUP BY da.author_id +) +SELECT doc.iri AS doc_iri, + COALESCE(s.external_citations, 0) AS author_external_citations +FROM json_each(?) je +JOIN resources doc ON doc.iri = je.value +LEFT JOIN author_scores s ON s.author_id = doc.owner +`) + +// buildRankMap creates a map from IRI to 1-based rank, sorted by score desc. +func buildRankMap(results []fullDataSearchResult, scoreFn func(fullDataSearchResult) int) map[string]int { + type entry struct { + iri string + score int + } + seen := make(map[string]bool) + var entries []entry + for _, r := range results { + if !seen[r.iri] { + seen[r.iri] = true + entries = append(entries, entry{r.iri, scoreFn(r)}) + } + } + slices.SortFunc(entries, func(a, b entry) int { + if a.score > b.score { + return -1 + } + if a.score < b.score { + return 1 + } + return 0 + }) + ranks := make(map[string]int, len(entries)) + for i, e := range entries { + ranks[e.iri] = i + 1 + } + return ranks +} + +// applyAuthorityRanking re-scores results using citation-based authority signals. +// The weight parameter controls the balance between text relevance and authority. +func applyAuthorityRanking(ctx context.Context, db *sqlitex.Pool, + results []fullDataSearchResult, bodyMatches []fuzzy.Match, + weight float32, +) ([]fullDataSearchResult, []fuzzy.Match, error) { + if len(results) == 0 { + return results, bodyMatches, nil + } + + // Collect unique IRIs. + iris := make([]string, 0, len(results)) + seen := make(map[string]bool) + for _, r := range results { + if !seen[r.iri] { + seen[r.iri] = true + iris = append(iris, r.iri) + } + } + irisJSON, err := json.Marshal(iris) + if err != nil { + return nil, nil, err + } + + // Run both authority queries in a single DB connection. + docCitations := make(map[string]int) + authorCitations := make(map[string]int) + + if err := db.WithSave(ctx, func(conn *sqlite.Conn) error { + if err := sqlitex.Exec(conn, qDocAuthority(), func(stmt *sqlite.Stmt) error { + docCitations[stmt.ColumnText(0)] = stmt.ColumnInt(1) + return nil + }, string(irisJSON)); err != nil { + return err + } + return sqlitex.Exec(conn, qAuthorAuthority(), func(stmt *sqlite.Stmt) error { + authorCitations[stmt.ColumnText(0)] = stmt.ColumnInt(1) + return nil + }, string(irisJSON), string(irisJSON)) + }); err != nil { + return nil, nil, err + } + + // Build rank maps from citation counts. + docAuthRanks := buildRankMap(results, func(r fullDataSearchResult) int { return docCitations[r.iri] }) + authorAuthRanks := buildRankMap(results, func(r fullDataSearchResult) int { return authorCitations[r.iri] }) + + // Re-score each result. + const rrfK = 60 + textWeight := 1.0 - weight + docAuthWeight := 0.7 * weight + authorAuthWeight := 0.3 * weight + + for i := range results { + textRank := i + 1 // Current position is the text rank (results are already sorted by score). + textRRF := 1.0 / float32(rrfK+textRank) + + var docRRF float32 + if r, ok := docAuthRanks[results[i].iri]; ok { + docRRF = 1.0 / float32(rrfK+r) + } + + var authRRF float32 + if r, ok := authorAuthRanks[results[i].iri]; ok { + authRRF = 1.0 / float32(rrfK+r) + } + + results[i].score = textWeight*textRRF + docAuthWeight*docRRF + authorAuthWeight*authRRF + } + + // Re-sort results and bodyMatches together by new score. + // Use rowID as tie-breaker for deterministic ordering when scores are equal. + indices := make([]int, len(results)) + for i := range indices { + indices[i] = i + } + slices.SortFunc(indices, func(a, b int) int { + if results[a].score > results[b].score { + return -1 + } + if results[a].score < results[b].score { + return 1 + } + // Tie-breaker: sort by rowID for deterministic ordering. + if results[a].rowID < results[b].rowID { + return -1 + } + if results[a].rowID > results[b].rowID { + return 1 + } + return 0 + }) + + sorted := make([]fullDataSearchResult, len(results)) + sortedMatches := make([]fuzzy.Match, len(bodyMatches)) + for newIdx, oldIdx := range indices { + sorted[newIdx] = results[oldIdx] + bm := bodyMatches[oldIdx] + bm.Index = newIdx + sortedMatches[newIdx] = bm + } + + return sorted, sortedMatches, nil +} + var qIsDeletedComment = dqb.Str(` SELECT CASE WHEN extra_attrs->>'deleted' = '1' THEN 1 ELSE 0 END AS is_deleted @@ -317,7 +640,7 @@ type commentIdentifier struct { tsid string } -type searchResult struct { +type fullDataSearchResult struct { content string rawContent string icon string @@ -335,8 +658,12 @@ type searchResult struct { version string versionTime *timestamppb.Timestamp latestVersion string + latestBlobCID string // CID of the latest blob (first head), used for version upgrade. commentKey commentIdentifier isDeleted bool + score float32 + parentTitles []string + id string } // MovedResource represents a resource that has been relocated. @@ -354,13 +681,8 @@ type MovedResource struct { LatestVersion string } -// SearchEntities implements the Fuzzy search of entities. -func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntitiesRequest) (*entities.SearchEntitiesResponse, error) { - //start := time.Now() - //defer func() { - // fmt.Println("SearchEntities duration:", time.Since(start)) - //}() - searchResults := []searchResult{} +// SearchEntities implements the Fuzzy search of entpb. +func (srv *Server) SearchEntities(ctx context.Context, in *entpb.SearchEntitiesRequest) (*entpb.SearchEntitiesResponse, error) { type value struct { Value string `json:"v"` } @@ -383,12 +705,29 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti return nil, nil } var bodyMatches []fuzzy.Match - const entityTypeTitle = "title" - var entityTypeContact, entityTypeDoc, entityTypeComment interface{} + contentTypes := map[string]bool{} + if len(in.ContentTypeFilter) > 0 { + for _, ct := range in.ContentTypeFilter { + switch ct { + case entpb.ContentTypeFilter_CONTENT_TYPE_TITLE: + contentTypes["title"] = true + case entpb.ContentTypeFilter_CONTENT_TYPE_DOCUMENT: + contentTypes["document"] = true + case entpb.ContentTypeFilter_CONTENT_TYPE_COMMENT: + contentTypes["comment"] = true + case entpb.ContentTypeFilter_CONTENT_TYPE_CONTACT: + contentTypes["contact"] = true + } + } + } else { + // Legacy fallback. + contentTypes["title"] = true + contentTypes["contact"] = true + if in.IncludeBody { + contentTypes["document"] = true + contentTypes["comment"] = true - if in.IncludeBody { - entityTypeDoc = "document" - entityTypeComment = "comment" + } } var loggedAccountID int64 = 0 if in.LoggedAccountUid != "" { @@ -398,65 +737,147 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti } ppalHex := hex.EncodeToString(ppal) if err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { - return sqlitex.ExecTransient(conn, qGetAccountID(), func(stmt *sqlite.Stmt) error { + return sqlitex.Exec(conn, qGetAccountID(), func(stmt *sqlite.Stmt) error { loggedAccountID = stmt.ColumnInt64(0) return nil }, strings.ToUpper(ppalHex)) }); err != nil { return nil, status.Errorf(codes.InvalidArgument, "Problem getting logged account ID %s: %v", in.LoggedAccountUid, err) } - entityTypeContact = "contact" + // TODO: Remove auto-include of contacts once frontend uses content_type_filter explicitly. + contentTypes["contact"] = true } - resultsLmit := 1000 - - if len(cleanQuery) < 3 { + // Adjust results limit based on search type + resultsLmit := 300 + if in.SearchType == entpb.SearchType_SEARCH_HYBRID || in.SearchType == entpb.SearchType_SEARCH_SEMANTIC { resultsLmit = 200 + } else if len(cleanQuery) < 3 { + resultsLmit = 100 } - ftsStr := strings.ReplaceAll(cleanQuery, " ", "+") - if ftsStr[len(ftsStr)-1] == '+' { - ftsStr = ftsStr[:len(ftsStr)-1] + ftsStrKeySearch := strings.ReplaceAll(cleanQuery, " ", "+") + if ftsStrKeySearch[len(ftsStrKeySearch)-1] == '+' { + ftsStrKeySearch = ftsStrKeySearch[:len(ftsStrKeySearch)-1] } - ftsStr += "*" + ftsStrKeySearch += "*" if in.ContextSize < 2 { in.ContextSize = 48 } - //fmt.Println("context size:", in.ContextSize) - var iriGlob string = "hm://" + in.AccountUid + "*" + + var iriGlob string + if in.IriFilter != "" { + if !isValidIriFilter(in.IriFilter) { + return nil, status.Errorf(codes.InvalidArgument, "iri_filter contains invalid characters") + } + iriGlob = in.IriFilter + } else if in.AccountUid != "" { + iriGlob = "hm://" + in.AccountUid + "*" + } else { + iriGlob = "hm://*" + } contextBefore := int(math.Ceil(float64(in.ContextSize) / 2.0)) contextAfter := int(in.ContextSize) - contextBefore var numResults int = 0 - //before := time.Now() - //fmt.Println("BeforeFTS Elapsed time:", time.Since(start)) + + // Prepare variables for semantic/hybrid search + query := cleanQuery + + winners := llm.SearchResultMap{} + const semanticThreshold = 0.45 // 0.55 Minimum similarity for relevant results with granite-embedding-107m-multilingual model. + + // Check if semantic search is requested but embedder is not available. + if srv.embedder == nil && (in.SearchType == entpb.SearchType_SEARCH_HYBRID || in.SearchType == entpb.SearchType_SEARCH_SEMANTIC) { + return nil, status.Errorf(codes.Unavailable, "semantic search is not available: embedding service is disabled") + } + + switch in.SearchType { + case entpb.SearchType_SEARCH_HYBRID: + // Hybrid search: run semantic + keyword concurrently, blend with RRF + var semanticResults, keywordResults llm.SearchResultMap + var semanticErr, keywordErr error + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + semanticResults, semanticErr = srv.embedder.SemanticSearch(ctx, query, resultsLmit*3, contentTypes, iriGlob, semanticThreshold) + }() + go func() { + defer wg.Done() + keywordErr = srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { + var err error + keywordResults, err = keywordSearch(conn, ftsStrKeySearch, resultsLmit*3, contentTypes, iriGlob) + return err + }) + }() + wg.Wait() + if keywordErr != nil { + return nil, fmt.Errorf("keyword search failed: %w", keywordErr) + } + + // Handle semantic search errors. + if semanticErr != nil { + if errors.Is(semanticErr, llm.ErrUnreliableEmbedding) { + // Query embedding is unreliable (rare/unknown word). Fall back to keyword-only results. + winners = keywordResults + } else { + return nil, fmt.Errorf("semantic search failed: %w", semanticErr) + } + } else { + // Blend results with RRF. + winners = blendSearchResults(semanticResults, keywordResults, resultsLmit*2, query) + } + + case entpb.SearchType_SEARCH_SEMANTIC: + // Semantic-only search. + var err error + winners, err = srv.embedder.SemanticSearch(ctx, query, resultsLmit*2, contentTypes, iriGlob, semanticThreshold) + if err != nil { + if errors.Is(err, llm.ErrUnreliableEmbedding) { + // Query embedding is unreliable. Return empty results for semantic-only search. + winners = llm.SearchResultMap{} + } else { + return nil, fmt.Errorf("semantic search failed: %w", err) + } + } + + default: + // Keyword only search: + err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { + var err error + winners, err = keywordSearch(conn, ftsStrKeySearch, resultsLmit, contentTypes, iriGlob) + return err + }) + if err != nil { + return nil, fmt.Errorf("keyword search failed: %w", err) + } + } + winnerIDsJSON, err := json.Marshal(winners.Keys()) + if err != nil { + return nil, fmt.Errorf("failed to marshal winner IDs: %w", err) + } + searchResults := []fullDataSearchResult{} if err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { - return sqlitex.ExecTransient(conn, qGetFTS(), func(stmt *sqlite.Stmt) error { - var res searchResult + return sqlitex.Exec(conn, qGetFTSByIDs(), func(stmt *sqlite.Stmt) error { + var res fullDataSearchResult var icon icon var heads []head res.rawContent = stmt.ColumnText(0) + + // Semantic results may not contain the query pattern (fuzzy match). + // So we find the first occurrence of the query pattern for context extraction. firstRuneOffset, _, matchedRunes, _ := indexOfQueryPattern(res.rawContent, cleanQuery) - if firstRuneOffset == -1 { - return nil - } - // before extracting matchStr, convert fullMatchStr to runes fullRunes := []rune(res.rawContent) nRunes := len(fullRunes) - var contextStart, contextEndRune int - // default to full slice contextEndRune = nRunes - if firstRuneOffset > contextBefore { contextStart = firstRuneOffset - contextBefore } if firstRuneOffset+matchedRunes < nRunes-contextAfter { contextEndRune = firstRuneOffset + matchedRunes + contextAfter } - - // build substring on rune boundaries res.content = string(fullRunes[contextStart:contextEndRune]) res.blobCID = cid.NewCidV1(uint64(stmt.ColumnInt64(9)), stmt.ColumnBytesUnsafe(10)).String() - res.contentType = stmt.ColumnText(1) res.blockID = stmt.ColumnText(2) res.version = stmt.ColumnText(3) @@ -482,6 +903,9 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti cids[i] = cid.NewCidV1(h.Codec, mhBinary) } res.latestVersion = docmodel.NewVersion(cids...).String() + if len(cids) > 0 { + res.latestBlobCID = cids[0].String() + } ts := hlc.Timestamp(stmt.ColumnInt64(14) * 1000).Time() res.versionTime = timestamppb.New(ts) @@ -490,41 +914,40 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti res.genesisBlobID = res.blobID } res.rowID = stmt.ColumnInt64(16) - if res.contentType == "comment" { + res.score = winners[res.rowID] + switch res.contentType { + case "comment": res.iri = "hm://" + res.owner + "/" + res.tsid res.commentKey = commentIdentifier{ authorID: stmt.ColumnInt64(17), tsid: res.tsid, } - } else if res.contentType == "contact" { + case "contact": res.iri = "hm://" + subjectID + "/" + res.tsid if err := json.Unmarshal(stmt.ColumnBytes(12), &icon); err != nil { icon.Icon.Value = "" } - } else { + default: res.iri = res.docID } res.icon = icon.Icon.Value - offsets := []int{firstRuneOffset} - for i := firstRuneOffset + 1; i < firstRuneOffset+matchedRunes; i++ { - offsets = append(offsets, i) - } + + // For semantic, no fuzzy matching offsets bodyMatches = append(bodyMatches, fuzzy.Match{ Str: res.content, Index: numResults, Score: 1, - MatchedIndexes: offsets, + MatchedIndexes: []int{}, }) searchResults = append(searchResults, res) numResults++ return nil - }, ftsStr, entityTypeTitle, entityTypeContact, entityTypeDoc, entityTypeComment, loggedAccountID, iriGlob, resultsLmit) + }, string(winnerIDsJSON), loggedAccountID) }); err != nil { return nil, err } - seen := make(map[string]int) - var uniqueResults []searchResult + var uniqueResults []fullDataSearchResult var uniqueBodyMatches []fuzzy.Match for i, res := range searchResults { key := fmt.Sprintf("%s|%s|%s|%s", res.iri, res.blockID, res.rawContent, res.contentType) @@ -545,46 +968,83 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti uniqueBodyMatches = append(uniqueBodyMatches, bm) } } - //fmt.Println("unique results:", len(uniqueResults), "out of", len(searchResults)) bodyMatches = uniqueBodyMatches searchResults = uniqueResults - //after := time.Now() - //elapsed := after.Sub(before) - //fmt.Printf("qGetFTS took %.3f s and returned %d results\n", elapsed.Seconds(), len(bodyMatches)) - matchingEntities := []*entities.Entity{} - //fmt.Println("BeforeParents Elapsed time:", time.Since(start)) - getParentsFcn := func(match fuzzy.Match) ([]string, error) { - parents := make(map[string]interface{}) - breadcrum := strings.Split(strings.TrimPrefix(searchResults[match.Index].iri, "hm://"), "/") - var root string - for i, _ := range breadcrum { - parents["hm://"+strings.Join(breadcrum[:i+1], "/")] = nil - if i == 0 { - root = "hm://" + strings.Join(breadcrum[:i+1], "") + "*" + // Authority-based re-ranking. + if in.AuthorityWeight > 0 { + if in.AuthorityWeight > 1 { + return nil, status.Errorf(codes.InvalidArgument, "authority_weight must be between 0 and 1") + } + + // Sort results by score before authority ranking. + // applyAuthorityRanking uses position as textRank, so results must be sorted by + // text relevance first. Use rowID as tie-breaker for deterministic ordering. + indices := make([]int, len(searchResults)) + for i := range indices { + indices[i] = i + } + slices.SortFunc(indices, func(a, b int) int { + if searchResults[a].score > searchResults[b].score { + return -1 + } + if searchResults[a].score < searchResults[b].score { + return 1 + } + if searchResults[a].rowID < searchResults[b].rowID { + return -1 } + if searchResults[a].rowID > searchResults[b].rowID { + return 1 + } + return 0 + }) + + // Reorder searchResults and bodyMatches according to sorted indices. + sortedResults := make([]fullDataSearchResult, len(searchResults)) + sortedMatches := make([]fuzzy.Match, len(bodyMatches)) + for newIdx, oldIdx := range indices { + sortedResults[newIdx] = searchResults[oldIdx] + bm := bodyMatches[oldIdx] + bm.Index = newIdx + sortedMatches[newIdx] = bm } - var parentTitles []string - if err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { - return sqlitex.ExecTransient(conn, qGetParentsMetadata(), func(stmt *sqlite.Stmt) error { - var title title - iri := stmt.ColumnText(1) - if _, ok := parents[iri]; !ok { - return nil - } - if err := json.Unmarshal(stmt.ColumnBytes(0), &title); err != nil { - return nil - } - if title.Name.Value == match.Str { - return nil - } - parentTitles = append(parentTitles, title.Name.Value) + searchResults = sortedResults + bodyMatches = sortedMatches + + var err error + searchResults, bodyMatches, err = applyAuthorityRanking(ctx, srv.db, searchResults, bodyMatches, in.AuthorityWeight) + if err != nil { + return nil, fmt.Errorf("authority ranking failed: %w", err) + } + } + + matchingEntities := []*entpb.Entity{} + // Pre-fetch all parent metadata in a single query instead of per-result. + parentTitleMap := make(map[string]string) // iri -> title + if err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { + return sqlitex.Exec(conn, qGetParentsMetadata(), func(stmt *sqlite.Stmt) error { + var t title + if err := json.Unmarshal(stmt.ColumnBytes(0), &t); err != nil { return nil - }, root) - }); err != nil { - return nil, err + } + parentTitleMap[stmt.ColumnText(1)] = t.Name.Value + return nil + }, iriGlob) + }); err != nil { + return nil, err + } + + getParentsFcn := func(match fuzzy.Match) []string { + breadcrumb := strings.Split(strings.TrimPrefix(searchResults[match.Index].iri, "hm://"), "/") + var parentTitles []string + for i := range breadcrumb { + parentIRI := "hm://" + strings.Join(breadcrumb[:i+1], "/") + if t, ok := parentTitleMap[parentIRI]; ok && t != match.Str { + parentTitles = append(parentTitles, t) + } } - return parentTitles, nil + return parentTitles } totalLatestBlockTime := time.Duration(0) timesCalled, timesCalled2 := 0, 0 @@ -597,9 +1057,8 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti var movedResources []MovedResource genesisBlobJson := "[" + strings.Join(genesisBlobIDs, ",") + "]" - //fmt.Println("BeforeMovedBlocks Elapsed time:", time.Since(start)) - err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { - return sqlitex.ExecTransient(conn, QGetMovedBlocks(), func(stmt *sqlite.Stmt) error { + err = srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { + return sqlitex.Exec(conn, QGetMovedBlocks(), func(stmt *sqlite.Stmt) error { var heads []head if err := json.Unmarshal(stmt.ColumnBytes(3), &heads); err != nil { return err @@ -637,26 +1096,22 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti } } } - //fmt.Println("BeforeUnrelated Elapsed time:", time.Since(start)) startParents := time.Now() totalGetParentsTime := time.Duration(0) totalDeletedTime := time.Duration(0) totalCommentsTime := time.Duration(0) totalNonCommentsTime := time.Duration(0) + finalResults := []fullDataSearchResult{} for _, match := range bodyMatches { totalGetParentsTime += time.Since(startParents) startParents = time.Now() - var parentTitles []string - var err error if searchResults[match.Index].isDeleted { // Skip deleted resources totalDeletedTime += time.Since(startParents) continue } if searchResults[match.Index].contentType != "contact" { - if parentTitles, err = getParentsFcn(match); err != nil { - return nil, err - } + searchResults[match.Index].parentTitles = getParentsFcn(match) } offsets := make([]int64, len(match.MatchedIndexes)) @@ -665,16 +1120,47 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti } id := searchResults[match.Index].iri + // Version Upgrade Heuristic: + // + // Search results are indexed at specific versions (when content was added/modified). + // To provide useful deep links, we upgrade versions to show the "best" version: + // + // 1. If the indexed version IS already in the document's latest version, keep it + // and mark with "&l" (latest) suffix. + // + // 2. If the indexed version is NOT the latest: + // a. Query for all changes after the indexed version (qGetLatestBlockChange). + // b. Iterate through changes in chronological order: + // - If the SAME BLOCK (same type + blockID) was modified, stop iteration. + // This means the content has changed, so keep the original version. + // - Otherwise, track this change as the latest "unrelated" change. + // c. If no same-block change was found (relatedFound=false): + // - Upgrade to the latest unrelated change's version (content still exists). + // - If that's still not the document's latest, upgrade to latest version. + // d. If same-block change WAS found (relatedFound=true): + // - Keep the original indexed version (content may have changed). + // + // Special cases: + // - Titles have empty blockID, so any title change triggers "same block" detection. + // - Multi-block commits: Multiple blocks modified in same commit share a version. + // We must check for same-block BEFORE updating latestUnrelated to avoid + // incorrectly using a sibling block's version from the same commit. + // + // Fields updated: version, blobID, blobCID, versionTime. + // The "&l" suffix is added later if the final version is in latestVersion. if searchResults[match.Index].version != "" && searchResults[match.Index].contentType != "comment" { - startLatestBlockTime := time.Now() + + // Change tracks version info during the upgrade heuristic iteration. type Change struct { blobID int64 + blobCID string version string ts *timestamppb.Timestamp } latestUnrelated := Change{ blobID: searchResults[match.Index].blobID, + blobCID: searchResults[match.Index].blobCID, version: searchResults[match.Index].version, ts: searchResults[match.Index].versionTime, } @@ -685,13 +1171,14 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti //prevIter = iter relatedFound := false err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { - return sqlitex.ExecTransient(conn, qGetLatestBlockChange(), func(stmt *sqlite.Stmt) error { + return sqlitex.Exec(conn, qGetLatestBlockChange(), func(stmt *sqlite.Stmt) error { iter++ ts := hlc.Timestamp(stmt.ColumnInt64(3) * 1000).Time() blockID := stmt.ColumnText(2) changeType := stmt.ColumnText(4) currentChange := Change{ blobID: stmt.ColumnInt64(0), + blobCID: cid.NewCidV1(uint64(stmt.ColumnInt64(5)), stmt.ColumnBytesUnsafe(6)).String(), version: stmt.ColumnText(1), ts: timestamppb.New(ts), } @@ -700,36 +1187,34 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti } latestUnrelated = currentChange return nil - }, searchResults[match.Index].versionTime.Seconds*1_000+int64(searchResults[match.Index].versionTime.Nanos)/1_000_000, searchResults[match.Index].genesisBlobID, searchResults[match.Index].rowID) + }, searchResults[match.Index].genesisBlobID, searchResults[match.Index].versionTime.Seconds*1_000+int64(searchResults[match.Index].versionTime.Nanos)/1_000_000, searchResults[match.Index].rowID) }) if err != nil && !errors.Is(err, errSameBlockChangeDetected) { - //fmt.Println("Error getting latest block change:", err, "blockID:", searchResults[match.Index].blockID, "genesisBlobID:", searchResults[match.Index].genesisBlobID, "rowID:", searchResults[match.Index].rowID) return nil, err } else if err != nil && errors.Is(err, errSameBlockChangeDetected) { relatedFound = true - //fmt.Println("Found related change:", currentChange, "BlockID:", searchResults[match.Index].blockID) } + // If the latest unrelated change is still not the document's latest version, + // upgrade to the document's latest version and use the latest blob CID. if !relatedFound && !slices.Contains(strings.Split(searchResults[match.Index].latestVersion, "."), latestUnrelated.version) { - //fmt.Println("Found unrelated change:", latestUnrelated, "for:", searchResults[match.Index]) latestUnrelated.version = searchResults[match.Index].latestVersion + latestUnrelated.blobCID = searchResults[match.Index].latestBlobCID + } + + // Only update version if no same-block change was detected. + // When relatedFound is true, the block was modified after the indexed version, + // so we keep the original version (where the content existed). + if !relatedFound { + searchResults[match.Index].version = latestUnrelated.version + searchResults[match.Index].blobID = latestUnrelated.blobID + searchResults[match.Index].blobCID = latestUnrelated.blobCID + searchResults[match.Index].versionTime = latestUnrelated.ts } - /* - if iter == prevIter { - fmt.Println("No iteration", searchResults[match.Index].contentType, searchResults[match.Index].versionTime.Seconds*1_000+int64(searchResults[match.Index].versionTime.Nanos)/1_000_000, searchResults[match.Index].genesisBlobID, searchResults[match.Index].blockID, searchResults[match.Index].blobID) - } - fmt.Println("Latest: ", searchResults[match.Index].latestVersion) - fmt.Println("Latest unrelated: ", latestUnrelated.version) - fmt.Println("Params: ", searchResults[match.Index].versionTime.Seconds*1_000+int64(searchResults[match.Index].versionTime.Nanos)/1_000_000, searchResults[match.Index].genesisBlobID, searchResults[match.Index].rowID) - */ } - searchResults[match.Index].version = latestUnrelated.version - searchResults[match.Index].blobID = latestUnrelated.blobID - searchResults[match.Index].versionTime = latestUnrelated.ts totalLatestBlockTime += time.Since(startLatestBlockTime) if slices.Contains(strings.Split(searchResults[match.Index].latestVersion, "."), searchResults[match.Index].version) { searchResults[match.Index].version += "&l" } - if searchResults[match.Index].version != "" { id += "?v=" + searchResults[match.Index].version } @@ -745,13 +1230,12 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti var isDeleted bool timesCalled2++ err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { - return sqlitex.ExecTransient(conn, qIsDeletedComment(), func(stmt *sqlite.Stmt) error { + return sqlitex.Exec(conn, qIsDeletedComment(), func(stmt *sqlite.Stmt) error { isDeleted = stmt.ColumnInt(0) == 1 return nil }, searchResults[match.Index].commentKey.authorID, searchResults[match.Index].commentKey.tsid) }) if err != nil { - //fmt.Println("Error getting latest block change:", err, "blockID:", searchResults[match.Index].blockID, "genesisBlobID:", searchResults[match.Index].genesisBlobID, "rowID:", searchResults[match.Index].rowID) return nil, err } totalCommentsTime += time.Since(startParents) @@ -760,67 +1244,111 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti continue } } - - matchingEntities = append(matchingEntities, &entities.Entity{ - DocId: searchResults[match.Index].docID, - Id: id, - BlobId: searchResults[match.Index].blobCID, - Type: searchResults[match.Index].contentType, - VersionTime: searchResults[match.Index].versionTime, - Content: match.Str, - ParentNames: parentTitles, - Icon: searchResults[match.Index].icon, - Owner: searchResults[match.Index].owner, - Metadata: searchResults[match.Index].metadata, - }) + searchResults[match.Index].id = id + searchResults[match.Index].content = match.Str + finalResults = append(finalResults, searchResults[match.Index]) } //after = time.Now() - //fmt.Println("BeforeSortingElapsed time:", time.Since(start)) //fmt.Printf("getParentsFcn took %.3f s\n", totalGetParentsTime.Seconds()) //fmt.Printf("totalDeletedTime took %.3f s\n", totalDeletedTime.Seconds()) //fmt.Printf("totalNonCommentsTime took %.3f s\n", totalNonCommentsTime.Seconds()) //fmt.Printf("totalCommentsTime took %.3f s and called %d times\n", totalCommentsTime.Seconds(), timesCalled2) //fmt.Printf("qGetLatestBlockChange took %.3f s and was called %d times and iterated over %d records\n", totalLatestBlockTime.Seconds(), timesCalled, iter) + slices.SortFunc(finalResults, orderBySimilarity) + for _, match := range finalResults { + matchingEntities = append(matchingEntities, &entpb.Entity{ + DocId: match.docID, + Id: match.id, + BlobId: match.blobCID, + Type: match.contentType, + VersionTime: match.versionTime, + Content: match.content, + ParentNames: match.parentTitles, + Icon: match.icon, + Owner: match.owner, + Metadata: match.metadata, + }) + } - sort.Slice(matchingEntities, func(i, j int) bool { - a, b := matchingEntities[i], matchingEntities[j] - - // 1) contacts first - isContactA := a.Type == "contact" - isContactB := b.Type == "contact" - if isContactA != isContactB { - return isContactA + // Paginate if page_size is set. When 0, return everything (backwards compatible). + var nextPageToken string + if in.PageSize > 0 { + var cursor struct { + Offset int `json:"o"` } - - // 2) then titles - isTitleA := a.Type == "title" - isTitleB := b.Type == "title" - if isTitleA != isTitleB { - return isTitleA + if in.PageToken != "" { + if err := apiutil.DecodePageToken(in.PageToken, &cursor, nil); err != nil { + return nil, status.Errorf(codes.InvalidArgument, "invalid page_token: %v", err) + } } - if isTitleA && isTitleB { - lenA := utf8.RuneCountInString(a.Content) - lenB := utf8.RuneCountInString(b.Content) - if lenA != lenB { - return lenA < lenB + if cursor.Offset >= len(matchingEntities) { + matchingEntities = nil + } else { + end := cursor.Offset + int(in.PageSize) + if end < len(matchingEntities) { + nextCursor := struct { + Offset int `json:"o"` + }{Offset: end} + nextPageToken = apiutil.EncodePageToken(nextCursor, nil) + matchingEntities = matchingEntities[cursor.Offset:end] + } else { + matchingEntities = matchingEntities[cursor.Offset:] } } + } + + return &entpb.SearchEntitiesResponse{ + Entities: matchingEntities, + NextPageToken: nextPageToken, + }, nil +} + +func orderByTitle(a, b fullDataSearchResult) int { + // 1) contacts first + isContactA := a.contentType == "contact" + isContactB := b.contentType == "contact" + if isContactA != isContactB { + if isContactA { + return -1 + } + return 1 + } - // 3) then by DocId (lexicographically) - if a.DocId != b.DocId { - return a.DocId < b.DocId + // 2) then titles + isTitleA := a.contentType == "title" + isTitleB := b.contentType == "title" + if isTitleA != isTitleB { + if isTitleA { + return -1 } + return 1 + } - // 4) finally by VersionTime descending - return a.VersionTime.AsTime().After(b.VersionTime.AsTime()) - }) + // 3) everything else (including within contacts and titles) by Score descending (higher first) + if a.score != b.score { + if a.score > b.score { + return -1 // a comes first (higher score) + } + return 1 // b comes first (higher score) + } + return 0 +} - return &entities.SearchEntitiesResponse{Entities: matchingEntities}, nil +// orderBySimilarity sorts entities by similarity score descending (higher scores first). +func orderBySimilarity(a, b fullDataSearchResult) int { + // Higher scores first (descending order) + if a.score > b.score { + return -1 + } else if a.score < b.score { + return 1 + } + // If scores are equal, fall back to title ordering + return orderByTitle(a, b) } // DeleteEntity implements the corresponding gRPC method. -// func (api *Server) DeleteEntity(ctx context.Context, in *entities.DeleteEntityRequest) (*emptypb.Empty, error) { +// func (api *Server) DeleteEntity(ctx context.Context, in *entpb.DeleteEntityRequest) (*emptypb.Empty, error) { // var meta string // var qGetResourceMetadata = dqb.Str(` // SELECT meta from meta_view @@ -834,7 +1362,7 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti // eid := hyper.EntityID(in.Id) // err := api.blobs.Query(ctx, func(conn *sqlite.Conn) error { -// return sqlitex.ExecTransient(conn, qGetResourceMetadata(), func(stmt *sqlite.Stmt) error { +// return sqlitex.Exec(conn, qGetResourceMetadata(), func(stmt *sqlite.Stmt) error { // meta = stmt.ColumnText(0) // return nil // }, in.Id) @@ -902,7 +1430,7 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti // } // // UndeleteEntity implements the corresponding gRPC method. -// func (api *Server) UndeleteEntity(ctx context.Context, in *entities.UndeleteEntityRequest) (*emptypb.Empty, error) { +// func (api *Server) UndeleteEntity(ctx context.Context, in *entpb.UndeleteEntityRequest) (*emptypb.Empty, error) { // if in.Id == "" { // return nil, status.Errorf(codes.InvalidArgument, "must specify entity ID to restore") // } @@ -915,9 +1443,9 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti // } // // ListDeletedEntities implements the corresponding gRPC method. -// func (api *Server) ListDeletedEntities(ctx context.Context, _ *entities.ListDeletedEntitiesRequest) (*entities.ListDeletedEntitiesResponse, error) { -// resp := &entities.ListDeletedEntitiesResponse{ -// DeletedEntities: make([]*entities.DeletedEntity, 0), +// func (api *Server) ListDeletedEntities(ctx context.Context, _ *entpb.ListDeletedEntitiesRequest) (*entpb.ListDeletedEntitiesResponse, error) { +// resp := &entpb.ListDeletedEntitiesResponse{ +// DeletedEntities: make([]*entpb.DeletedEntity, 0), // } // err := api.blobs.Query(ctx, func(conn *sqlite.Conn) error { @@ -926,7 +1454,7 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti // return err // } // for _, entity := range list { -// resp.DeletedEntities = append(resp.DeletedEntities, &entities.DeletedEntity{ +// resp.DeletedEntities = append(resp.DeletedEntities, &entpb.DeletedEntity{ // Id: entity.DeletedResourcesIRI, // DeleteTime: ×tamppb.Timestamp{Seconds: entity.DeletedResourcesDeleteTime}, // DeletedReason: entity.DeletedResourcesReason, @@ -940,7 +1468,7 @@ func (srv *Server) SearchEntities(ctx context.Context, in *entities.SearchEntiti // } // ListEntityMentions implements listing mentions of an entity in other resources. -func (api *Server) ListEntityMentions(ctx context.Context, in *entities.ListEntityMentionsRequest) (*entities.ListEntityMentionsResponse, error) { +func (srv *Server) ListEntityMentions(ctx context.Context, in *entpb.ListEntityMentionsRequest) (*entpb.ListEntityMentionsResponse, error) { if in.Id == "" { return nil, errutil.MissingArgument("id") } @@ -963,15 +1491,14 @@ func (api *Server) ListEntityMentions(ctx context.Context, in *entities.ListEnti in.PageSize = 10 } - resp := &entities.ListEntityMentionsResponse{} + resp := &entpb.ListEntityMentionsResponse{} var genesisBlobIDs []string var deletedList []string - if err := api.db.WithSave(ctx, func(conn *sqlite.Conn) error { + if err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { var eid int64 - if err := sqlitex.ExecTransient(conn, qEntitiesLookupID(), func(stmt *sqlite.Stmt) error { + if err := sqlitex.Exec(conn, qEntitiesLookupID(), func(stmt *sqlite.Stmt) error { eid = stmt.ColumnInt64(0) return nil - }, in.Id); err != nil { return err } @@ -983,7 +1510,7 @@ func (api *Server) ListEntityMentions(ctx context.Context, in *entities.ListEnti var lastCursor mentionsCursor var count int32 - if err := sqlitex.ExecTransient(conn, qListMentions(in.ReverseOrder), func(stmt *sqlite.Stmt) error { + if err := sqlitex.Exec(conn, qListMentions(in.ReverseOrder), func(stmt *sqlite.Stmt) error { // We query for pageSize + 1 items to know if there's more items on the next page, // because if not we don't need to return the page token in the response. if count == in.PageSize { @@ -1026,11 +1553,11 @@ func (api *Server) ListEntityMentions(ctx context.Context, in *entities.ListEnti deletedList = append(deletedList, source) } - resp.Mentions = append(resp.Mentions, &entities.Mention{ + resp.Mentions = append(resp.Mentions, &entpb.Mention{ Source: source, SourceType: blobType, SourceContext: anchor, - SourceBlob: &entities.Mention_BlobInfo{ + SourceBlob: &entpb.Mention_BlobInfo{ Cid: sourceBlob, Author: author, CreateTime: timestamppb.New(ts), @@ -1054,8 +1581,8 @@ func (api *Server) ListEntityMentions(ctx context.Context, in *entities.ListEnti } genesisBlobJson := "[" + strings.Join(genesisBlobIDs, ",") + "]" var movedResources []MovedResource - err := api.db.WithSave(ctx, func(conn *sqlite.Conn) error { - return sqlitex.ExecTransient(conn, QGetMovedBlocks(), func(stmt *sqlite.Stmt) error { + err := srv.db.WithSave(ctx, func(conn *sqlite.Conn) error { + return sqlitex.Exec(conn, QGetMovedBlocks(), func(stmt *sqlite.Stmt) error { movedResources = append(movedResources, MovedResource{ NewIri: stmt.ColumnText(0), OldIri: stmt.ColumnText(1), @@ -1076,7 +1603,7 @@ func (api *Server) ListEntityMentions(ctx context.Context, in *entities.ListEnti } seenMentions := make(map[string]bool) - uniqueMentions := make([]*entities.Mention, 0, len(resp.Mentions)) + uniqueMentions := make([]*entpb.Mention, 0, len(resp.Mentions)) for _, m := range resp.Mentions { key := fmt.Sprintf("%s|%s|%s|%s|%t", m.Source, m.SourceType, m.TargetVersion, m.TargetFragment, m.IsExactVersion) if !seenMentions[key] && !slices.Contains(deletedList, m.Source) { @@ -1251,7 +1778,7 @@ func indexOfQueryPattern(haystack, pattern string) (startRunes, startChars, matc re := regexp.MustCompile(regexPattern) loc := re.FindStringIndex(haystack) if loc == nil { - return -1, -1, 0, 0 + return 0, 0, 0, 0 } // The start index in runes. startRunes = utf8.RuneCountInString(haystack[:loc[0]]) diff --git a/backend/api/entities/v1alpha/entities_test.go b/backend/api/entities/v1alpha/entities_test.go new file mode 100644 index 000000000..75a897a7d --- /dev/null +++ b/backend/api/entities/v1alpha/entities_test.go @@ -0,0 +1,63 @@ +package entities + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsValidIriFilter(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + valid bool + }{ + {"valid single doc", "hm://abc123/cars/honda", true}, + {"valid subtree glob", "hm://abc123/cars/*", true}, + {"valid account glob", "hm://abc123*", true}, + {"valid all", "hm://*", true}, + {"valid with dashes", "hm://my-account/my-doc", true}, + {"valid with dots", "hm://acc.123/path", true}, + {"valid question mark glob", "hm://abc/?", true}, + {"valid bracket glob", "hm://abc/[abc]", true}, + {"invalid no prefix", "abc://bad", false}, + {"invalid empty", "", false}, + {"invalid sql injection", "hm://; DROP TABLE fts;--", false}, + {"invalid spaces", "hm://acc/path with spaces", false}, + {"invalid quotes", "hm://acc/path'quote", false}, + {"invalid parens", "hm://acc/path()", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := isValidIriFilter(tt.input) + require.Equal(t, tt.valid, got, "isValidIriFilter(%q) must be %v", tt.input, tt.valid) + }) + } +} + +func TestBuildRankMap(t *testing.T) { + t.Parallel() + + results := []fullDataSearchResult{ + {iri: "hm://a/doc1"}, + {iri: "hm://a/doc2"}, + {iri: "hm://a/doc3"}, + {iri: "hm://a/doc1"}, // Duplicate IRI — must be deduped. + } + + scores := map[string]int{ + "hm://a/doc1": 10, + "hm://a/doc2": 50, + "hm://a/doc3": 30, + } + + ranks := buildRankMap(results, func(r fullDataSearchResult) int { return scores[r.iri] }) + + require.Equal(t, 1, ranks["hm://a/doc2"], "doc2 has highest score (50) so must be rank 1") + require.Equal(t, 2, ranks["hm://a/doc3"], "doc3 has score 30 so must be rank 2") + require.Equal(t, 3, ranks["hm://a/doc1"], "doc1 has lowest score (10) so must be rank 3") + require.Len(t, ranks, 3, "must have 3 unique IRIs") +} diff --git a/backend/blob/blob_change.go b/backend/blob/blob_change.go index 760dcd34a..cf19ed7bb 100644 --- a/backend/blob/blob_change.go +++ b/backend/blob/blob_change.go @@ -510,7 +510,6 @@ func indexChange(ictx *indexingCtx, id int64, eb Encoded[*Change]) error { } if content == "" { continue - //fmt.Println("WARNING: empty content for block", blk, "in change", sb.CID, "with id", id, "and genesis", sb.GenesisBlob.Hash().String()) } else if err := dbFTSInsertOrReplace(ictx.conn, content, "document", id, blk, sb.CID.String(), sb.Ts, sb.GenesisBlob.Hash().String()); err != nil { return fmt.Errorf("failed to insert record in fts table: %w", err) } diff --git a/backend/blob/blob_comment_test.go b/backend/blob/blob_comment_test.go index 30d0ea996..0e9c6b93d 100644 --- a/backend/blob/blob_comment_test.go +++ b/backend/blob/blob_comment_test.go @@ -34,7 +34,7 @@ func TestCommentOldEncoding(t *testing.T) { require.NoError(t, err) var comment map[string]any - cbornode.DecodeInto(data, &comment) + require.NoError(t, cbornode.DecodeInto(data, &comment)) signer := core.Principal(comment["signer"].([]byte)) sig := core.Signature(comment["sig"].([]byte)) diff --git a/backend/cmd/monitord/server/server.go b/backend/cmd/monitord/server/server.go index ce83d848f..b67bb15e5 100644 --- a/backend/cmd/monitord/server/server.go +++ b/backend/cmd/monitord/server/server.go @@ -90,7 +90,7 @@ func (s *Srv) Start(numPings int, scanPeriod time.Duration, peerTimeout time.Dur s.ticker = time.NewTicker(scanPeriod) s.numPings = numPings s.templateFile = templateFile - go s.httpServer.ListenAndServe() + go func() { _ = s.httpServer.ListenAndServe() }() go s.scan(peerTimeout) } diff --git a/backend/cmd/seed-daemon/Dockerfile b/backend/cmd/seed-daemon/Dockerfile index 138519795..31e32f05b 100644 --- a/backend/cmd/seed-daemon/Dockerfile +++ b/backend/cmd/seed-daemon/Dockerfile @@ -1,21 +1,45 @@ # Build from the root with `docker build . -f ./backend/cmd/seed-daemon/Dockerfile`. -FROM golang:1.25.4-alpine AS builder + +# Use Debian Trixie for glibc compatibility and newer Vulkan SDK (1.4.309+). +# Alpine's musl libc is incompatible with GCC 14's libstdc++ which uses +# glibc-specific symbols (__libc_single_threaded, __isoc23_* functions). +# Bookworm's Vulkan SDK (1.3.239) is too old for llama.cpp's ggml-vulkan. +FROM golang:1.25-trixie AS builder WORKDIR /code ARG COMMIT_HASH ARG BRANCH ARG DATE COPY go.mod go.sum ./ +COPY backend/util/llama-go/go.mod backend/util/llama-go/go.sum ./backend/util/llama-go/ RUN go mod download COPY backend ./backend COPY monitoring ./monitoring -RUN apk add build-base + +# Install build dependencies for llama.cpp with Vulkan GPU support. +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake g++ libvulkan-dev glslc \ + && rm -rf /var/lib/apt/lists/* + +# Build llama.cpp with Vulkan GPU support (falls back to CPU at runtime if no GPU available). +# Clean any pre-existing build artifacts copied from host to ensure fresh build. +WORKDIR /code/backend/util/llama-go +RUN rm -rf build llama.cpp/*.o *.o *.a *.so && \ + BUILD_TYPE=vulkan CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF" make libbinding.a + +# Build seed-daemon with llama.cpp support. +WORKDIR /code +ENV LIBRARY_PATH=/code/backend/util/llama-go +ENV C_INCLUDE_PATH=/code/backend/util/llama-go RUN go install -ldflags="-X 'seed/backend/daemon.commit=$COMMIT_HASH' -X 'seed/backend/daemon.branch=$BRANCH' -X 'seed/backend/daemon.date=$DATE'" ./backend/cmd/seed-daemon/ -FROM alpine:latest -RUN apk add rsync +FROM debian:trixie-slim +RUN apt-get update && apt-get install -y --no-install-recommends \ + rsync libvulkan1 ca-certificates \ + && rm -rf /var/lib/apt/lists/* COPY --from=builder /go/bin/seed-daemon /usr/local/bin/seed-daemon COPY --from=builder /code/monitoring/grafana /monitoring/grafana COPY --from=builder /code/monitoring/prometheus /monitoring/prometheus EXPOSE 55000 55001 55002 ENV SEED_PUBLIC_ONLY=true +ENV LLAMA_LOG=error CMD ["/usr/local/bin/seed-daemon"] diff --git a/backend/config/config.go b/backend/config/config.go index 4eefd1b02..f9d91b2c6 100644 --- a/backend/config/config.go +++ b/backend/config/config.go @@ -4,12 +4,15 @@ package config import ( "flag" "fmt" + "net/url" "os" "seed/backend/ipfs" "seed/backend/util/must" "strings" "time" + "seed/backend/llm" + "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" ) @@ -58,6 +61,7 @@ type Config struct { HTTP HTTP GRPC GRPC P2P P2P + LLM LLM Lndhub Lndhub Syncing Syncing Debug Debug @@ -74,6 +78,7 @@ func (c *Config) BindFlags(fs *flag.FlagSet) { c.HTTP.BindFlags(fs) c.GRPC.BindFlags(fs) c.P2P.BindFlags(fs) + c.LLM.BindFlags(fs) c.Lndhub.BindFlags(fs) c.Syncing.BindFlags(fs) c.Debug.BindFlags(fs) @@ -86,6 +91,7 @@ func Default() Config { HTTP: HTTP{}.Default(), GRPC: GRPC{}.Default(), P2P: P2P{}.Default(), + LLM: LLM{}.Default(), Lndhub: Lndhub{}.Default(), Syncing: Syncing{}.Default(), Debug: Debug{}.Default(), @@ -135,6 +141,35 @@ func newAddrsFlag(val []multiaddr.Multiaddr, p *[]multiaddr.Multiaddr) flag.Valu return (*addrsFlag)(p) } +type urlFlag url.URL + +func (al *urlFlag) String() string { + if al == nil { + return "" + } + + return (*url.URL)(al).String() +} + +func (al *urlFlag) Set(s string) error { + trimmed := strings.TrimSpace(s) + if trimmed == "" { + return fmt.Errorf("URL flag value cannot be empty") + } + parsedURL, err := url.Parse(trimmed) + if err != nil { + return err + } + + *al = urlFlag(*parsedURL) + return nil +} + +func newURLFlag(val url.URL, p *url.URL) flag.Value { + *p = val + return (*urlFlag)(p) +} + // HTTP configuration. type HTTP struct { Port int @@ -167,6 +202,84 @@ func (c *GRPC) BindFlags(fs *flag.FlagSet) { fs.IntVar(&c.Port, "grpc.port", c.Port, "Port for the gRPC server") } +// Embedder configures the embedding indexer. +type Embedder struct { + // PeriodicInterval is the period between each indexing run. Default is 1 minute. + PeriodicInterval time.Duration + // SleepBetweenPasses is the time to sleep between passes when indexing. + SleepBetweenPasses time.Duration + // IndexPassSize is the number of FTS rows to keep in memory per pass. Default is 100. + IndexPassSize int + // Model is the LLM model to use for embeddings. + Model string + // DocumentPrefix is the prefix to add to document texts before embedding. + DocumentPrefix string + // QueryPrefix is the prefix to add to query texts before embedding. + QueryPrefix string + // Enabled indicates whether the embedder is enabled. + Enabled bool +} + +// BackendCfg configures the LLM backend connection. +type BackendCfg struct { + // URL is the base URL of the Ollama server. + // It could be an HTTP URL or a file URL depending on the backend. + URL url.URL + + // SleepBetweenBatches is the time to wait between embedding batches. + SleepBetweenBatches time.Duration + + // BatchSize is the number of inputs to process in a single batch. + BatchSize int +} + +// Backend wraps the backend configuration. +type Backend struct { + Cfg BackendCfg +} + +// LLM configuration. +type LLM struct { + Backend Backend + Embedding Embedder +} + +// Default returns the default LLM configuration. +func (c LLM) Default() LLM { + return LLM{ + Backend: Backend{ + Cfg: BackendCfg{ + URL: url.URL{}, // empty = use embedded llamacpp model + SleepBetweenBatches: 750 * time.Millisecond, + BatchSize: 16, + }, + }, + Embedding: Embedder{ + PeriodicInterval: llm.DefaultEmbeddingRunInterval, + SleepBetweenPasses: llm.DefaultEmbeddingSleepBetweenPasses, + IndexPassSize: llm.DefaultEmbeddingIndexPassSize, + Model: llm.DefaultEmbeddingModel, + DocumentPrefix: "", + QueryPrefix: "", + Enabled: false, + }, + } +} + +// BindFlags binds the flags to the given FlagSet. +func (c *LLM) BindFlags(fs *flag.FlagSet) { + fs.Var(newURLFlag(c.Backend.Cfg.URL, &c.Backend.Cfg.URL), "llm.backend.url", "Empty = embedded model, or Ollama URL (http://localhost:11434), or file URL (file:///path/to.gguf)") + fs.DurationVar(&c.Backend.Cfg.SleepBetweenBatches, "llm.backend.sleep-between-batches", c.Backend.Cfg.SleepBetweenBatches, "Wait time between embedding batches") + fs.IntVar(&c.Backend.Cfg.BatchSize, "llm.backend.batch-size", c.Backend.Cfg.BatchSize, "How many FTS rows to scan at once") + fs.DurationVar(&c.Embedding.PeriodicInterval, "llm.embedding.periodic-interval", c.Embedding.PeriodicInterval, "Interval between embedding runs") + fs.DurationVar(&c.Embedding.SleepBetweenPasses, "llm.embedding.sleep-between-pass", c.Embedding.SleepBetweenPasses, "Wait time between embedding passes") + fs.IntVar(&c.Embedding.IndexPassSize, "llm.embedding.index-pass-size", c.Embedding.IndexPassSize, "How many FTS rows to scan at once") + fs.StringVar(&c.Embedding.Model, "llm.embedding.model", c.Embedding.Model, "Embedding model to use. Only applicable for Ollama backend") + fs.StringVar(&c.Embedding.DocumentPrefix, "llm.embedding.document-prefix", c.Embedding.DocumentPrefix, "Prefix to add to document texts before embedding") + fs.StringVar(&c.Embedding.QueryPrefix, "llm.embedding.query-prefix", c.Embedding.QueryPrefix, "Prefix to add to query texts before embedding") + fs.BoolVar(&c.Embedding.Enabled, "llm.embedding.enabled", c.Embedding.Enabled, "Whether the embedding indexer is enabled") +} + // Lndhub related config. type Lndhub struct { Mainnet bool diff --git a/backend/crdt/tree_test.go b/backend/crdt/tree_test.go index e8575aa3a..33b47c280 100644 --- a/backend/crdt/tree_test.go +++ b/backend/crdt/tree_test.go @@ -289,7 +289,7 @@ func TestUndoRedo(t *testing.T) { testPlacement(t, want, d.Iterator()) for i := 1; i < len(d.movesLog); i++ { - d.redoMove(d.movesLog[i], i) + _ = d.redoMove(d.movesLog[i], i) } want = []testWant{ diff --git a/backend/daemon/daemon.go b/backend/daemon/daemon.go index 59a51e869..24466b702 100644 --- a/backend/daemon/daemon.go +++ b/backend/daemon/daemon.go @@ -21,6 +21,10 @@ import ( daemon "seed/backend/genproto/daemon/v1alpha" "seed/backend/hmnet" "seed/backend/hmnet/syncing" + embeddings "seed/backend/llm" + "seed/backend/llm/backends" + "seed/backend/llm/backends/llamacpp" + "seed/backend/llm/backends/ollama" "seed/backend/logging" "seed/backend/storage" "seed/backend/util/cleanup" @@ -195,7 +199,6 @@ func Load(ctx context.Context, cfg config.Config, r *storage.Store, oo ...Option if _, err := a.taskMgr.DeleteTask(taskID); err != nil { a.log.Warn("failed to delete reindexing task", zap.Error(err)) } - return nil }) } @@ -213,8 +216,19 @@ func Load(ctx context.Context, cfg config.Config, r *storage.Store, oo ...Option dlink := devicelink.NewService(a.Net.Libp2p().Host, a.Storage.KeyStore(), a.Index, logging.New("seed/devicelink", cfg.LogLevel)) + embedder, err := initLLM(ctx, cfg.LLM, a.Storage.DB(), logging.New("seed/llm", cfg.LogLevel), a.taskMgr) + if err != nil { + return nil, err + } + + // Convert typed nil to untyped nil for proper interface nil check downstream. + var lightEmbedder embeddings.LightEmbedder + if embedder != nil { + lightEmbedder = embedder + } + a.GRPCServer, a.GRPCListener, a.RPC, err = initGRPC(cfg.Base, cfg.GRPC.Port, &a.clean, a.g, a.Storage, a.Index, a.Net, - a.Syncing, activitySrv, cfg.LogLevel, cfg.Lndhub.Mainnet, opts.grpc, dlink, a.taskMgr) + a.Syncing, activitySrv, cfg.LogLevel, cfg.Lndhub.Mainnet, opts.grpc, dlink, a.taskMgr, lightEmbedder) if err != nil { return nil, err } @@ -235,9 +249,7 @@ func Load(ctx context.Context, cfg config.Config, r *storage.Store, oo ...Option if err != nil { return nil, err } - a.setupLogging(ctx, cfg) - select { case <-ctx.Done(): return nil, ctx.Err() @@ -376,6 +388,7 @@ func initGRPC( opts grpcOpts, dlink *devicelink.Service, taskMgr *taskmanager.TaskManager, + embedder embeddings.LightEmbedder, ) (srv *grpc.Server, lis net.Listener, apis api.Server, err error) { lis, err = net.Listen("tcp", ":"+strconv.Itoa(port)) if err != nil { @@ -383,7 +396,7 @@ func initGRPC( } srv = grpc.NewServer(opts.serverOptions...) - apis = api.New(cfg, repo, idx, node, sync, activity, LogLevel, isMainnet, dlink, taskMgr) + apis = api.New(cfg, repo, idx, node, sync, activity, LogLevel, isMainnet, dlink, taskMgr, embedder) apis.Register(srv) for _, extra := range opts.extraServices { @@ -402,6 +415,76 @@ func initGRPC( return } +func initLLM( + ctx context.Context, + cfg config.LLM, + db *sqlitex.Pool, + log *zap.Logger, + tskMgr *taskmanager.TaskManager, +) (*embeddings.Embedder, error) { + if !cfg.Embedding.Enabled { + log.Info("LLM embedding indexer is disabled") + return nil, nil + } + + log.Info("Initializing LLM embedding indexer", + zap.String("model", cfg.Embedding.Model), + zap.String("documentPrefix", cfg.Embedding.DocumentPrefix), + zap.String("queryPrefix", cfg.Embedding.QueryPrefix), + zap.Duration("periodicInterval", cfg.Embedding.PeriodicInterval), + zap.Duration("SleepBetweenPasses", cfg.Embedding.SleepBetweenPasses), + zap.Int("indexPassSize", cfg.Embedding.IndexPassSize), + ) + var backend backends.Backend + switch cfg.Backend.Cfg.URL.Scheme { + case "", "file": + llamaCppOpts := []llamacpp.Option{ + llamacpp.WithWaitBetweenBatches(cfg.Backend.Cfg.SleepBetweenBatches), + llamacpp.WithBatchSize(cfg.Backend.Cfg.BatchSize), + } + + llamacpp, err := llamacpp.NewClient(cfg.Backend.Cfg.URL, llamaCppOpts...) + if err != nil { + return nil, err + } + if cfg.Backend.Cfg.URL.Scheme == "" { + log.Info("LLM Backend initialized with embedded model") + } else { + log.Info("LLM Backend initialized", zap.String("LlamaCpp File URL", cfg.Backend.Cfg.URL.String())) + } + backend = llamacpp + case "http", "https": + ollamaOpts := []ollama.Option{ + ollama.WithWaitBetweenBatches(cfg.Backend.Cfg.SleepBetweenBatches), + ollama.WithBatchSize(cfg.Backend.Cfg.BatchSize), + } + + ollama, err := ollama.NewClient(cfg.Backend.Cfg.URL, ollamaOpts...) + if err != nil { + return nil, err + } + log.Info("LLM Backend initialized", zap.String("Ollama URL", cfg.Backend.Cfg.URL.String())) + backend = ollama + default: + return nil, errors.New("unsupported LLM backend URL scheme: " + cfg.Backend.Cfg.URL.Scheme) + } + embedderOpts := []embeddings.EmbedderOption{ + + embeddings.WithIndexPassSize(cfg.Embedding.IndexPassSize), + embeddings.WithDocumentPrefix(cfg.Embedding.DocumentPrefix), + embeddings.WithQueryPrefix(cfg.Embedding.QueryPrefix), + embeddings.WithSleepPerPass(cfg.Embedding.SleepBetweenPasses), + embeddings.WithInterval(cfg.Embedding.PeriodicInterval), + embeddings.WithModel(cfg.Embedding.Model), + } + embedder, err := embeddings.NewEmbedder(db, backend, log, tskMgr, embedderOpts...) + if err != nil { + return nil, err + } + embedder.Init(ctx) + return embedder, nil +} + // WithMiddleware generates an grpc option with the given middleware. func WithMiddleware(i grpc.UnaryServerInterceptor) grpc.ServerOption { return grpc.UnaryInterceptor(i) diff --git a/backend/daemon/daemon_e2e_test.go b/backend/daemon/daemon_e2e_test.go index 8bf653d20..9bf49161b 100644 --- a/backend/daemon/daemon_e2e_test.go +++ b/backend/daemon/daemon_e2e_test.go @@ -3030,3 +3030,862 @@ func pullDocument(t *testing.T, app *App, account, path, wantVersion string) { time.Sleep(100 * time.Millisecond) } } + +func TestSearchEntitiesFilters(t *testing.T) { + t.Parallel() + alice := makeTestApp(t, "alice", makeTestConfig(t), true) + ctx := context.Background() + aliceIdentity := coretest.NewTester("alice") + aliceAccount := aliceIdentity.Account.PublicKey.String() + + // Create documents with distinct, searchable content at different paths. + _, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "", + SigningKeyName: "main", + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_SetMetadata_{ + SetMetadata: &documents.DocumentChange_SetMetadata{Key: "title", Value: "Alice Home Page"}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b1", Parent: "", LeftSibling: ""}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "Welcome to my page"}, + }}, + }, + }) + require.NoError(t, err) + + _, err = alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/cars/honda", + SigningKeyName: "main", + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_SetMetadata_{ + SetMetadata: &documents.DocumentChange_SetMetadata{Key: "title", Value: "Why Honda rocks"}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b1", Parent: "", LeftSibling: ""}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "Honda reliability is legendary"}, + }}, + }, + }) + require.NoError(t, err) + + _, err = alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/cars/toyota", + SigningKeyName: "main", + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_SetMetadata_{ + SetMetadata: &documents.DocumentChange_SetMetadata{Key: "title", Value: "Why Toyota rocks"}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b1", Parent: "", LeftSibling: ""}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "Toyota durability is unmatched"}, + }}, + }, + }) + require.NoError(t, err) + + _, err = alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/bikes/yamaha", + SigningKeyName: "main", + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_SetMetadata_{ + SetMetadata: &documents.DocumentChange_SetMetadata{Key: "title", Value: "Why Yamaha rocks"}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b1", Parent: "", LeftSibling: ""}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "Yamaha speed is unreal"}, + }}, + }, + }) + require.NoError(t, err) + + t.Run("IriFilterSubtree", func(t *testing.T) { + // Search with iri_filter scoped to /cars/* — must only return honda and toyota. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "rocks", + IncludeBody: true, + IriFilter: "hm://" + aliceAccount + "/cars/*", + }) + require.NoError(t, err) + require.Greater(t, len(res.Entities), 0, "must return results under /cars/*") + for _, e := range res.Entities { + require.Contains(t, e.Id, "/cars/", "all results must be under /cars/ subtree") + } + }) + + t.Run("IriFilterInvalid", func(t *testing.T) { + // Invalid pattern must return error. + _, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "rocks", + IriFilter: "not-hm://injection; DROP TABLE fts", + }) + require.Error(t, err, "invalid iri_filter must be rejected") + }) + + t.Run("DeprecatedAccountUidFallback", func(t *testing.T) { + // Empty iri_filter + account_uid set must still work (legacy). + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "rocks", + AccountUid: aliceAccount, + }) + require.NoError(t, err) + require.Greater(t, len(res.Entities), 0, "must return results for the account") + }) + + t.Run("ContentTypeFilterExplicit", func(t *testing.T) { + // content_type_filters = [TITLE] must only return title matches. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "rocks", + IncludeBody: true, + ContentTypeFilter: []entities.ContentTypeFilter{entities.ContentTypeFilter_CONTENT_TYPE_TITLE}, + }) + require.NoError(t, err) + require.Greater(t, len(res.Entities), 0, "must return title results") + for _, e := range res.Entities { + require.Equal(t, "title", e.Type, "must only return title results when filter is explicit") + } + }) + + t.Run("ContentTypeLegacyWithBody", func(t *testing.T) { + // Empty content_type_filters + include_body=true must search body content. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "reliability", + IncludeBody: true, + }) + require.NoError(t, err) + require.Greater(t, len(res.Entities), 0, "must find body content with include_body") + }) + + t.Run("ContentTypeTitleOnlyDefault", func(t *testing.T) { + // Empty content_type_filters + include_body=false must only search titles. + // "reliability" is only in the body → no results. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "reliability", + }) + require.NoError(t, err) + require.Len(t, res.Entities, 0, "must not find body content without include_body") + }) + + t.Run("AuthorityWeightInvalid", func(t *testing.T) { + // authority_weight > 1 must be rejected. + _, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "rocks", + AuthorityWeight: 1.5, + }) + require.Error(t, err, "authority_weight > 1 must be rejected") + }) + + t.Run("AuthorityWeightZero", func(t *testing.T) { + // authority_weight = 0 (default) must work normally. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "rocks", + }) + require.NoError(t, err) + require.Greater(t, len(res.Entities), 0, "must return results with default authority_weight") + }) + + t.Run("AuthorityWeightValid", func(t *testing.T) { + // authority_weight within range must not error. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "rocks", + AuthorityWeight: 0.3, + }) + require.NoError(t, err) + require.Greater(t, len(res.Entities), 0, "must return results with valid authority_weight") + }) +} + +// parseEntityVersion extracts version info from an entity ID. +// Entity IDs have the format: "hm://account/path?v=#blockId[offset:end]" +// The version may end with "&l" if it represents the latest version. +// Returns the version string (without &l suffix) and whether it has the latest marker. +func parseEntityVersion(entityID string) (version string, isLatest bool) { + // Find the version parameter. + vIdx := strings.Index(entityID, "?v=") + if vIdx == -1 { + return "", false + } + + // Extract everything after "?v=". + versionPart := entityID[vIdx+3:] + + // Remove the fragment (block ID) if present. + if hashIdx := strings.Index(versionPart, "#"); hashIdx != -1 { + versionPart = versionPart[:hashIdx] + } + + // Check for latest marker. + if strings.HasSuffix(versionPart, "&l") { + return strings.TrimSuffix(versionPart, "&l"), true + } + + return versionPart, false +} + +func TestSearchVersionConsistency(t *testing.T) { + t.Parallel() + + // Setup with embeddings enabled for semantic/hybrid search. + cfg := makeTestConfig(t) + cfg.LLM.Embedding.Enabled = true + alice := makeTestApp(t, "alice", cfg, true) + ctx := context.Background() + aliceIdentity := coretest.NewTester("alice") + aliceAccount := aliceIdentity.Account.PublicKey.String() + + // ===== DOCUMENT 1 SETUP: /version-test-animals ===== + // This document has 3 blocks (b1, b2, b3) with 5 changes: + // C1: b1="alpha dinosaur", b2="static forever", b3="beta elephant" + // C2: b1="beta elephant" (b2, b3 untouched) + // C3: b3="beta giraffe" (b1, b2 untouched) + // C4: b1="gamma hippo" (b2, b3 untouched) + // C5: b3="delta iguana" (b1, b2 untouched) + + // C1: Create document with 3 blocks. + c1, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/version-test-animals", + SigningKeyName: "main", + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_SetMetadata_{ + SetMetadata: &documents.DocumentChange_SetMetadata{Key: "title", Value: "Animal Versions Test"}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b1", Parent: "", LeftSibling: ""}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b2", Parent: "", LeftSibling: "b1"}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b3", Parent: "", LeftSibling: "b2"}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "alpha dinosaur"}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b2", Type: "paragraph", Text: "static forever"}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b3", Type: "paragraph", Text: "beta elephant"}, + }}, + }, + }) + require.NoError(t, err) + + // C2: Modify b1 only. + c2, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/version-test-animals", + SigningKeyName: "main", + BaseVersion: c1.Version, + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "beta elephant"}, + }}, + }, + }) + require.NoError(t, err) + + // C3: Modify b3 only. + c3, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/version-test-animals", + SigningKeyName: "main", + BaseVersion: c2.Version, + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b3", Type: "paragraph", Text: "beta giraffe"}, + }}, + }, + }) + require.NoError(t, err) + + // C4: Modify b1 only. + c4, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/version-test-animals", + SigningKeyName: "main", + BaseVersion: c3.Version, + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "gamma hippo"}, + }}, + }, + }) + require.NoError(t, err) + + // C5: Modify b3 only (final version for doc1). + c5, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/version-test-animals", + SigningKeyName: "main", + BaseVersion: c4.Version, + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b3", Type: "paragraph", Text: "delta iguana"}, + }}, + }, + }) + require.NoError(t, err) + _ = c5 // c5.Version is the latest for doc1. + + // ===== DOCUMENT 2 SETUP: /version-test-creatures ===== + // This document has 2 blocks (b1, b2) with 3 changes: + // D1: b1="omega tiger", b2="beta koala" + // D2: b1="beta koala" (b2 untouched) + // D3: b1="epsilon panda" (b2 untouched) + + // D1: Create document with 2 blocks. + d1, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/version-test-creatures", + SigningKeyName: "main", + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_SetMetadata_{ + SetMetadata: &documents.DocumentChange_SetMetadata{Key: "title", Value: "Creature Versions Test"}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b1", Parent: "", LeftSibling: ""}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b2", Parent: "", LeftSibling: "b1"}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "omega tiger"}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b2", Type: "paragraph", Text: "beta koala"}, + }}, + }, + }) + require.NoError(t, err) + + // D2: Modify b1 only. + d2, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/version-test-creatures", + SigningKeyName: "main", + BaseVersion: d1.Version, + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "beta koala"}, + }}, + }, + }) + require.NoError(t, err) + + // D3: Modify b1 only (final version for doc2). + d3, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/version-test-creatures", + SigningKeyName: "main", + BaseVersion: d2.Version, + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "epsilon panda"}, + }}, + }, + }) + require.NoError(t, err) + _ = d3 // d3.Version is the latest for doc2. + + // ===== DOCUMENT 3 SETUP: /multi-block-commit-test ===== + // This document tests a bug where multiple blocks modified in the same commit + // can cause version corruption in search results. + // + // M1: Create 3 blocks + // b1="zulu unique content", b2="yankee other stuff", b3="xray more things" + // M2: Modify ALL 3 blocks in ONE commit (including deleting b1's content) + // b1="" (deletion), b2="yankee modified", b3="xray modified" + // + // The bug: When searching for "zulu" (deleted in M2), the version lookup + // iterates through M2's changes. It updates latestUnrelated for b2 and b3 + // changes (same commit, different blocks) BEFORE detecting that b1 was also + // modified. This causes the returned version to be M2's instead of M1's. + + // M1: Create document with 3 blocks. + m1, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/multi-block-commit-test", + SigningKeyName: "main", + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_SetMetadata_{ + SetMetadata: &documents.DocumentChange_SetMetadata{Key: "title", Value: "Multi Block Commit Test"}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b1", Parent: "", LeftSibling: ""}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b2", Parent: "", LeftSibling: "b1"}, + }}, + {Op: &documents.DocumentChange_MoveBlock_{ + MoveBlock: &documents.DocumentChange_MoveBlock{BlockId: "b3", Parent: "", LeftSibling: "b2"}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: "zulu unique content"}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b2", Type: "paragraph", Text: "yankee other stuff"}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b3", Type: "paragraph", Text: "xray more things"}, + }}, + }, + }) + require.NoError(t, err) + + // M2: Modify ALL 3 blocks in ONE commit (b1 content deleted, b2 and b3 modified). + m2, err := alice.RPC.DocumentsV3.CreateDocumentChange(ctx, &documents.CreateDocumentChangeRequest{ + Account: aliceAccount, + Path: "/multi-block-commit-test", + SigningKeyName: "main", + BaseVersion: m1.Version, + Changes: []*documents.DocumentChange{ + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b1", Type: "paragraph", Text: ""}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b2", Type: "paragraph", Text: "yankee modified"}, + }}, + {Op: &documents.DocumentChange_ReplaceBlock{ + ReplaceBlock: &documents.Block{Id: "b3", Type: "paragraph", Text: "xray modified"}, + }}, + }, + }) + require.NoError(t, err) + _ = m2 // m2.Version is the latest for doc3. + + // expectedResult describes what we expect for a search result. + type expectedResult struct { + contentSubstr string // Substring that must appear in content. + isLatest bool // Whether version should have &l marker. + docPath string // Which document path this should be from. + } + + // verifySearchResults checks that the search results match expectations. + // For each expectation, it verifies that at least one matching result exists with the expected &l status. + verifySearchResults := func(t *testing.T, results []*entities.Entity, expectations []expectedResult) { + t.Helper() + + // Verify each expectation is met. + for _, exp := range expectations { + foundMatching := false + for _, e := range results { + if strings.Contains(e.Content, exp.contentSubstr) && strings.Contains(e.DocId, exp.docPath) { + _, isLatest := parseEntityVersion(e.Id) + if exp.isLatest == isLatest { + foundMatching = true + break + } + } + } + if exp.isLatest { + require.True(t, foundMatching, + "expected to find content containing %q from %s WITH &l marker", exp.contentSubstr, exp.docPath) + } else { + require.True(t, foundMatching, + "expected to find content containing %q from %s WITHOUT &l marker", exp.contentSubstr, exp.docPath) + } + } + } + + // ===== KEYWORD SEARCH TESTS ===== + t.Run("Keyword", func(t *testing.T) { + t.Run("SearchAlpha_OnlyInDoc1V1", func(t *testing.T) { + // "alpha" only existed in C1, C2 changed b1. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "alpha", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 1, "must return exactly 1 result for 'alpha'") + + e := res.Entities[0] + require.Contains(t, e.Content, "alpha dinosaur") + _, isLatest := parseEntityVersion(e.Id) + require.False(t, isLatest, "old content 'alpha dinosaur' must NOT have &l marker") + }) + + t.Run("SearchStaticForever_NeverModified", func(t *testing.T) { + // "static forever" in b2 was never modified after C1. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "static", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 1, "must return exactly 1 result for 'static'") + + e := res.Entities[0] + require.Contains(t, e.Content, "static forever") + _, isLatest := parseEntityVersion(e.Id) + require.True(t, isLatest, "never-modified content 'static forever' must have &l marker") + }) + + t.Run("SearchGamma_LatestInDoc1", func(t *testing.T) { + // "gamma" exists in C4 for b1, and b1 wasn't touched after C4 (C5 only touched b3). + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "gamma", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 1, "must return exactly 1 result for 'gamma'") + + e := res.Entities[0] + require.Contains(t, e.Content, "gamma hippo") + _, isLatest := parseEntityVersion(e.Id) + require.True(t, isLatest, "current content 'gamma hippo' must have &l marker") + }) + + t.Run("SearchDelta_LatestInDoc1", func(t *testing.T) { + // "delta" exists in C5 for b3, which is the latest version. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "delta", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 1, "must return exactly 1 result for 'delta'") + + e := res.Entities[0] + require.Contains(t, e.Content, "delta iguana") + _, isLatest := parseEntityVersion(e.Id) + require.True(t, isLatest, "current content 'delta iguana' must have &l marker") + }) + + t.Run("SearchOmega_OnlyInDoc2V1", func(t *testing.T) { + // "omega" only existed in D1, D2 changed b1. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "omega", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 1, "must return exactly 1 result for 'omega'") + + e := res.Entities[0] + require.Contains(t, e.Content, "omega tiger") + _, isLatest := parseEntityVersion(e.Id) + require.False(t, isLatest, "old content 'omega tiger' must NOT have &l marker") + }) + + t.Run("SearchEpsilon_LatestInDoc2", func(t *testing.T) { + // "epsilon" exists in D3 for b1, which is the latest version. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "epsilon", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 1, "must return exactly 1 result for 'epsilon'") + + e := res.Entities[0] + require.Contains(t, e.Content, "epsilon panda") + _, isLatest := parseEntityVersion(e.Id) + require.True(t, isLatest, "current content 'epsilon panda' must have &l marker") + }) + + t.Run("SearchBeta_MultipleVersionsAcrossDocs", func(t *testing.T) { + // "beta" appears in multiple places: + // Doc1: b1@C2 ("beta elephant") -> version C3 (no &l, C4 touched b1) + // Doc1: b3@C1 ("beta elephant") -> version C2 (no &l, C3 touched b3) + // Doc1: b3@C3 ("beta giraffe") -> version C4 (no &l, C5 touched b3) + // Doc2: b1@D2 ("beta koala") -> version D2 (no &l, D3 touched b1) + // Doc2: b2@D1 ("beta koala") -> version D3 (has &l, b2 never touched after D1) + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "beta", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 5, "must return 5 results for 'beta' across both docs") + + // Count results by document and check &l markers. + doc1Count := 0 + doc2Count := 0 + latestCount := 0 + for _, e := range res.Entities { + _, isLatest := parseEntityVersion(e.Id) + if isLatest { + latestCount++ + } + if strings.Contains(e.DocId, "/version-test-animals") { + doc1Count++ + } + if strings.Contains(e.DocId, "/version-test-creatures") { + doc2Count++ + } + } + require.Equal(t, 3, doc1Count, "must have 3 'beta' results from doc1") + require.Equal(t, 2, doc2Count, "must have 2 'beta' results from doc2") + require.Equal(t, 1, latestCount, "only 1 'beta' result should have &l marker (doc2/b2)") + + // Verify the specific expectations. + // Note: "beta koala" appears twice in doc2 - b1 (no &l) and b2 (has &l). + verifySearchResults(t, res.Entities, []expectedResult{ + {contentSubstr: "beta elephant", isLatest: false, docPath: "/version-test-animals"}, + {contentSubstr: "beta giraffe", isLatest: false, docPath: "/version-test-animals"}, + {contentSubstr: "beta koala", isLatest: false, docPath: "/version-test-creatures"}, // b1 version + {contentSubstr: "beta koala", isLatest: true, docPath: "/version-test-creatures"}, // b2 version + }) + }) + + t.Run("SearchElephant_TwoBlocksInDoc1", func(t *testing.T) { + // "elephant" appears in: + // Doc1: b1@C2 ("beta elephant") -> version C3 (no &l) + // Doc1: b3@C1 ("beta elephant") -> version C2 (no &l) + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "elephant", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 2, "must return 2 results for 'elephant'") + + for _, e := range res.Entities { + require.Contains(t, e.Content, "elephant") + _, isLatest := parseEntityVersion(e.Id) + require.False(t, isLatest, "superseded content with 'elephant' must NOT have &l marker") + } + }) + + t.Run("SearchKoala_TwoBlocksInDoc2", func(t *testing.T) { + // "koala" appears in: + // Doc2: b1@D2 ("beta koala") -> version D2 (no &l, D3 touched b1) + // Doc2: b2@D1 ("beta koala") -> version D3 (has &l, b2 never touched) + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "koala", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 2, "must return 2 results for 'koala'") + + latestCount := 0 + for _, e := range res.Entities { + require.Contains(t, e.Content, "koala") + _, isLatest := parseEntityVersion(e.Id) + if isLatest { + latestCount++ + } + } + require.Equal(t, 1, latestCount, "exactly 1 'koala' result should have &l marker (b2)") + }) + + t.Run("SearchGiraffe_SupersededInDoc1", func(t *testing.T) { + // "giraffe" only in Doc1: b3@C3 ("beta giraffe") -> version C4 (no &l, C5 touched b3). + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "giraffe", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 1, "must return 1 result for 'giraffe'") + + e := res.Entities[0] + require.Contains(t, e.Content, "beta giraffe") + _, isLatest := parseEntityVersion(e.Id) + require.False(t, isLatest, "superseded content 'beta giraffe' must NOT have &l marker") + }) + + t.Run("SearchMultiBlockCommit_VersionBlobIdConsistency", func(t *testing.T) { + // This test reproduces a bug where multiple blocks modified in the same commit + // causes the version in the URL to differ from the blobId. + // + // When searching for "zulu" (content deleted in M2): + // - blobId should be M1's blob (where content existed) + // - version in URL should ALSO be M1's version + // - Bug: version in URL was incorrectly showing M2's version + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "zulu", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 1, "must return exactly 1 result for 'zulu'") + + e := res.Entities[0] + require.Contains(t, e.Content, "zulu unique content") + + version, isLatest := parseEntityVersion(e.Id) + require.False(t, isLatest, "deleted content must NOT have &l marker") + + // CRITICAL: blobId must match the version in the URL. + // This is the bug we're testing - when multiple blocks are modified in the + // same commit, the version lookup incorrectly picks up the version from + // a sibling block's change instead of keeping the original version. + require.Equal(t, e.BlobId, version, + "blobId (%s) must match version in URL (%s) - version mismatch indicates bug in multi-block commit handling", + e.BlobId, version) + }) + + t.Run("SearchMultiBlockCommit_DeletionDoesNotCorruptVersion", func(t *testing.T) { + // When content is deleted in a commit that also modifies other blocks, + // the search result for the deleted content must show the pre-deletion version, + // not the deletion commit's version. + // + // Timeline: + // M1: b1="zulu unique content" (version X) + // M2: b1="" (deleted), b2 and b3 also modified (version Y) + // + // Search "zulu" should return version X, not version Y. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "zulu", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 1, "must return exactly 1 result for 'zulu'") + + e := res.Entities[0] + + // The version should be M1's version (where content existed), + // NOT M2's version (where content was deleted). + version, _ := parseEntityVersion(e.Id) + + // blobId points to the blob where content was indexed (M1). + // If version != blobId, it means we incorrectly picked up M2's version. + require.Equal(t, e.BlobId, version, + "deleted content must show pre-deletion version, not deletion commit version") + }) + + t.Run("SearchMultiBlockCommit_BlockOrderingDoesNotAffectResult", func(t *testing.T) { + // Test that the order in which blocks are processed doesn't affect the result. + // Search for content in different blocks that were all modified in M2. + // + // "yankee" exists in both M1 and M2: + // - M1: b2="yankee other stuff" -> should show M1 version (no &l, modified in M2) + // - M2: b2="yankee modified" -> should show M2 version (has &l) + // + // Both results must have consistent blobId/version. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "yankee", + IncludeBody: true, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 2, "must return 2 results for 'yankee' (M1 and M2 versions)") + + for _, e := range res.Entities { + version, _ := parseEntityVersion(e.Id) + require.Equal(t, e.BlobId, version, + "blobId (%s) must match version in URL (%s) for content: %s", + e.BlobId, version, e.Content) + } + + // Verify we have one with &l and one without. + latestCount := 0 + for _, e := range res.Entities { + _, isLatest := parseEntityVersion(e.Id) + if isLatest { + latestCount++ + require.Contains(t, e.Content, "yankee modified", "latest version must be M2's content") + } else { + require.Contains(t, e.Content, "yankee other stuff", "non-latest must be M1's content") + } + } + require.Equal(t, 1, latestCount, "exactly one result should have &l marker") + }) + }) + + // ===== SEMANTIC SEARCH TESTS ===== + t.Run("Semantic", func(t *testing.T) { + // Wait for embeddings to be generated. Skip if not available within timeout. + embeddingsReady := false + for i := 0; i < 20; i++ { // Try for ~10 seconds. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "animal", + SearchType: entities.SearchType_SEARCH_SEMANTIC, + IncludeBody: true, + }) + if err == nil && len(res.Entities) > 0 { + embeddingsReady = true + break + } + time.Sleep(500 * time.Millisecond) + } + if !embeddingsReady { + t.Skip("Skipping semantic search tests: embeddings not ready within timeout") + } + + t.Run("SearchAnimal_VersionConsistency", func(t *testing.T) { + // Semantic search for "animal" should return animal-related content. + // Each result should have consistent version markers. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "animal", + SearchType: entities.SearchType_SEARCH_SEMANTIC, + IncludeBody: true, + }) + require.NoError(t, err) + require.Greater(t, len(res.Entities), 0, "semantic search for 'animal' must return results") + + // Verify version consistency for results we know about. + for _, e := range res.Entities { + _, isLatest := parseEntityVersion(e.Id) + + // Check specific content we know the expected state for. + switch { + case strings.Contains(e.Content, "alpha dinosaur"): + require.False(t, isLatest, "'alpha dinosaur' must NOT have &l") + case strings.Contains(e.Content, "gamma hippo"): + require.True(t, isLatest, "'gamma hippo' must have &l") + case strings.Contains(e.Content, "delta iguana"): + require.True(t, isLatest, "'delta iguana' must have &l") + case strings.Contains(e.Content, "omega tiger"): + require.False(t, isLatest, "'omega tiger' must NOT have &l") + case strings.Contains(e.Content, "epsilon panda"): + require.True(t, isLatest, "'epsilon panda' must have &l") + case strings.Contains(e.Content, "beta elephant"): + require.False(t, isLatest, "'beta elephant' must NOT have &l") + case strings.Contains(e.Content, "beta giraffe"): + require.False(t, isLatest, "'beta giraffe' must NOT have &l") + } + } + }) + }) + + // ===== HYBRID SEARCH TESTS ===== + t.Run("Hybrid", func(t *testing.T) { + t.Run("SearchBeta_BlendedResults", func(t *testing.T) { + // Hybrid search for "beta" should blend keyword and semantic results. + // Filter to document content only to avoid title matches from semantic search. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "beta", + SearchType: entities.SearchType_SEARCH_HYBRID, + IncludeBody: true, + ContentTypeFilter: []entities.ContentTypeFilter{entities.ContentTypeFilter_CONTENT_TYPE_DOCUMENT}, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 5, "hybrid search for 'beta' must return 5 results") + + // Same assertions as keyword search. + latestCount := 0 + for _, e := range res.Entities { + _, isLatest := parseEntityVersion(e.Id) + if isLatest { + latestCount++ + } + } + require.Equal(t, 1, latestCount, "only 1 'beta' result should have &l marker in hybrid search") + }) + + t.Run("SearchElephant_BlendedResults", func(t *testing.T) { + // Hybrid search for "elephant". + // Filter to document content only to avoid unrelated matches from semantic search. + res, err := alice.RPC.Entities.SearchEntities(ctx, &entities.SearchEntitiesRequest{ + Query: "elephant", + SearchType: entities.SearchType_SEARCH_HYBRID, + IncludeBody: true, + ContentTypeFilter: []entities.ContentTypeFilter{entities.ContentTypeFilter_CONTENT_TYPE_DOCUMENT}, + }) + require.NoError(t, err) + require.Len(t, res.Entities, 2, "hybrid search for 'elephant' must return 2 results") + + for _, e := range res.Entities { + _, isLatest := parseEntityVersion(e.Id) + require.False(t, isLatest, "all 'elephant' results must NOT have &l marker in hybrid search") + } + }) + }) +} diff --git a/backend/genproto/daemon/v1alpha/daemon.pb.go b/backend/genproto/daemon/v1alpha/daemon.pb.go index 1d87a0502..455611025 100644 --- a/backend/genproto/daemon/v1alpha/daemon.pb.go +++ b/backend/genproto/daemon/v1alpha/daemon.pb.go @@ -85,6 +85,10 @@ const ( TaskName_TASK_NAME_UNSPECIFIED TaskName = 0 // Task for reindexing the database. TaskName_REINDEXING TaskName = 1 + // Task for generating embeddings. + TaskName_EMBEDDING TaskName = 2 + // Task for loading a machine learning model. + TaskName_LOADING_MODEL TaskName = 3 ) // Enum value maps for TaskName. @@ -92,10 +96,14 @@ var ( TaskName_name = map[int32]string{ 0: "TASK_NAME_UNSPECIFIED", 1: "REINDEXING", + 2: "EMBEDDING", + 3: "LOADING_MODEL", } TaskName_value = map[string]int32{ "TASK_NAME_UNSPECIFIED": 0, "REINDEXING": 1, + "EMBEDDING": 2, + "LOADING_MODEL": 3, } ) @@ -1446,11 +1454,13 @@ const file_daemon_v1alpha_daemon_proto_rawDesc = "" + "\bSTARTING\x10\x00\x12\r\n" + "\tMIGRATING\x10\x01\x12\n" + "\n" + - "\x06ACTIVE\x10\x03*5\n" + + "\x06ACTIVE\x10\x03*W\n" + "\bTaskName\x12\x19\n" + "\x15TASK_NAME_UNSPECIFIED\x10\x00\x12\x0e\n" + "\n" + - "REINDEXING\x10\x012\x87\n" + + "REINDEXING\x10\x01\x12\r\n" + + "\tEMBEDDING\x10\x02\x12\x11\n" + + "\rLOADING_MODEL\x10\x032\x87\n" + "\n" + "\x06Daemon\x12h\n" + "\vGenMnemonic\x12+.com.seed.daemon.v1alpha.GenMnemonicRequest\x1a,.com.seed.daemon.v1alpha.GenMnemonicResponse\x12]\n" + diff --git a/backend/genproto/entities/v1alpha/entities.pb.go b/backend/genproto/entities/v1alpha/entities.pb.go index dd6256cad..67b2c9bd8 100644 --- a/backend/genproto/entities/v1alpha/entities.pb.go +++ b/backend/genproto/entities/v1alpha/entities.pb.go @@ -77,6 +77,112 @@ func (DiscoveryTaskState) EnumDescriptor() ([]byte, []int) { return file_entities_v1alpha_entities_proto_rawDescGZIP(), []int{0} } +// Describes the state of the discovery task. +type SearchType int32 + +const ( + // Keyword-based search. + SearchType_SEARCH_KEYWORD SearchType = 0 + // Semantic search. + SearchType_SEARCH_SEMANTIC SearchType = 1 + // Hybrid search. with RRFusion. + SearchType_SEARCH_HYBRID SearchType = 2 +) + +// Enum value maps for SearchType. +var ( + SearchType_name = map[int32]string{ + 0: "SEARCH_KEYWORD", + 1: "SEARCH_SEMANTIC", + 2: "SEARCH_HYBRID", + } + SearchType_value = map[string]int32{ + "SEARCH_KEYWORD": 0, + "SEARCH_SEMANTIC": 1, + "SEARCH_HYBRID": 2, + } +) + +func (x SearchType) Enum() *SearchType { + p := new(SearchType) + *p = x + return p +} + +func (x SearchType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SearchType) Descriptor() protoreflect.EnumDescriptor { + return file_entities_v1alpha_entities_proto_enumTypes[1].Descriptor() +} + +func (SearchType) Type() protoreflect.EnumType { + return &file_entities_v1alpha_entities_proto_enumTypes[1] +} + +func (x SearchType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SearchType.Descriptor instead. +func (SearchType) EnumDescriptor() ([]byte, []int) { + return file_entities_v1alpha_entities_proto_rawDescGZIP(), []int{1} +} + +// Content type to filter search results by. +type ContentTypeFilter int32 + +const ( + ContentTypeFilter_CONTENT_TYPE_TITLE ContentTypeFilter = 0 + ContentTypeFilter_CONTENT_TYPE_DOCUMENT ContentTypeFilter = 1 + ContentTypeFilter_CONTENT_TYPE_COMMENT ContentTypeFilter = 2 + ContentTypeFilter_CONTENT_TYPE_CONTACT ContentTypeFilter = 3 +) + +// Enum value maps for ContentTypeFilter. +var ( + ContentTypeFilter_name = map[int32]string{ + 0: "CONTENT_TYPE_TITLE", + 1: "CONTENT_TYPE_DOCUMENT", + 2: "CONTENT_TYPE_COMMENT", + 3: "CONTENT_TYPE_CONTACT", + } + ContentTypeFilter_value = map[string]int32{ + "CONTENT_TYPE_TITLE": 0, + "CONTENT_TYPE_DOCUMENT": 1, + "CONTENT_TYPE_COMMENT": 2, + "CONTENT_TYPE_CONTACT": 3, + } +) + +func (x ContentTypeFilter) Enum() *ContentTypeFilter { + p := new(ContentTypeFilter) + *p = x + return p +} + +func (x ContentTypeFilter) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ContentTypeFilter) Descriptor() protoreflect.EnumDescriptor { + return file_entities_v1alpha_entities_proto_enumTypes[2].Descriptor() +} + +func (ContentTypeFilter) Type() protoreflect.EnumType { + return &file_entities_v1alpha_entities_proto_enumTypes[2] +} + +func (x ContentTypeFilter) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ContentTypeFilter.Descriptor instead. +func (ContentTypeFilter) EnumDescriptor() ([]byte, []int) { + return file_entities_v1alpha_entities_proto_rawDescGZIP(), []int{2} +} + // Request to get a change by ID. type GetChangeRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -922,29 +1028,50 @@ func (x *DeletedEntity) GetMetadata() string { return "" } -// Request to +// Request to search entities. type SearchEntitiesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` - // Query to find. We Ssupport wildcards and phrases. + // Query to find. We support wildcards and phrases. // See https://sqlite.org/fts5.html#full_text_query_syntax. Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - // Whether to look into all content available or just the titles. - // If false, comments are not included in the search. - // Default is false. + // Deprecated, use content_type_filters instead to specify + // which content types to include in the search. + // + // Deprecated: Marked as deprecated in entities/v1alpha/entities.proto. IncludeBody bool `protobuf:"varint,2,opt,name=include_body,json=includeBody,proto3" json:"include_body,omitempty"` // Optional. The size of the text accompanying the search match. // Half of the size is before the match, and half after. // Default is 48 runes. ContextSize int32 `protobuf:"varint,3,opt,name=context_size,json=contextSize,proto3" json:"context_size,omitempty"` - // Optional. The account uid to filter the search by. - // If not set, the search will be performed across all accounts. + // Deprecated. Use iri_filter instead. + // + // Deprecated: Marked as deprecated in entities/v1alpha/entities.proto. AccountUid string `protobuf:"bytes,4,opt,name=account_uid,json=accountUid,proto3" json:"account_uid,omitempty"` // Optional. The account uid the user is logged in with. // This is used to filter out contacts that the user doesn't have access to. // If not set, we won't provide any contact entities in the response. LoggedAccountUid string `protobuf:"bytes,5,opt,name=logged_account_uid,json=loggedAccountUid,proto3" json:"logged_account_uid,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Optional. Type of search to perform. Could be keyword, semantic or hybrid. + // if not set, keyword search is used. + SearchType SearchType `protobuf:"varint,6,opt,name=search_type,json=searchType,proto3,enum=com.seed.entities.v1alpha.SearchType" json:"search_type,omitempty"` + // Optional. hm:// URL with optional GLOB wildcards to scope search. + // Examples: "hm:///cars/honda" (single doc), "hm:///cars/*" (subtree). + // When empty, falls back to account_uid if set, otherwise matches all. + IriFilter string `protobuf:"bytes,7,opt,name=iri_filter,json=iriFilter,proto3" json:"iri_filter,omitempty"` + // Optional. Fine-grained content type selection. Overrides include_body when set. + // When empty, legacy behavior (title + body types based on include_body). + ContentTypeFilter []ContentTypeFilter `protobuf:"varint,8,rep,packed,name=content_type_filter,json=contentTypeFilter,proto3,enum=com.seed.entities.v1alpha.ContentTypeFilter" json:"content_type_filter,omitempty"` + // Optional. Authority weight for citation-based ranking. Range [0, 1]. + // 0 (default) disables authority scoring. Higher values increase citation influence. + // Final score: (1-weight)*textRRF + 0.7*weight*docAuthRRF + 0.3*weight*authorAuthRRF. + AuthorityWeight float32 `protobuf:"fixed32,9,opt,name=authority_weight,json=authorityWeight,proto3" json:"authority_weight,omitempty"` + // Optional. Maximum number of results per page. + // When 0 (default), all results are returned (backwards compatible). + PageSize int32 `protobuf:"varint,10,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + // Optional. Token from a previous SearchEntitiesResponse to get the next page. + PageToken string `protobuf:"bytes,11,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SearchEntitiesRequest) Reset() { @@ -984,6 +1111,7 @@ func (x *SearchEntitiesRequest) GetQuery() string { return "" } +// Deprecated: Marked as deprecated in entities/v1alpha/entities.proto. func (x *SearchEntitiesRequest) GetIncludeBody() bool { if x != nil { return x.IncludeBody @@ -998,6 +1126,7 @@ func (x *SearchEntitiesRequest) GetContextSize() int32 { return 0 } +// Deprecated: Marked as deprecated in entities/v1alpha/entities.proto. func (x *SearchEntitiesRequest) GetAccountUid() string { if x != nil { return x.AccountUid @@ -1012,6 +1141,48 @@ func (x *SearchEntitiesRequest) GetLoggedAccountUid() string { return "" } +func (x *SearchEntitiesRequest) GetSearchType() SearchType { + if x != nil { + return x.SearchType + } + return SearchType_SEARCH_KEYWORD +} + +func (x *SearchEntitiesRequest) GetIriFilter() string { + if x != nil { + return x.IriFilter + } + return "" +} + +func (x *SearchEntitiesRequest) GetContentTypeFilter() []ContentTypeFilter { + if x != nil { + return x.ContentTypeFilter + } + return nil +} + +func (x *SearchEntitiesRequest) GetAuthorityWeight() float32 { + if x != nil { + return x.AuthorityWeight + } + return 0 +} + +func (x *SearchEntitiesRequest) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *SearchEntitiesRequest) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + // A list of entities matching the request. type SearchEntitiesResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1688,14 +1859,24 @@ const file_entities_v1alpha_entities_proto_rawDesc = "" + "\vdelete_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\n" + "deleteTime\x12%\n" + "\x0edeleted_reason\x18\x03 \x01(\tR\rdeletedReason\x12\x1a\n" + - "\bmetadata\x18\x04 \x01(\tR\bmetadata\"\xc2\x01\n" + + "\bmetadata\x18\x04 \x01(\tR\bmetadata\"\xf6\x03\n" + "\x15SearchEntitiesRequest\x12\x14\n" + - "\x05query\x18\x01 \x01(\tR\x05query\x12!\n" + - "\finclude_body\x18\x02 \x01(\bR\vincludeBody\x12!\n" + - "\fcontext_size\x18\x03 \x01(\x05R\vcontextSize\x12\x1f\n" + - "\vaccount_uid\x18\x04 \x01(\tR\n" + + "\x05query\x18\x01 \x01(\tR\x05query\x12%\n" + + "\finclude_body\x18\x02 \x01(\bB\x02\x18\x01R\vincludeBody\x12!\n" + + "\fcontext_size\x18\x03 \x01(\x05R\vcontextSize\x12#\n" + + "\vaccount_uid\x18\x04 \x01(\tB\x02\x18\x01R\n" + "accountUid\x12,\n" + - "\x12logged_account_uid\x18\x05 \x01(\tR\x10loggedAccountUid\"\x7f\n" + + "\x12logged_account_uid\x18\x05 \x01(\tR\x10loggedAccountUid\x12F\n" + + "\vsearch_type\x18\x06 \x01(\x0e2%.com.seed.entities.v1alpha.SearchTypeR\n" + + "searchType\x12\x1d\n" + + "\n" + + "iri_filter\x18\a \x01(\tR\tiriFilter\x12\\\n" + + "\x13content_type_filter\x18\b \x03(\x0e2,.com.seed.entities.v1alpha.ContentTypeFilterR\x11contentTypeFilter\x12)\n" + + "\x10authority_weight\x18\t \x01(\x02R\x0fauthorityWeight\x12\x1b\n" + + "\tpage_size\x18\n" + + " \x01(\x05R\bpageSize\x12\x1d\n" + + "\n" + + "page_token\x18\v \x01(\tR\tpageToken\"\x7f\n" + "\x16SearchEntitiesResponse\x12=\n" + "\bentities\x18\x01 \x03(\v2!.com.seed.entities.v1alpha.EntityR\bentities\x12&\n" + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken\"=\n" + @@ -1742,7 +1923,17 @@ const file_entities_v1alpha_entities_proto_rawDesc = "" + "\x12DiscoveryTaskState\x12\x1a\n" + "\x16DISCOVERY_TASK_STARTED\x10\x00\x12\x1e\n" + "\x1aDISCOVERY_TASK_IN_PROGRESS\x10\x01\x12\x1c\n" + - "\x18DISCOVERY_TASK_COMPLETED\x10\x022\x89\a\n" + + "\x18DISCOVERY_TASK_COMPLETED\x10\x02*H\n" + + "\n" + + "SearchType\x12\x12\n" + + "\x0eSEARCH_KEYWORD\x10\x00\x12\x13\n" + + "\x0fSEARCH_SEMANTIC\x10\x01\x12\x11\n" + + "\rSEARCH_HYBRID\x10\x02*z\n" + + "\x11ContentTypeFilter\x12\x16\n" + + "\x12CONTENT_TYPE_TITLE\x10\x00\x12\x19\n" + + "\x15CONTENT_TYPE_DOCUMENT\x10\x01\x12\x18\n" + + "\x14CONTENT_TYPE_COMMENT\x10\x02\x12\x18\n" + + "\x14CONTENT_TYPE_CONTACT\x10\x032\x89\a\n" + "\bEntities\x12[\n" + "\tGetChange\x12+.com.seed.entities.v1alpha.GetChangeRequest\x1a!.com.seed.entities.v1alpha.Change\x12s\n" + "\x11GetEntityTimeline\x123.com.seed.entities.v1alpha.GetEntityTimelineRequest\x1a).com.seed.entities.v1alpha.EntityTimeline\x12u\n" + @@ -1765,72 +1956,76 @@ func file_entities_v1alpha_entities_proto_rawDescGZIP() []byte { return file_entities_v1alpha_entities_proto_rawDescData } -var file_entities_v1alpha_entities_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_entities_v1alpha_entities_proto_enumTypes = make([]protoimpl.EnumInfo, 3) var file_entities_v1alpha_entities_proto_msgTypes = make([]protoimpl.MessageInfo, 21) var file_entities_v1alpha_entities_proto_goTypes = []any{ (DiscoveryTaskState)(0), // 0: com.seed.entities.v1alpha.DiscoveryTaskState - (*GetChangeRequest)(nil), // 1: com.seed.entities.v1alpha.GetChangeRequest - (*GetEntityTimelineRequest)(nil), // 2: com.seed.entities.v1alpha.GetEntityTimelineRequest - (*DiscoverEntityRequest)(nil), // 3: com.seed.entities.v1alpha.DiscoverEntityRequest - (*DiscoverEntityResponse)(nil), // 4: com.seed.entities.v1alpha.DiscoverEntityResponse - (*DiscoveryProgress)(nil), // 5: com.seed.entities.v1alpha.DiscoveryProgress - (*Change)(nil), // 6: com.seed.entities.v1alpha.Change - (*EntityTimeline)(nil), // 7: com.seed.entities.v1alpha.EntityTimeline - (*AuthorVersion)(nil), // 8: com.seed.entities.v1alpha.AuthorVersion - (*Entity)(nil), // 9: com.seed.entities.v1alpha.Entity - (*DeletedEntity)(nil), // 10: com.seed.entities.v1alpha.DeletedEntity - (*SearchEntitiesRequest)(nil), // 11: com.seed.entities.v1alpha.SearchEntitiesRequest - (*SearchEntitiesResponse)(nil), // 12: com.seed.entities.v1alpha.SearchEntitiesResponse - (*DeleteEntityRequest)(nil), // 13: com.seed.entities.v1alpha.DeleteEntityRequest - (*ListDeletedEntitiesRequest)(nil), // 14: com.seed.entities.v1alpha.ListDeletedEntitiesRequest - (*ListDeletedEntitiesResponse)(nil), // 15: com.seed.entities.v1alpha.ListDeletedEntitiesResponse - (*UndeleteEntityRequest)(nil), // 16: com.seed.entities.v1alpha.UndeleteEntityRequest - (*ListEntityMentionsRequest)(nil), // 17: com.seed.entities.v1alpha.ListEntityMentionsRequest - (*ListEntityMentionsResponse)(nil), // 18: com.seed.entities.v1alpha.ListEntityMentionsResponse - (*Mention)(nil), // 19: com.seed.entities.v1alpha.Mention - nil, // 20: com.seed.entities.v1alpha.EntityTimeline.ChangesEntry - (*Mention_BlobInfo)(nil), // 21: com.seed.entities.v1alpha.Mention.BlobInfo - (*timestamppb.Timestamp)(nil), // 22: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 23: google.protobuf.Empty + (SearchType)(0), // 1: com.seed.entities.v1alpha.SearchType + (ContentTypeFilter)(0), // 2: com.seed.entities.v1alpha.ContentTypeFilter + (*GetChangeRequest)(nil), // 3: com.seed.entities.v1alpha.GetChangeRequest + (*GetEntityTimelineRequest)(nil), // 4: com.seed.entities.v1alpha.GetEntityTimelineRequest + (*DiscoverEntityRequest)(nil), // 5: com.seed.entities.v1alpha.DiscoverEntityRequest + (*DiscoverEntityResponse)(nil), // 6: com.seed.entities.v1alpha.DiscoverEntityResponse + (*DiscoveryProgress)(nil), // 7: com.seed.entities.v1alpha.DiscoveryProgress + (*Change)(nil), // 8: com.seed.entities.v1alpha.Change + (*EntityTimeline)(nil), // 9: com.seed.entities.v1alpha.EntityTimeline + (*AuthorVersion)(nil), // 10: com.seed.entities.v1alpha.AuthorVersion + (*Entity)(nil), // 11: com.seed.entities.v1alpha.Entity + (*DeletedEntity)(nil), // 12: com.seed.entities.v1alpha.DeletedEntity + (*SearchEntitiesRequest)(nil), // 13: com.seed.entities.v1alpha.SearchEntitiesRequest + (*SearchEntitiesResponse)(nil), // 14: com.seed.entities.v1alpha.SearchEntitiesResponse + (*DeleteEntityRequest)(nil), // 15: com.seed.entities.v1alpha.DeleteEntityRequest + (*ListDeletedEntitiesRequest)(nil), // 16: com.seed.entities.v1alpha.ListDeletedEntitiesRequest + (*ListDeletedEntitiesResponse)(nil), // 17: com.seed.entities.v1alpha.ListDeletedEntitiesResponse + (*UndeleteEntityRequest)(nil), // 18: com.seed.entities.v1alpha.UndeleteEntityRequest + (*ListEntityMentionsRequest)(nil), // 19: com.seed.entities.v1alpha.ListEntityMentionsRequest + (*ListEntityMentionsResponse)(nil), // 20: com.seed.entities.v1alpha.ListEntityMentionsResponse + (*Mention)(nil), // 21: com.seed.entities.v1alpha.Mention + nil, // 22: com.seed.entities.v1alpha.EntityTimeline.ChangesEntry + (*Mention_BlobInfo)(nil), // 23: com.seed.entities.v1alpha.Mention.BlobInfo + (*timestamppb.Timestamp)(nil), // 24: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 25: google.protobuf.Empty } var file_entities_v1alpha_entities_proto_depIdxs = []int32{ 0, // 0: com.seed.entities.v1alpha.DiscoverEntityResponse.state:type_name -> com.seed.entities.v1alpha.DiscoveryTaskState - 22, // 1: com.seed.entities.v1alpha.DiscoverEntityResponse.last_result_time:type_name -> google.protobuf.Timestamp - 22, // 2: com.seed.entities.v1alpha.DiscoverEntityResponse.result_expire_time:type_name -> google.protobuf.Timestamp - 5, // 3: com.seed.entities.v1alpha.DiscoverEntityResponse.progress:type_name -> com.seed.entities.v1alpha.DiscoveryProgress - 22, // 4: com.seed.entities.v1alpha.Change.create_time:type_name -> google.protobuf.Timestamp - 20, // 5: com.seed.entities.v1alpha.EntityTimeline.changes:type_name -> com.seed.entities.v1alpha.EntityTimeline.ChangesEntry - 8, // 6: com.seed.entities.v1alpha.EntityTimeline.author_versions:type_name -> com.seed.entities.v1alpha.AuthorVersion - 22, // 7: com.seed.entities.v1alpha.AuthorVersion.version_time:type_name -> google.protobuf.Timestamp - 22, // 8: com.seed.entities.v1alpha.Entity.version_time:type_name -> google.protobuf.Timestamp - 22, // 9: com.seed.entities.v1alpha.DeletedEntity.delete_time:type_name -> google.protobuf.Timestamp - 9, // 10: com.seed.entities.v1alpha.SearchEntitiesResponse.entities:type_name -> com.seed.entities.v1alpha.Entity - 10, // 11: com.seed.entities.v1alpha.ListDeletedEntitiesResponse.deleted_entities:type_name -> com.seed.entities.v1alpha.DeletedEntity - 19, // 12: com.seed.entities.v1alpha.ListEntityMentionsResponse.mentions:type_name -> com.seed.entities.v1alpha.Mention - 21, // 13: com.seed.entities.v1alpha.Mention.source_blob:type_name -> com.seed.entities.v1alpha.Mention.BlobInfo - 6, // 14: com.seed.entities.v1alpha.EntityTimeline.ChangesEntry.value:type_name -> com.seed.entities.v1alpha.Change - 22, // 15: com.seed.entities.v1alpha.Mention.BlobInfo.create_time:type_name -> google.protobuf.Timestamp - 1, // 16: com.seed.entities.v1alpha.Entities.GetChange:input_type -> com.seed.entities.v1alpha.GetChangeRequest - 2, // 17: com.seed.entities.v1alpha.Entities.GetEntityTimeline:input_type -> com.seed.entities.v1alpha.GetEntityTimelineRequest - 3, // 18: com.seed.entities.v1alpha.Entities.DiscoverEntity:input_type -> com.seed.entities.v1alpha.DiscoverEntityRequest - 11, // 19: com.seed.entities.v1alpha.Entities.SearchEntities:input_type -> com.seed.entities.v1alpha.SearchEntitiesRequest - 13, // 20: com.seed.entities.v1alpha.Entities.DeleteEntity:input_type -> com.seed.entities.v1alpha.DeleteEntityRequest - 14, // 21: com.seed.entities.v1alpha.Entities.ListDeletedEntities:input_type -> com.seed.entities.v1alpha.ListDeletedEntitiesRequest - 16, // 22: com.seed.entities.v1alpha.Entities.UndeleteEntity:input_type -> com.seed.entities.v1alpha.UndeleteEntityRequest - 17, // 23: com.seed.entities.v1alpha.Entities.ListEntityMentions:input_type -> com.seed.entities.v1alpha.ListEntityMentionsRequest - 6, // 24: com.seed.entities.v1alpha.Entities.GetChange:output_type -> com.seed.entities.v1alpha.Change - 7, // 25: com.seed.entities.v1alpha.Entities.GetEntityTimeline:output_type -> com.seed.entities.v1alpha.EntityTimeline - 4, // 26: com.seed.entities.v1alpha.Entities.DiscoverEntity:output_type -> com.seed.entities.v1alpha.DiscoverEntityResponse - 12, // 27: com.seed.entities.v1alpha.Entities.SearchEntities:output_type -> com.seed.entities.v1alpha.SearchEntitiesResponse - 23, // 28: com.seed.entities.v1alpha.Entities.DeleteEntity:output_type -> google.protobuf.Empty - 15, // 29: com.seed.entities.v1alpha.Entities.ListDeletedEntities:output_type -> com.seed.entities.v1alpha.ListDeletedEntitiesResponse - 23, // 30: com.seed.entities.v1alpha.Entities.UndeleteEntity:output_type -> google.protobuf.Empty - 18, // 31: com.seed.entities.v1alpha.Entities.ListEntityMentions:output_type -> com.seed.entities.v1alpha.ListEntityMentionsResponse - 24, // [24:32] is the sub-list for method output_type - 16, // [16:24] is the sub-list for method input_type - 16, // [16:16] is the sub-list for extension type_name - 16, // [16:16] is the sub-list for extension extendee - 0, // [0:16] is the sub-list for field type_name + 24, // 1: com.seed.entities.v1alpha.DiscoverEntityResponse.last_result_time:type_name -> google.protobuf.Timestamp + 24, // 2: com.seed.entities.v1alpha.DiscoverEntityResponse.result_expire_time:type_name -> google.protobuf.Timestamp + 7, // 3: com.seed.entities.v1alpha.DiscoverEntityResponse.progress:type_name -> com.seed.entities.v1alpha.DiscoveryProgress + 24, // 4: com.seed.entities.v1alpha.Change.create_time:type_name -> google.protobuf.Timestamp + 22, // 5: com.seed.entities.v1alpha.EntityTimeline.changes:type_name -> com.seed.entities.v1alpha.EntityTimeline.ChangesEntry + 10, // 6: com.seed.entities.v1alpha.EntityTimeline.author_versions:type_name -> com.seed.entities.v1alpha.AuthorVersion + 24, // 7: com.seed.entities.v1alpha.AuthorVersion.version_time:type_name -> google.protobuf.Timestamp + 24, // 8: com.seed.entities.v1alpha.Entity.version_time:type_name -> google.protobuf.Timestamp + 24, // 9: com.seed.entities.v1alpha.DeletedEntity.delete_time:type_name -> google.protobuf.Timestamp + 1, // 10: com.seed.entities.v1alpha.SearchEntitiesRequest.search_type:type_name -> com.seed.entities.v1alpha.SearchType + 2, // 11: com.seed.entities.v1alpha.SearchEntitiesRequest.content_type_filter:type_name -> com.seed.entities.v1alpha.ContentTypeFilter + 11, // 12: com.seed.entities.v1alpha.SearchEntitiesResponse.entities:type_name -> com.seed.entities.v1alpha.Entity + 12, // 13: com.seed.entities.v1alpha.ListDeletedEntitiesResponse.deleted_entities:type_name -> com.seed.entities.v1alpha.DeletedEntity + 21, // 14: com.seed.entities.v1alpha.ListEntityMentionsResponse.mentions:type_name -> com.seed.entities.v1alpha.Mention + 23, // 15: com.seed.entities.v1alpha.Mention.source_blob:type_name -> com.seed.entities.v1alpha.Mention.BlobInfo + 8, // 16: com.seed.entities.v1alpha.EntityTimeline.ChangesEntry.value:type_name -> com.seed.entities.v1alpha.Change + 24, // 17: com.seed.entities.v1alpha.Mention.BlobInfo.create_time:type_name -> google.protobuf.Timestamp + 3, // 18: com.seed.entities.v1alpha.Entities.GetChange:input_type -> com.seed.entities.v1alpha.GetChangeRequest + 4, // 19: com.seed.entities.v1alpha.Entities.GetEntityTimeline:input_type -> com.seed.entities.v1alpha.GetEntityTimelineRequest + 5, // 20: com.seed.entities.v1alpha.Entities.DiscoverEntity:input_type -> com.seed.entities.v1alpha.DiscoverEntityRequest + 13, // 21: com.seed.entities.v1alpha.Entities.SearchEntities:input_type -> com.seed.entities.v1alpha.SearchEntitiesRequest + 15, // 22: com.seed.entities.v1alpha.Entities.DeleteEntity:input_type -> com.seed.entities.v1alpha.DeleteEntityRequest + 16, // 23: com.seed.entities.v1alpha.Entities.ListDeletedEntities:input_type -> com.seed.entities.v1alpha.ListDeletedEntitiesRequest + 18, // 24: com.seed.entities.v1alpha.Entities.UndeleteEntity:input_type -> com.seed.entities.v1alpha.UndeleteEntityRequest + 19, // 25: com.seed.entities.v1alpha.Entities.ListEntityMentions:input_type -> com.seed.entities.v1alpha.ListEntityMentionsRequest + 8, // 26: com.seed.entities.v1alpha.Entities.GetChange:output_type -> com.seed.entities.v1alpha.Change + 9, // 27: com.seed.entities.v1alpha.Entities.GetEntityTimeline:output_type -> com.seed.entities.v1alpha.EntityTimeline + 6, // 28: com.seed.entities.v1alpha.Entities.DiscoverEntity:output_type -> com.seed.entities.v1alpha.DiscoverEntityResponse + 14, // 29: com.seed.entities.v1alpha.Entities.SearchEntities:output_type -> com.seed.entities.v1alpha.SearchEntitiesResponse + 25, // 30: com.seed.entities.v1alpha.Entities.DeleteEntity:output_type -> google.protobuf.Empty + 17, // 31: com.seed.entities.v1alpha.Entities.ListDeletedEntities:output_type -> com.seed.entities.v1alpha.ListDeletedEntitiesResponse + 25, // 32: com.seed.entities.v1alpha.Entities.UndeleteEntity:output_type -> google.protobuf.Empty + 20, // 33: com.seed.entities.v1alpha.Entities.ListEntityMentions:output_type -> com.seed.entities.v1alpha.ListEntityMentionsResponse + 26, // [26:34] is the sub-list for method output_type + 18, // [18:26] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_entities_v1alpha_entities_proto_init() } @@ -1843,7 +2038,7 @@ func file_entities_v1alpha_entities_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_entities_v1alpha_entities_proto_rawDesc), len(file_entities_v1alpha_entities_proto_rawDesc)), - NumEnums: 1, + NumEnums: 3, NumMessages: 21, NumExtensions: 0, NumServices: 1, diff --git a/backend/hmnet/filemanager.go b/backend/hmnet/filemanager.go index 79f9ac162..5b3577206 100644 --- a/backend/hmnet/filemanager.go +++ b/backend/hmnet/filemanager.go @@ -252,7 +252,7 @@ func (fm *FileManager) UploadFile(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusCreated) w.Header().Add("Content-Type", "text/plain") - w.Write([]byte(n.Cid().String())) + _, _ = w.Write([]byte(n.Cid().String())) } // addFile chunks and adds content to the DAGService from a reader. The content diff --git a/backend/hmnet/filemanager_test.go b/backend/hmnet/filemanager_test.go index 0871386c1..1c0916d60 100644 --- a/backend/hmnet/filemanager_test.go +++ b/backend/hmnet/filemanager_test.go @@ -65,7 +65,7 @@ func TestPostGet(t *testing.T) { lis, err := net.Listen("tcp", srv.Addr) require.NoError(t, err) - go srv.Serve(lis) + go func() { _ = srv.Serve(lis) }() t.Cleanup(func() { require.NoError(t, srv.Shutdown(context.Background())) @@ -99,7 +99,7 @@ func TestRangeRequests(t *testing.T) { lis, err := net.Listen("tcp", srv.Addr) require.NoError(t, err) - go srv.Serve(lis) + go func() { _ = srv.Serve(lis) }() t.Cleanup(func() { require.NoError(t, srv.Shutdown(context.Background())) diff --git a/backend/hmnet/hmnet.go b/backend/hmnet/hmnet.go index 96d576354..f2d05d237 100644 --- a/backend/hmnet/hmnet.go +++ b/backend/hmnet/hmnet.go @@ -331,7 +331,7 @@ func (n *Node) Start(ctx context.Context) (err error) { case <-t.C: for pid, next := range localPeers { if time.Now().After(next) { - go n.storeRemotePeers(pid) + go func(pid peer.ID) { _ = n.storeRemotePeers(pid) }(pid) } } t.Reset(15 * time.Second) diff --git a/backend/hmnet/syncing/discovery.go b/backend/hmnet/syncing/discovery.go index f8080fc8d..dd3948b2a 100644 --- a/backend/hmnet/syncing/discovery.go +++ b/backend/hmnet/syncing/discovery.go @@ -76,7 +76,7 @@ func (s *Service) DiscoverObjectWithProgress(ctx context.Context, entityID blob. iri += "?v=" + vstr } - if version != "" { + if version != "" && s.resources != nil { res, err := s.resources.GetResource(ctxLocalPeers, &docspb.GetResourceRequest{ Iri: iri, }) @@ -153,7 +153,7 @@ func (s *Service) DiscoverObjectWithProgress(ctx context.Context, entityID blob. } res := s.syncWithManyPeers(ctxLocalPeers, subsMap, store, prog, auth) - if res.NumSyncOK > 0 { + if res.NumSyncOK > 0 && s.resources != nil { doc, err := s.resources.GetResource(ctxLocalPeers, &docspb.GetResourceRequest{ Iri: iri, }) @@ -190,7 +190,7 @@ func (s *Service) DiscoverObjectWithProgress(ctx context.Context, entityID blob. } res := s.syncWithManyPeers(ctxDHT, subsMap, store, prog, auth) - if res.NumSyncOK > 0 { + if res.NumSyncOK > 0 && s.resources != nil { doc, err := s.resources.GetResource(ctxDHT, &docspb.GetResourceRequest{ Iri: iri, }) diff --git a/backend/llm/backends/backends.go b/backend/llm/backends/backends.go new file mode 100644 index 000000000..8e95ff788 --- /dev/null +++ b/backend/llm/backends/backends.go @@ -0,0 +1,66 @@ +// Package backends defines the embedding backend interface and types. +package backends + +import ( + "context" + "net/url" + "seed/backend/daemon/taskmanager" + "time" +) + +// BackendType identifies an embedding backend implementation. +type BackendType int + +// Backend type constants. +const ( + Ollama BackendType = iota + LlamaCpp +) + +// ModelInfo contains information about an embedding model. +type ModelInfo struct { + // Dimensions is the dimensions of the embedding vector. + Dimensions int + + // ContextSize is the context size of the model. + ContextSize int + + // Checksum is the unique identifier of the model. No other model + // or the same model with different quantization should have the same checksum. + // If the model is updated in any form this value must change. + Checksum string +} + +// ClientCfg contains configuration for an embedding backend client. +type ClientCfg struct { + // URL is the base URL of the embedding backend service. + // It could be an HTTP URL or a file URL depending on the backend. + URL url.URL + //BatchSize is the number of inputs to process in a single batch. + BatchSize int + // WaitBetweenBatches is the duration to wait between processing batches. + WaitBetweenBatches time.Duration + // Model is the name of the model to use. + Model string +} + +// Backend is the interface for embedding model backends. +type Backend interface { + // LoadModel loads the specified model. If force is true, it + // downloads the necesseary files to load the model when not present. + LoadModel(ctx context.Context, model string, force bool, taskMgr *taskmanager.TaskManager) (ModelInfo, error) + // Embed generates embeddings for the given inputs. + // LoadModel must be called before calling Embed. + // Results are normalized. + Embed(ctx context.Context, inputs []string) ([][]float32, error) + // RetrieveSingle generates a single embedding for the given input. + // LoadModel must be called before calling RetrieveSingle. + // Result is normalized. + RetrieveSingle(ctx context.Context, input string) ([]float32, error) + // CloseModel closes the currently active model so no resources are used. + CloseModel(ctx context.Context) error + // Version returns the version of the backend. + Version(ctx context.Context) (string, error) + // TokenLength returns the number of tokens in the input string. + TokenLength(ctx context.Context, input string) (int, error) +} diff --git a/backend/llm/backends/llamacpp/llamacpp.go b/backend/llm/backends/llamacpp/llamacpp.go new file mode 100644 index 000000000..51d9b7611 --- /dev/null +++ b/backend/llm/backends/llamacpp/llamacpp.go @@ -0,0 +1,305 @@ +// Package llamacpp provides an embedding backend using llama.cpp. +package llamacpp + +import ( + "context" + "crypto/sha256" + "embed" + "encoding/hex" + "errors" + "fmt" + "math" + "net/url" + "os" + "runtime" + "seed/backend/daemon/taskmanager" + daemonpb "seed/backend/genproto/daemon/v1alpha" + "seed/backend/llm/backends" + "strings" + "sync" + "time" + + llama "github.com/tcpipuk/llama-go" +) + +//go:embed models/*.gguf +var embeddedModels embed.FS + +const embeddedModelPath = "models/granite-embedding-107m-multilingual-Q8_0.gguf" + +// writeEmbeddedModelToTempFile extracts the embedded GGUF model to a temp file +// and returns its path. Caller is responsible for cleanup. +func writeEmbeddedModelToTempFile() (string, error) { + data, err := embeddedModels.ReadFile(embeddedModelPath) + if err != nil { + return "", fmt.Errorf("reading embedded model: %w", err) + } + f, err := os.CreateTemp("", "seed-embed-*.gguf") + if err != nil { + return "", fmt.Errorf("creating temp file for model: %w", err) + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + _ = os.Remove(f.Name()) + return "", fmt.Errorf("writing embedded model to temp file: %w", err) + } + if err := f.Close(); err != nil { + _ = os.Remove(f.Name()) + return "", err + } + return f.Name(), nil +} + +// Client is an embedding client backed by llama.cpp. +type Client struct { + model *llama.Model + embeddingContext *llama.Context // For generating embeddings + muEmbed sync.Mutex // protects embeddingContext from concurrent access + retrievalContext *llama.Context // For retrieving similar embeddings + muRetrieval sync.Mutex // protects retrievalContext from concurrent access + cfg backends.ClientCfg +} + +// Option configures the Client. +type Option func(*Client) error + +const ( + defaultBatchSize = 10 + maxParallelContexts = 16 + taskID = "llamacpp-load-model-task" + taskDescription = "Loading LlamaCpp model" +) + +// NewClient creates a new LlamaCpp client. +// If fileURL is zero-value (empty scheme), the embedded model is extracted to a temp file. +// If fileURL has scheme "file", the model at that path is used directly. +func NewClient(fileURL url.URL, opts ...Option) (*Client, error) { + if fileURL.Scheme == "" { + // Use embedded model. + tmpPath, err := writeEmbeddedModelToTempFile() + if err != nil { + return nil, fmt.Errorf("extracting embedded model: %w", err) + } + fileURL = url.URL{Scheme: "file", Path: tmpPath} + } + if fileURL.Scheme != "file" { + return nil, fmt.Errorf("llamacpp file URL scheme must be file:///path/to-model, got scheme: %s", fileURL.Scheme) + } + client := &Client{cfg: backends.ClientCfg{BatchSize: defaultBatchSize, URL: fileURL}} + + for _, opt := range opts { + if err := opt(client); err != nil { + return nil, err + } + } + + if client.cfg.BatchSize <= 0 { + return nil, errors.New("llamacpp batch size must be positive") + } + + return client, nil +} + +// WithBatchSize sets the batch size for embedding requests. +func WithBatchSize(size int) Option { + return func(client *Client) error { + client.cfg.BatchSize = size + return nil + } +} + +// WithWaitBetweenBatches waits duration between a full batch size and +// the next full batch size when embedding. +func WithWaitBetweenBatches(duration time.Duration) Option { + return func(client *Client) error { + client.cfg.WaitBetweenBatches = duration + return nil + } +} + +// LoadModel loads a model from the gguf espeficied when initializing the client. +func (client *Client) LoadModel(_ context.Context, _ string, _ bool, taskMgr *taskmanager.TaskManager) (backends.ModelInfo, error) { + path := strings.TrimSpace(client.cfg.URL.Path) + //TODO read gguf model to compute checksum + data, err := os.ReadFile(path) + if err != nil { + return backends.ModelInfo{}, fmt.Errorf("error reading model file: %w", err) + } + localHash := sha256.Sum256(data) + checksum := hex.EncodeToString(localHash[:]) + ret := backends.ModelInfo{Checksum: checksum} + if path == "" { + return ret, errors.New("gguf model name is required") + } + if taskMgr != nil { + if _, err := taskMgr.AddTask(taskID, daemonpb.TaskName_LOADING_MODEL, taskDescription, 100); err != nil { + if errors.Is(err, taskmanager.ErrTaskExists) { + return ret, fmt.Errorf("another model is being loaded, please wait until it ends before loading a new one: %w", err) + } + return ret, err + } + defer func() { + _, _ = taskMgr.DeleteTask(taskID) + }() + } + + client.model, err = llama.LoadModel(path, + llama.WithGPULayers(-1), // Load all layer to GPU + llama.WithMMap(true), + llama.WithSilentLoading(), + llama.WithProgressCallback(func(progress float32) bool { + if taskMgr != nil { + _, _ = taskMgr.UpdateProgress(taskID, 100, int64(progress*100)) + } + return true + }), + ) + if err != nil { + return ret, fmt.Errorf("error loading model: %w", err) + } + + client.embeddingContext, err = client.model.NewContext( + llama.WithThreads(runtime.NumCPU()), + llama.WithEmbeddings(), + llama.WithF16Memory(), + llama.WithParallel(min(maxParallelContexts, client.cfg.BatchSize)), + ) + if err != nil { + return ret, fmt.Errorf("could not create embedding context: %w", err) + } + _, err = client.model.Stats() + if err != nil { + return ret, fmt.Errorf("could not get model stats: %w", err) + } + + client.retrievalContext, err = client.model.NewContext( + llama.WithThreads(runtime.NumCPU()), + llama.WithF16Memory(), + llama.WithParallel(min(maxParallelContexts, client.cfg.BatchSize)), + llama.WithEmbeddings(), + ) + if err != nil { + return ret, fmt.Errorf("could not create retrieval context: %w", err) + } + ret.Dimensions = 384 // Hardcoded for now as llama-go does not expose embedding length yet + ret.ContextSize = 512 // Hardcoded for now as llama-go does not expose context size yet + + // Warm up both contexts to avoid cold-start latency on first real call. + // Yes, this is an ancient hack, ... but it works. + if _, err := client.embeddingContext.GetEmbeddingsBatch([]string{"warmup"}); err != nil { + return ret, fmt.Errorf("failed to warm up embedding context: %w", err) + } + if _, err := client.retrievalContext.GetEmbeddings("warmup"); err != nil { + return ret, fmt.Errorf("failed to warm up retrieval context: %w", err) + } + + return ret, nil +} + +// RetrieveSingle returns the embedding for a single input string. +// The model must be loaded via LoadModel before calling RetrieveSingle. +// Thread-safe: uses mutex to prevent concurrent access to retrievalContext. +func (client *Client) RetrieveSingle(_ context.Context, input string) ([]float32, error) { + client.muRetrieval.Lock() + defer client.muRetrieval.Unlock() + if client.retrievalContext == nil { + return nil, errors.New("llamacpp embedding model is not loaded") + } + embed, err := client.retrievalContext.GetEmbeddings(input) + if err != nil { + return nil, fmt.Errorf("error generating embeddings: %w", err) + } + norm := normalize([][]float32{embed}) + return norm[0], nil +} + +// Embed returns embeddings for inputs in batches sized by the client. +// The model must be loaded via LoadModel before calling Embed. +// Thread-safe: uses mutex to prevent concurrent access to embeddingContext. +func (client *Client) Embed(ctx context.Context, inputs []string) ([][]float32, error) { + client.muEmbed.Lock() // We can't use the same context concurrently + defer client.muEmbed.Unlock() + if client.embeddingContext == nil { + return nil, errors.New("llamacpp embedding model is not loaded") + } + out := make([][]float32, 0, len(inputs)) + var wasPreviousBatchFull bool + for start := 0; start < len(inputs); start += client.cfg.BatchSize { + end := start + client.cfg.BatchSize + if end > len(inputs) { + end = len(inputs) + } + + batch := inputs[start:end] + isBatchFull := len(batch) == client.cfg.BatchSize + if client.cfg.WaitBetweenBatches > 0 && wasPreviousBatchFull && isBatchFull { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(client.cfg.WaitBetweenBatches): + } + } + wasPreviousBatchFull = isBatchFull + res, err := client.embeddingContext.GetEmbeddingsBatch(batch) + if err != nil { + return nil, fmt.Errorf("error generating embeddings: %w", err) + } + + if len(res) != len(batch) { + return nil, fmt.Errorf("llama embeddings count mismatch: got %d want %d", len(res), len(batch)) + } + norm := normalize(res) + out = append(out, norm...) + } + return out, nil +} + +func normalize(vectors [][]float32) [][]float32 { + for _, batch := range vectors { + magnitude := float32(0.0) + for _, val := range batch { + magnitude += val * val + } + norm := float32(math.Sqrt(float64(magnitude))) + if norm > 0 { + for i := range batch { + batch[i] /= norm + } + } + } + return vectors +} + +// Version returns the Ollama server version string. +// Version returns the model version string. +func (client *Client) Version(_ context.Context) (string, error) { + stats, err := client.model.Stats() + if err != nil { + return "", err + } + return strings.Join([]string{stats.Metadata.Name, + stats.Metadata.Architecture, + stats.Metadata.QuantizedBy, + stats.Metadata.SizeLabel}, "_"), nil +} + +// TokenLength returns the number of tokens in the input string. +func (client *Client) TokenLength(_ context.Context, input string) (int, error) { + tokens, err := client.embeddingContext.Tokenize(input) + if err != nil { + return 0, err + } + return len(tokens), nil +} + +// CloseModel releases the model and its contexts. +func (client *Client) CloseModel(_ context.Context) error { + var errs []error + if client.embeddingContext != nil { + errs = append(errs, client.embeddingContext.Close()) + } + if client.model != nil { + errs = append(errs, client.model.Close()) + } + return errors.Join(errs...) +} diff --git a/backend/llm/backends/llamacpp/llamacpp_test.go b/backend/llm/backends/llamacpp/llamacpp_test.go new file mode 100644 index 000000000..8efcd5f31 --- /dev/null +++ b/backend/llm/backends/llamacpp/llamacpp_test.go @@ -0,0 +1,140 @@ +package llamacpp + +import ( + "context" + "math" + "net/url" + "seed/backend/daemon/taskmanager" + "seed/backend/testutil" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestLlamaCppClientEmbeddings(t *testing.T) { + testutil.Manual(t) + ctx := t.Context() + client, err := NewClient(url.URL{}, WithBatchSize(2)) + require.NoError(t, err) + t.Cleanup(func() { _ = client.CloseModel(ctx) }) + + info, err := client.LoadModel(ctx, "", false, taskmanager.NewTaskManager()) + require.NoError(t, err) + require.Greater(t, info.Dimensions, 0) + require.Greater(t, info.ContextSize, 0) + + inputs := []string{"alpha", "bravo", "charlie", "delta", "echo"} + embeddings, err := client.Embed(ctx, inputs) + require.NoError(t, err) + require.Len(t, embeddings, len(inputs)) + require.Len(t, embeddings, len(inputs)) + + for i, embedding := range embeddings { + require.Len(t, embedding, info.Dimensions) + // Calculate L2 norm (magnitude) + var magnitude float32 + for _, val := range embedding { + magnitude += val * val + } + norm := float32(math.Sqrt(float64(magnitude))) + + // Post-normalization L2 norm should be ~1.0 + require.InDelta(t, 1.0, norm, 0.0001, "embedding %d should have L2 norm of 1.0, got %.6f", i, norm) + } +} + +func TestLlamaCppClientEmbedEmptyInput(t *testing.T) { + testutil.Manual(t) + ctx := t.Context() + client, err := NewClient(url.URL{}) + require.NoError(t, err) + t.Cleanup(func() { _ = client.CloseModel(ctx) }) + + _, err = client.LoadModel(ctx, "", false, taskmanager.NewTaskManager()) + require.NoError(t, err) + embeddings, err := client.Embed(ctx, nil) + require.NoError(t, err) + require.Empty(t, embeddings) +} + +func TestLlamaCppClientRequiresFileScheme(t *testing.T) { + httpURL, err := url.Parse("http://example.com") + require.NoError(t, err) + _, err = NewClient(*httpURL) + require.Error(t, err) + require.Contains(t, err.Error(), "file") +} + +func TestLlamaCppClientBatchSizeMustBePositive(t *testing.T) { + _, err := NewClient(url.URL{}, WithBatchSize(0)) + require.Error(t, err) + require.Contains(t, err.Error(), "positive") +} + +func TestLlamaCppClientEmbed_WaitsBetweenFullBatches(t *testing.T) { + testutil.Manual(t) + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + + client, err := NewClient( + url.URL{}, + WithBatchSize(2), + WithWaitBetweenBatches(5*time.Second), + ) + require.NoError(t, err) + t.Cleanup(func() { _ = client.CloseModel(context.Background()) }) + + _, err = client.LoadModel(ctx, "", false, taskmanager.NewTaskManager()) + require.NoError(t, err) + + // Two full batches (2 + 2). The client must wait before the 2nd batch. + _, err = client.Embed(ctx, []string{"a", "b", "c", "d"}) + require.Error(t, err) + require.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestNormalizeFunction(t *testing.T) { + // Test the normalize function directly with known values + vectors := [][]float32{ + {3.0, 4.0}, // norm = 5, normalized = {0.6, 0.8} + {1.0, 0.0, 0.0}, // norm = 1, normalized = {1, 0, 0} + {2.0, 2.0, 1.0}, // norm = 3, normalized = {2/3, 2/3, 1/3} + } + + result := normalize(vectors) + + // First vector: [3,4] -> norm=5 -> [0.6, 0.8] + require.InDelta(t, 0.6, result[0][0], 0.0001) + require.InDelta(t, 0.8, result[0][1], 0.0001) + + // Second vector: [1,0,0] -> norm=1 -> [1, 0, 0] + require.InDelta(t, 1.0, result[1][0], 0.0001) + require.InDelta(t, 0.0, result[1][1], 0.0001) + require.InDelta(t, 0.0, result[1][2], 0.0001) + + // Third vector: [2,2,1] -> norm=3 -> [2/3, 2/3, 1/3] + require.InDelta(t, 2.0/3.0, result[2][0], 0.0001) + require.InDelta(t, 2.0/3.0, result[2][1], 0.0001) + require.InDelta(t, 1.0/3.0, result[2][2], 0.0001) + + // Verify all vectors now have L2 norm of 1.0 + for i, vec := range result { + var magnitude float32 + for _, val := range vec { + magnitude += val * val + } + norm := float32(math.Sqrt(float64(magnitude))) + require.InDelta(t, 1.0, norm, 0.0001, "vector %d should have L2 norm of 1.0", i) + } +} + +func TestNormalizeZeroVector(t *testing.T) { + // Zero vectors should remain zero (avoid division by zero) + vectors := [][]float32{{0.0, 0.0, 0.0}} + result := normalize(vectors) + + require.Equal(t, float32(0.0), result[0][0]) + require.Equal(t, float32(0.0), result[0][1]) + require.Equal(t, float32(0.0), result[0][2]) +} diff --git a/backend/llm/backends/ollama/ollama.go b/backend/llm/backends/ollama/ollama.go new file mode 100644 index 000000000..395c925dd --- /dev/null +++ b/backend/llm/backends/ollama/ollama.go @@ -0,0 +1,347 @@ +// Package ollama provides an embedding backend using an Ollama server. +package ollama + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "seed/backend/daemon/taskmanager" + "seed/backend/llm/backends" + "strconv" + "strings" + "time" + + "github.com/ollama/ollama/api" + "github.com/ollama/ollama/types/model" +) + +const ( + defaultBatchSize = 10 + defaultHTTPTimeout = 5 * time.Minute +) + +// Client is an embedding client backed by an Ollama server. +type Client struct { + cfg backends.ClientCfg + http *http.Client + client *api.Client +} + +// Option configures the Client. +type Option func(*Client) error + +// NewClient creates a new Ollama client bound to the provided base URL. +func NewClient(baseURL url.URL, opts ...Option) (*Client, error) { + client := &Client{ + http: &http.Client{Timeout: defaultHTTPTimeout}, + cfg: backends.ClientCfg{BatchSize: defaultBatchSize, URL: baseURL}, + } + + for _, opt := range opts { + if err := opt(client); err != nil { + return nil, err + } + } + + if client.cfg.BatchSize <= 0 { + return nil, errors.New("ollama batch size must be positive") + } + + client.client = api.NewClient(&client.cfg.URL, client.http) + + return client, nil +} + +// WithHTTPTransport overrides the HTTP client used for Ollama requests. +func WithHTTPTransport(httpClient *http.Client) Option { + return func(client *Client) error { + if httpClient == nil { + return errors.New("ollama http client is required") + } + + client.http = httpClient + return nil + } +} + +// WithBatchSize sets the batch size for embedding requests. +func WithBatchSize(size int) Option { + return func(client *Client) error { + client.cfg.BatchSize = size + return nil + } +} + +// WithWaitBetweenBatches waits duration between a full batch size and +// the next full batch size when embedding. +func WithWaitBetweenBatches(duration time.Duration) Option { + return func(client *Client) error { + client.cfg.WaitBetweenBatches = duration + return nil + } +} + +// WithHTTPTimeout sets the HTTP client timeout used for Ollama requests. +// This covers the entire request (connect + send + wait for headers/body). +func WithHTTPTimeout(timeout time.Duration) Option { + return func(client *Client) error { + if timeout <= 0 { + return errors.New("ollama http timeout must be positive") + } + if client.http == nil { + client.http = &http.Client{} + } + client.http.Timeout = timeout + return nil + } +} + +// CloseModel is a no-op for Ollama (no local resources to release). +func (client *Client) CloseModel(_ context.Context) error { + return nil +} + +// LoadModel ensures a model is available; when force is true it pulls it. +// It returns the embedding dimensions and context size from the model metadata. +func (client *Client) LoadModel(ctx context.Context, model string, force bool, _ *taskmanager.TaskManager) (backends.ModelInfo, error) { + model = strings.TrimSpace(model) + ret := backends.ModelInfo{} + if model == "" { + return ret, errors.New("ollama model name is required") + } + + showResponse, err := client.client.Show(ctx, &api.ShowRequest{Model: model}) + if err == nil { + ret, parseErr := parseModelInfo(model, showResponse) + if parseErr != nil { + return ret, parseErr + } + client.cfg.Model = model + + return ret, nil + } else if !force { + var statusError api.StatusError + if errors.As(err, &statusError) && statusError.StatusCode == http.StatusNotFound { + return ret, fmt.Errorf("ollama model not found: %s", model) + } + + return ret, err + } + + stream := false + request := &api.PullRequest{ + Model: model, + Stream: &stream, + } + + if err := client.client.Pull(ctx, request, func(api.ProgressResponse) error { + return nil + }); err != nil { + return backends.ModelInfo{}, err + } + + showResponse, err = client.client.Show(ctx, &api.ShowRequest{Model: model}) + if err != nil { + return backends.ModelInfo{}, err + } + + info, err := parseModelInfo(model, showResponse) + if err != nil { + return backends.ModelInfo{}, err + } + + client.cfg.Model = model + return info, nil +} + +// RetrieveSingle returns a single embedding for the input. +func (client *Client) RetrieveSingle(ctx context.Context, input string) ([]float32, error) { + model := strings.TrimSpace(client.cfg.Model) + if model == "" { + return nil, errors.New("ollama model not loaded; call LoadModel first") + } + + request := &api.EmbedRequest{ + Model: model, + Input: []string{input}, + } + response, err := client.client.Embed(ctx, request) + if err != nil { + return nil, err + } + + if len(response.Embeddings) != 1 { + return nil, fmt.Errorf("ollama single embedding count mismatch: got %d want %d", len(response.Embeddings), 1) + } + + return response.Embeddings[0], nil +} + +// Embed returns embeddings for inputs in batches sized by the client. +// The model must be loaded via LoadModel before calling Embed. +func (client *Client) Embed(ctx context.Context, inputs []string) ([][]float32, error) { + model := strings.TrimSpace(client.cfg.Model) + if model == "" { + return nil, errors.New("ollama model not loaded; call LoadModel first") + } + if len(inputs) == 0 { + return [][]float32{}, nil + } + + embeddings := make([][]float32, 0, len(inputs)) + var wasPreviousBatchFull bool + for start := 0; start < len(inputs); start += client.cfg.BatchSize { + end := start + client.cfg.BatchSize + if end > len(inputs) { + end = len(inputs) + } + + batch := inputs[start:end] + isBatchFull := len(batch) == client.cfg.BatchSize + if client.cfg.WaitBetweenBatches > 0 && wasPreviousBatchFull && isBatchFull { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(client.cfg.WaitBetweenBatches): + } + } + wasPreviousBatchFull = isBatchFull + + request := &api.EmbedRequest{ + Model: model, + Input: batch, + } + response, err := client.client.Embed(ctx, request) + if err != nil { + return nil, err + } + + if len(response.Embeddings) != len(batch) { + return nil, fmt.Errorf("ollama embeddings count mismatch: got %d want %d", len(response.Embeddings), len(batch)) + } + + embeddings = append(embeddings, response.Embeddings...) + } + + return embeddings, nil +} + +// TokenLength returns the number of tokens in the input string. +func (client *Client) TokenLength(_ context.Context, _ string) (int, error) { + return 0, errors.New("ollama client does not support token length calculation") +} + +func parseModelInfo(model string, response *api.ShowResponse) (backends.ModelInfo, error) { + if response == nil { + return backends.ModelInfo{}, fmt.Errorf("ollama model info missing: %s", model) + } + + if !hasEmbeddingCapability(response.Capabilities) { + return backends.ModelInfo{}, fmt.Errorf("ollama model does not support embeddings: %s", model) + } + + dimensions := readIntFromInfo(response.ModelInfo, embeddingDimensionKeys) + if dimensions == 0 { + dimensions = readIntFromInfo(response.ProjectorInfo, embeddingDimensionKeys) + } + if dimensions == 0 { + return backends.ModelInfo{}, fmt.Errorf("ollama model embedding dimensions missing: %s", model) + } + + contextSize := readIntFromInfo(response.ModelInfo, contextSizeKeys) + if contextSize == 0 { + contextSize = readIntFromInfo(response.ProjectorInfo, contextSizeKeys) + } + if contextSize == 0 { + return backends.ModelInfo{}, fmt.Errorf("ollama model context size missing: %s", model) + } + data, err := json.Marshal(response) + if err != nil { + return backends.ModelInfo{}, fmt.Errorf("ollama model info marshal error: %w", err) + } + + localHash := sha256.Sum256(data) + checksum := hex.EncodeToString(localHash[:]) + return backends.ModelInfo{Dimensions: dimensions, ContextSize: contextSize, Checksum: checksum}, nil +} + +func readIntFromInfo(info map[string]any, keys []string) int { + if len(info) == 0 { + return 0 + } + + for infoKey, value := range info { + lowerKey := strings.ToLower(infoKey) + if !matchesAnyKey(lowerKey, keys) { + continue + } + + switch typed := value.(type) { + case int: + return typed + case int32: + return int(typed) + case int64: + return int(typed) + case float32: + return int(typed) + case float64: + return int(typed) + case string: + parsed, err := strconv.Atoi(typed) + if err == nil { + return parsed + } + } + } + + return 0 +} + +func matchesAnyKey(infoKey string, keys []string) bool { + for _, key := range keys { + if strings.Contains(infoKey, key) { + return true + } + } + + return false +} + +func hasEmbeddingCapability(capabilities []model.Capability) bool { + for _, capability := range capabilities { + if capability.String() == "embedding" || capability.String() == "embeddings" { + return true + } + } + + return false +} + +var embeddingDimensionKeys = []string{ + "embedding_length", + "embedding_size", + "embedding_dim", + "embedding_dimension", + "n_embd", + "hidden_size", +} + +var contextSizeKeys = []string{ + "context_length", + "max_context_length", + "max_sequence_length", + "context_size", + "n_ctx", + "n_ctx_train", +} + +// Version returns the Ollama server version string. +func (client *Client) Version(ctx context.Context) (string, error) { + return client.client.Version(ctx) +} diff --git a/backend/llm/backends/ollama/ollama_test.go b/backend/llm/backends/ollama/ollama_test.go new file mode 100644 index 000000000..71e8bade2 --- /dev/null +++ b/backend/llm/backends/ollama/ollama_test.go @@ -0,0 +1,108 @@ +package ollama + +import ( + "context" + "net/url" + "seed/backend/testutil" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestOllamaClientEmbeddings(t *testing.T) { + ctx := t.Context() + const model = "embeddinggemma" + mockServer := testutil.NewMockOllamaServer(t) + t.Cleanup(mockServer.Server.Close) + url, err := url.Parse(mockServer.Server.URL) + require.NoError(t, err) + client, err := NewClient(*url, WithBatchSize(2)) + require.NoError(t, err) + + info, err := client.LoadModel(ctx, model, true, nil) + require.NoError(t, err) + require.Equal(t, 384, info.Dimensions) + require.Equal(t, 2048, info.ContextSize) + + inputs := []string{"alpha", "bravo", "charlie", "delta", "echo"} + embeddings, err := client.Embed(ctx, inputs) + require.NoError(t, err) + require.Len(t, embeddings, len(inputs)) + + for index, embedding := range embeddings { + require.Len(t, embedding, 384) + require.Equal(t, float32(len(inputs[index])), embedding[0]) + } + + mockServer.Mu.Lock() + defer mockServer.Mu.Unlock() + + require.Empty(t, mockServer.LoadedModels) + require.Equal(t, []int{2, 2, 1}, mockServer.BatchSizes) + require.Equal(t, len(inputs), mockServer.SeenEmbeddings) + require.Equal(t, 1, mockServer.ShowRequests) +} + +func TestOllamaClientEmbedEmptyInput(t *testing.T) { + ctx := t.Context() + const model = "embeddinggemma" + + mockServer := testutil.NewMockOllamaServer(t) + t.Cleanup(mockServer.Server.Close) + + url, err := url.Parse(mockServer.Server.URL) + require.NoError(t, err) + client, err := NewClient(*url) + require.NoError(t, err) + + _, err = client.LoadModel(ctx, model, true, nil) + require.NoError(t, err) + embeddings, err := client.Embed(ctx, nil) + require.NoError(t, err) + require.Empty(t, embeddings) +} + +func TestOllamaClientEmbedRequiresModel(t *testing.T) { + ctx := t.Context() + + url, err := url.Parse("http://example.com") + require.NoError(t, err) + client, err := NewClient(*url) + //client, err := NewClient("file:///home/julio/Documents/seed/backend/llm/backends/ollama/ollama.go") + require.NoError(t, err) + + _, err = client.Embed(ctx, []string{"alpha"}) + require.Error(t, err) + require.Contains(t, err.Error(), "LoadModel") +} + +func TestOllamaClientEmbed_WaitsBetweenFullBatches(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 50*time.Millisecond) + defer cancel() + + const model = "embeddinggemma" + mockServer := testutil.NewMockOllamaServer(t) + t.Cleanup(mockServer.Server.Close) + + url, err := url.Parse(mockServer.Server.URL) + require.NoError(t, err) + client, err := NewClient( + *url, + WithBatchSize(2), + WithWaitBetweenBatches(5*time.Second), + ) + require.NoError(t, err) + + _, err = client.LoadModel(ctx, model, true, nil) + require.NoError(t, err) + + // Two full batches (2 + 2). The client must wait before the 2nd batch. + _, err = client.Embed(ctx, []string{"a", "b", "c", "d"}) + require.Error(t, err) + require.ErrorIs(t, err, context.DeadlineExceeded) + + mockServer.Mu.Lock() + defer mockServer.Mu.Unlock() + require.Equal(t, 1, mockServer.EmbedRequests, "second embed request must not be sent once ctx expires during wait") +} diff --git a/backend/llm/embedding.go b/backend/llm/embedding.go new file mode 100644 index 000000000..81add5a03 --- /dev/null +++ b/backend/llm/embedding.go @@ -0,0 +1,911 @@ +// Package llm provides embedding generation and semantic search. +package llm + +import ( + "context" + "errors" + "fmt" + "math" + "slices" + "strings" + "sync" + "time" + + "seed/backend/daemon/taskmanager" + daemonpb "seed/backend/genproto/daemon/v1alpha" + "seed/backend/llm/backends" + "seed/backend/util/dqb" + "seed/backend/util/sqlite" + "seed/backend/util/sqlite/sqlitex" + + "go.uber.org/zap" +) + +const ( + // DefaultEmbeddingIndexPassSize is the default number of FTS rows to keep in memory per pass. + // After each pass, the embedder sleeps for a short time to avoid starving the CPU. + // Adjust the sleep duration via WithSleepPerPass. + DefaultEmbeddingIndexPassSize = 10 + + // DefaultEmbeddingSleepBetweenPasses is the default sleep duration after each indexing pass. + DefaultEmbeddingSleepBetweenPasses = time.Millisecond * 500 // to not starve the CPU. + + // DefaultEmbeddingRunInterval is the default wait time after a run finishes before starting the next one. + DefaultEmbeddingRunInterval = 1 * time.Minute + + // DefaultEmbeddingModel is the default model name for embeddings. + DefaultEmbeddingModel = "embeddinggemma" + + taskID = "embedding_indexer" + taskDescription = "Indexing embeddings" + embeddingColumnDims = 384 + pctOverlap = 0.1 + minRunInterval = 5 * time.Second + + kvEmbeddingModelChecksumKey = "embedding_model_checksum" + + // unreliableEmbeddingThreshold is the cosine similarity threshold above which a query + // embedding is considered unreliable (too similar to gibberish). + // The granite-embedding-107m-multilingual model produces higher base similarities across + // all queries (~0.55-0.70), so 0.85 catches only true nonsense strings. + unreliableEmbeddingThreshold = 0.85 +) + +// ErrUnreliableEmbedding is returned when the query embedding is detected as unreliable. +// This happens when rare/unknown single words produce embeddings highly similar to gibberish, +// making semantic search results meaningless. Callers should fall back to keyword search. +var ErrUnreliableEmbedding = errors.New("query embedding is unreliable for semantic search") + +// gibberishEmbedding is a precomputed quantized embedding for the nonsense string "asdadadsasda" +// using the granite-embedding-107m-multilingual model. This embedding is used to detect queries +// that produce unreliable embeddings (too similar to gibberish). When detected, semantic search +// results should be skipped in favor of keyword search. +// WARNING: This embedding is model-specific and MUST be recomputed if the embedding model changes. +var gibberishEmbedding = []int8{ + 23, -5, 15, 26, 11, 4, 16, 20, 31, -11, -23, 15, 10, 17, 5, -2, + -2, 7, 37, -11, -2, -4, 35, 45, -67, 19, 1, 17, 18, 5, -1, 24, + 18, -47, -16, 33, 44, 22, 10, 12, 2, 32, 22, 16, -8, 9, 9, 12, + 8, -48, -1, 11, 2, 0, -10, 21, 2, 22, -2, -7, 4, 22, 17, 61, + 8, -41, -2, 1, 22, -15, 24, 1, 15, 19, 6, 51, -5, 15, 16, 34, + 6, -8, 0, 9, 45, 64, 13, 3, 31, 15, 27, -40, 1, 7, -3, 1, + -104, 19, 5, 15, 4, 4, 7, 24, 12, 8, 48, 25, 16, 11, 50, -11, + 25, -8, -3, 32, 12, 36, 11, -22, 20, 15, 2, 28, -6, 6, -2, 36, + -11, 10, -9, 18, 26, -5, 8, -15, 27, -2, -127, -20, 1, 12, 31, 28, + -42, 36, 17, 30, 12, -19, 22, 19, 12, 9, 19, -1, 39, 16, 10, 10, + -7, 1, 10, 8, 9, -11, 7, 50, -7, 12, 0, 21, 7, 13, 2, 38, + 3, 13, 33, 31, -25, 1, 18, -21, -39, 16, -28, -57, 28, 31, 7, 41, + 11, 30, 2, 3, 26, 82, 10, 4, 1, 2, 7, 5, 5, 24, 31, 20, + 2, 11, 18, 15, 1, 0, -3, 4, 8, 9, 33, -16, 19, 13, 24, 11, + 30, 9, 7, -15, 23, -27, 7, 23, 6, -12, -2, -67, 20, 29, 4, 23, + 15, -11, 18, 10, 15, -2, 22, 10, 15, 21, 29, -3, 28, -30, 12, 54, + 34, -4, 20, -1, 9, 7, 6, 40, 33, -4, 39, 6, -5, 0, 5, -4, + -3, 35, 22, 4, 64, 19, -6, 33, 12, -23, 14, 3, 22, 15, -3, 14, + 32, -26, -12, -2, 3, 22, 28, 16, 31, 17, 26, 18, 23, -9, 20, 42, + 123, 25, -2, 2, 31, -13, -5, 0, 6, 2, 32, 8, -43, 10, 20, 24, + 8, 37, 30, -1, -16, 16, -28, 11, 18, 6, -1, -18, -31, 27, -27, 26, + -22, 35, -15, 22, 26, 13, 4, 12, 12, 31, -60, 14, 13, 20, -3, 17, + 39, -81, 12, 13, 2, -31, 37, 22, -48, -6, 23, 8, 15, -5, 20, -16, + -3, 4, 52, -54, 19, 2, -11, 37, -8, 3, 26, 25, 10, 14, -41, 33, +} + +// LightEmbedder defines a minimal interface for semantic search. +// Returns the top limit results matching the query. +// Threshold is the minimum similarity score (0.0 to 1.0) to include in results. +type LightEmbedder interface { + SemanticSearch(ctx context.Context, query string, limit int, contentTypes map[string]bool, iriGlob string, threshold float32) (SearchResultMap, error) +} + +// Embedder handles embedding generation and indexing. +type Embedder struct { + backend backends.Backend + pool *sqlitex.Pool + logger *zap.Logger + taskMgr *taskmanager.TaskManager + model string + indexPassSize int + interval time.Duration + SleepBetweenPasses time.Duration + forceLoad bool + dimensions int + contextSize int + modelLoaded bool + initialized bool + documentPrefix string + queryPrefix string + maxChunkLength int + mu sync.Mutex +} + +// EmbedderOption configures the embedder. +type EmbedderOption func(*Embedder) error + +// WithIndexPassSize sets the number of FTS rows to embed per pass. Default is 100. +// It is not the same as the backend batch size. This controls how many rows are +// fetched from the database per run. Also, after each pass, the embedder sleeps +// for a short time to avoid starving the CPU. Set the sleep interval via WithSleepPerPass. +func WithIndexPassSize(size int) EmbedderOption { + return func(embedder *Embedder) error { + if size <= 0 { + return errors.New("embedder pass size must be positive") + } + embedder.indexPassSize = size + return nil + } +} + +// WithSleepPerPass sets the sleep duration after each indexing pass. +// Default is 10ms. +func WithSleepPerPass(duration time.Duration) EmbedderOption { + return func(embedder *Embedder) error { + embedder.SleepBetweenPasses = duration + return nil + } +} + +// WithForceLoad makes LoadModel pull the model when it is missing on the backend. +func WithForceLoad(force bool) EmbedderOption { + return func(embedder *Embedder) error { + embedder.forceLoad = force + return nil + } +} + +// WithInterval sets the default wait time after a run finishes before starting the next one. +func WithInterval(interval time.Duration) EmbedderOption { + return func(embedder *Embedder) error { + if interval < minRunInterval { + return fmt.Errorf("embedder interval must be at least %s", minRunInterval) + } + embedder.interval = interval + return nil + } +} + +// WithModel sets the model name used by the embedder. +func WithModel(model string) EmbedderOption { + return func(embedder *Embedder) error { + trimmed := strings.TrimSpace(model) + if trimmed == "" { + return errors.New("embedder model name is required") + } + embedder.model = trimmed + return nil + } +} + +// WithDocumentPrefix sets the prefix to add to document texts before embedding. +func WithDocumentPrefix(prefix string) EmbedderOption { + return func(embedder *Embedder) error { + embedder.documentPrefix = prefix + return nil + } +} + +// WithQueryPrefix sets the prefix to add to query texts before semantic searching. +func WithQueryPrefix(prefix string) EmbedderOption { + return func(embedder *Embedder) error { + embedder.queryPrefix = prefix + return nil + } +} + +// NewEmbedder creates an embedder. +func NewEmbedder( + pool *sqlitex.Pool, + backend backends.Backend, + logger *zap.Logger, + taskMgr *taskmanager.TaskManager, + opts ...EmbedderOption, +) (*Embedder, error) { + if pool == nil { + return nil, errors.New("embedder pool is required") + } + if backend == nil { + return nil, errors.New("embedder backend is required") + } + if logger == nil { + return nil, errors.New("embedder logger is required") + } + if taskMgr == nil { + return nil, errors.New("embedder task manager is required") + } + + embedder := &Embedder{ + backend: backend, + pool: pool, + logger: logger, + taskMgr: taskMgr, + indexPassSize: DefaultEmbeddingIndexPassSize, + SleepBetweenPasses: DefaultEmbeddingSleepBetweenPasses, + interval: DefaultEmbeddingRunInterval, + } + + for _, opt := range opts { + if err := opt(embedder); err != nil { + return nil, err + } + } + + if strings.TrimSpace(embedder.model) == "" { + return nil, errors.New("embedder model name is required") + } + + return embedder, nil +} + +// Init starts the indexing loop using the provided interval in the constructor. +// It runs through the database getting textx, chunk them, and generating embeddings. +// Calling Init multiple times has no effect. +// If the user just wants to embed textx on demand (For semantic search), it can call +// EmbedText directly. +func (e *Embedder) Init(ctx context.Context) { + e.mu.Lock() + if e.initialized { + e.mu.Unlock() + return + } + e.mu.Unlock() + if err := e.ensureModel(ctx); err != nil { + e.logger.Warn("Could not ensure LLM model", zap.Error(err)) + return + } + e.mu.Lock() + e.initialized = true + e.mu.Unlock() + + // Start the indexing loop only once + go func() { + for { + if err := e.runOnce(ctx); err != nil && !errors.Is(err, context.Canceled) { + e.logger.Warn("embedding indexing failed", zap.Error(err)) + } + + if e.interval <= 0 { + e.logger.Info("embedding indexing completed, not restarting due to non-positive interval") + return + } + + select { + case <-ctx.Done(): + e.logger.Info("embedding indexing stopped", zap.Error(ctx.Err())) + return + case <-time.After(e.interval): + } + } + }() +} + +// SearchResultMap represents a minimal search result from semantic or keyword search. +// The key is the rowID of the FTS entry, and the value is the score. +// In the case of semantic search, the score is the similarity (0.0 to 1.0). +// The higher the score, the more relevant. +// In the case of keyword search, the score is the FTS rank. Usually the more +// negative, the more relevant. +type SearchResultMap map[int64]float32 + +// SearchResult is a single search result with a row ID and score. +type SearchResult struct { + // RowID is the FTS row ID. + RowID int64 + // Score is the relevance score. Depending on the search type, higher or lower is better. + Score float32 +} + +// Keys returns a sorted list of rowIDs in the SearchResultMap for deterministic ordering. +func (sr SearchResultMap) Keys() []int64 { + keys := make([]int64, 0, len(sr)) + for k := range sr { + keys = append(keys, k) + } + slices.Sort(keys) + return keys +} + +// Values returns an unordered list of scores in the SearchResultMap. +func (sr SearchResultMap) Values() []float32 { + values := []float32{} + for _, score := range sr { + values = append(values, score) + } + return values +} + +// Max returns the fts rowID if the maximum score found in the result set. +func (sr SearchResultMap) Max() SearchResult { + var maxScore float32 + first := true + var maxID int64 + for id, score := range sr { + if first || score > maxScore { + maxScore = score + maxID = id + first = false + } + } + return SearchResult{RowID: maxID, Score: maxScore} +} + +// Min returns the fts rowID of the minimum score found in the result set. +func (sr SearchResultMap) Min() SearchResult { + var minScore float32 + first := true + var minID int64 + for id, score := range sr { + if first || score < minScore { + minScore = score + minID = id + first = false + } + } + return SearchResult{RowID: minID, Score: minScore} +} + +// ToList converts the SearchResultMap to a sorted list of SearchResult. +// If desc is true, the list is sorted in descending order of Score. +// Uses RowID as tie-breaker for deterministic ordering when scores are equal. +func (sr SearchResultMap) ToList(desc bool) SearchResultList { + results := make([]SearchResult, 0, len(sr)) + for id, score := range sr { + results = append(results, SearchResult{RowID: id, Score: score}) + } + slices.SortFunc(results, func(a, b SearchResult) int { + if desc { + switch { + case a.Score > b.Score: + return -1 + case a.Score < b.Score: + return 1 + default: + // Tie-breaker: sort by RowID for deterministic ordering. + switch { + case a.RowID < b.RowID: + return -1 + case a.RowID > b.RowID: + return 1 + default: + return 0 + } + } + } + switch { + case a.Score < b.Score: + return -1 + case a.Score > b.Score: + return 1 + default: + // Tie-breaker: sort by RowID for deterministic ordering. + switch { + case a.RowID < b.RowID: + return -1 + case a.RowID > b.RowID: + return 1 + default: + return 0 + } + } + }) + return results +} + +// SearchResultList is an ordered list of SearchResult. +type SearchResultList []SearchResult + +// ToMap converts the SearchResultList to a SearchResultMap. +func (srList SearchResultList) ToMap() SearchResultMap { + resultMap := make(SearchResultMap) + for _, sr := range srList { + resultMap[sr.RowID] = sr.Score + } + return resultMap +} + +// SemanticSearch performs semantic search using sqlite-vec cosine similarity. +// contentTypes filters by FTS content types (e.g., "title", "document", "comment"). +// If empty, defaults to ["title", "document", "comment"]. +// iriGlob filters results by IRI pattern. If empty, defaults to "*" (all). +// Threshold filters results by minimum similarity score (0.0 to 1.0). Default is 0.0 (no filtering). +func (e *Embedder) SemanticSearch(ctx context.Context, query string, limit int, contentTypes map[string]bool, iriGlob string, threshold float32) (SearchResultMap, error) { + if limit <= 0 { + limit = 20 + } + + if iriGlob == "" { + iriGlob = "*" + } + e.mu.Lock() + if !e.modelLoaded { + e.mu.Unlock() + return nil, fmt.Errorf("embedder model not loaded") + } + e.mu.Unlock() + + // Embed query with optional prefix + queryText := query + if e.queryPrefix != "" { + queryText = e.queryPrefix + query + } + embedding, err := e.backend.RetrieveSingle(ctx, queryText) + if err != nil { + return nil, fmt.Errorf("failed to embed query: %w", err) + } + if len(embedding) != e.dimensions { + return nil, fmt.Errorf("embedding dimension mismatch: got %d want %d", len(embedding), e.dimensions) + } + queryEmbedding := quantizeEmbedding(embedding) + + // Detect unreliable embeddings by checking similarity to gibberish. + // Rare/unknown single words produce degenerate embeddings that are highly similar to nonsense. + if sim := cosineSimilarityInt8(queryEmbedding, gibberishEmbedding); sim > unreliableEmbeddingThreshold { + return nil, ErrUnreliableEmbedding + } + + var entityTypeTitle, entityTypeContact, entityTypeDoc, entityTypeComment interface{} + supportedType := false + if ok, val := contentTypes["title"]; ok && val { + entityTypeTitle = "title" + supportedType = true + } + if ok, val := contentTypes["contact"]; ok && val { + entityTypeContact = "contact" + supportedType = true + } + if ok, val := contentTypes["document"]; ok && val { + entityTypeDoc = "document" + supportedType = true + } + if ok, val := contentTypes["comment"]; ok && val { + entityTypeComment = "comment" + supportedType = true + } + if !supportedType { + return nil, fmt.Errorf("invalid content type filter: at least one of title, contact, document, comment must be specified") + } + conn, release, err := e.pool.Conn(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get database connection: %w", err) + } + defer release() + // Convert threshold from similarity to distance + if threshold <= 0 { + threshold = -0.1 // there could be distances slightly above 1.0 due to quantization errors + } + maxDistance := 1 - float64(threshold) + ret := make(map[int64]float32) + + // Determine if we need IRI pre-filtering. + // Generic patterns like "*" or "hm://*" don't need filtering. + needsIriFilter := iriGlob != "*" && iriGlob != "hm://*" + + resultHandler := func(stmt *sqlite.Stmt) error { + distance := stmt.ColumnFloat(1) + similarity := max(0, 1-distance) + ret[stmt.ColumnInt64(0)] = float32(similarity) + return nil + } + + if needsIriFilter { + // Use pre-filtered query with fts_id IN (subquery). + // The subquery parameters are duplicated for both UNION branches. + if err := sqlitex.Exec(conn, qEmbeddingsSearchFiltered(), resultHandler, + queryEmbedding, maxDistance, limit, + entityTypeTitle, entityTypeContact, entityTypeDoc, entityTypeComment, iriGlob, + entityTypeTitle, entityTypeContact, entityTypeDoc, entityTypeComment, iriGlob, + ); err != nil { + return nil, fmt.Errorf("semantic search query failed: %w", err) + } + } else { + // Use unfiltered query for generic IRI patterns. + if err := sqlitex.Exec(conn, qEmbeddingsSearchUnfiltered(), resultHandler, + queryEmbedding, maxDistance, limit, + entityTypeTitle, entityTypeContact, entityTypeDoc, entityTypeComment, + ); err != nil { + return nil, fmt.Errorf("semantic search query failed: %w", err) + } + } + + return ret, nil +} + +func (e *Embedder) runOnce(ctx context.Context) error { + conn, release, err := e.pool.Conn(ctx) + if err != nil { + return err + } + + totalEmbeddable, err := countTotalEmbeddable(conn) + if err != nil { + release() + return err + } + + alreadyEmbedded, err := countAlreadyEmbedded(conn) + if err != nil { + release() + return err + } + release() + + if e.taskMgr.GlobalState() != daemonpb.State_ACTIVE { + return fmt.Errorf("daemon must be fully active to run embedding indexing. Current state: %s", e.taskMgr.GlobalState().String()) + } + if _, err := e.taskMgr.AddTask(taskID, daemonpb.TaskName_EMBEDDING, taskDescription, totalEmbeddable); err != nil { + if errors.Is(err, taskmanager.ErrTaskExists) { + return fmt.Errorf("another embedding indexing task is already running") + } + return err + } + defer func() { + if _, err := e.taskMgr.DeleteTask(taskID); err != nil && !errors.Is(err, taskmanager.ErrTaskMissing) { + e.logger.Warn("failed to delete embedding task", zap.Error(err)) + } + }() + + processed := alreadyEmbedded + _, _ = e.taskMgr.UpdateProgress(taskID, totalEmbeddable, processed) + for { + conn, release, err := e.pool.Conn(ctx) + if err != nil { + return err + } + textsToEmbed, err := fetchPending(conn, e.indexPassSize) + if err != nil { + release() + return err + } + release() + if len(textsToEmbed) == 0 { + break + } + processed += int64(len(textsToEmbed)) + embeddings, err := e.embedTexts(ctx, textsToEmbed, pctOverlap) + if err != nil { + return err + } + + conn, release, err = e.pool.Conn(ctx) + if err != nil { + return err + } + if err := sqlitex.WithTx(conn, func() error { + for _, embedding := range embeddings { + if len(embedding.embeddingQuantized) != e.dimensions { + return fmt.Errorf("embedding dimension mismatch: got %d want %d", len(embedding.embeddingQuantized), e.dimensions) + } + if err := sqlitex.Exec(conn, qEmbeddingsInsert(), nil, embedding.embeddingQuantized, embedding.ftsID); err != nil { + return err + } + } + return nil + }); err != nil { + release() + return err + } + release() + + _, _ = e.taskMgr.UpdateProgress(taskID, totalEmbeddable, processed) + time.Sleep(e.SleepBetweenPasses) + } + + return nil +} + +func (e *Embedder) ensureModel(ctx context.Context) error { + e.mu.Lock() + if e.modelLoaded { + e.mu.Unlock() + return nil + } + e.mu.Unlock() + + info, err := e.backend.LoadModel(ctx, e.model, e.forceLoad, e.taskMgr) + if err != nil { + return err + } + if info.Dimensions != embeddingColumnDims { + return fmt.Errorf("embedding dimensions mismatch: got %d want %d", info.Dimensions, embeddingColumnDims) + } + if info.ContextSize <= 0 { + return fmt.Errorf("embedding context size invalid: %d", info.ContextSize) + } + if info.Checksum == "" { + return fmt.Errorf("embedding model checksum is empty") + } + checksum, err := sqlitex.GetKV(ctx, e.pool, kvEmbeddingModelChecksumKey) + if err != nil || checksum == "" || checksum != info.Checksum { + conn, release, err := e.pool.Conn(ctx) + if err != nil { + return fmt.Errorf("could not get database connection to store embedding model checksum: %w", err) + } + defer release() + var tables []string + if err := sqlitex.Exec(conn, "SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'embeddings%'", func(stmt *sqlite.Stmt) error { + tables = append(tables, stmt.ColumnText(0)) + return nil + }); err != nil { + return err + } + if err := sqlitex.WithTx(conn, func() error { + if err := sqlitex.Exec(conn, "delete from embeddings;", nil); err != nil { + return err + } + return nil + }); err != nil { + return fmt.Errorf("could not delete old embeddings: %w", err) + } + /* + + // delete from each table + for _, table := range tables { + if err := sqlitex.Exec(conn, fmt.Sprintf("DELETE FROM %s", table), nil); err != nil { + return fmt.Errorf("could not delete from table %s: %v", table, err) + } + } + */ + if err := sqlitex.SetKV(ctx, conn, kvEmbeddingModelChecksumKey, info.Checksum, true); err != nil { + return fmt.Errorf("could not store embedding model checksum: %w", err) + } + } + e.mu.Lock() + e.dimensions = info.Dimensions + e.contextSize = info.ContextSize + e.modelLoaded = true + chunkLen := int(math.Floor(float64(e.contextSize) * 0.9)) + if chunkLen < 1 { + e.maxChunkLength = e.contextSize + } else { + e.maxChunkLength = chunkLen + } + + e.mu.Unlock() + + return nil +} + +type embeddingInput struct { + ftsID int64 + text string +} + +type embeddingOutput struct { + ftsID int64 + embedding []float32 + embeddingQuantized []int8 +} + +func (e *Embedder) embedTexts(ctx context.Context, inputs []embeddingInput, pctOverlap float32) ([]embeddingOutput, error) { + chunkedInputs := []embeddingInput{} + chunkedTexts := []string{} + for _, input := range inputs { + chunks := chunkText(input.text, e.maxChunkLength, pctOverlap) + for _, chunk := range chunks { + chunkedTexts = append(chunkedTexts, chunk) + chunkedInputs = append(chunkedInputs, embeddingInput{ + ftsID: input.ftsID, + text: chunk, + }) + } + } + + response, err := e.backend.Embed(ctx, chunkedTexts) + if err != nil { + return nil, err + } + if len(response) != len(chunkedInputs) { + return nil, fmt.Errorf("embedding count mismatch: got %d want %d", len(response), len(chunkedInputs)) + } + outputs := make([]embeddingOutput, len(chunkedInputs)) + for i, embedding := range response { + if len(embedding) != e.dimensions { + return nil, fmt.Errorf("embedding dimension mismatch: got %d want %d", len(embedding), e.dimensions) + } + outputs[i] = embeddingOutput{ + ftsID: chunkedInputs[i].ftsID, + embedding: embedding, + embeddingQuantized: quantizeEmbedding(embedding), + } + } + return outputs, nil +} + +func countTotalEmbeddable(conn *sqlite.Conn) (int64, error) { + var total int64 + if err := sqlitex.Exec(conn, qEmbeddableTotalCount(), func(stmt *sqlite.Stmt) error { + total = stmt.ColumnInt64(0) + return nil + }); err != nil { + return 0, err + } + return total, nil +} + +func countAlreadyEmbedded(conn *sqlite.Conn) (int64, error) { + var count int64 + if err := sqlitex.Exec(conn, qAlreadyEmbeddedCount(), func(stmt *sqlite.Stmt) error { + count = stmt.ColumnInt64(0) + return nil + }); err != nil { + return 0, err + } + return count, nil +} + +func fetchPending(conn *sqlite.Conn, limit int) ([]embeddingInput, error) { + rows := make([]embeddingInput, 0, limit) + + if err := sqlitex.Exec(conn, qEmbeddingsPending(), func(stmt *sqlite.Stmt) error { + rows = append(rows, embeddingInput{ + ftsID: stmt.ColumnInt64(0), + text: stmt.ColumnText(1), + }) + return nil + }, limit); err != nil { + return nil, err + } + + return rows, nil +} + +func chunkText(text string, maxLen int, overlappingPct float32) []string { + if maxLen <= 0 { + return []string{text} + } + if overlappingPct < 0 { + overlappingPct = 0 + } + if overlappingPct > 1 { + overlappingPct = 1 + } + + overlap := int(math.Round(float64(overlappingPct) * float64(maxLen))) + if overlap >= maxLen { + overlap = maxLen - 1 + } + step := maxLen - overlap + if step <= 0 { + step = 1 + } + + runes := []rune(text) + if len(runes) <= maxLen { + return []string{text} + } + + chunks := make([]string, 0, (len(runes)/step)+1) + for start := 0; start < len(runes); start += step { + end := start + maxLen + if end > len(runes) { + end = len(runes) + } + chunks = append(chunks, string(runes[start:end])) + } + + return chunks +} +func quantizeEmbedding(input []float32) []int8 { + // Find max absolute value + var maxAbs float32 + for _, v := range input { + abs := v + if abs < 0 { + abs = -abs + } + if abs > maxAbs { + maxAbs = abs + } + } + + // Quantize with scaling factor + quantized := make([]int8, len(input)) + scale := float32(127.0) + if maxAbs > 0 { + scale = 127.0 / maxAbs + } + + for i, v := range input { + scaled := v * scale + scaled = float32(math.Round(float64(scaled))) + if scaled > 127 { + quantized[i] = 127 + } else if scaled < -128 { + quantized[i] = -128 + } else { + quantized[i] = int8(scaled) + } + } + return quantized +} + +// cosineSimilarityInt8 computes cosine similarity between two int8 vectors. +// Returns a value in [-1, 1], with 1 being identical direction. +func cosineSimilarityInt8(a, b []int8) float32 { + if len(a) != len(b) || len(a) == 0 { + return 0 + } + var dot, normA, normB int64 + for i := range a { + ai, bi := int64(a[i]), int64(b[i]) + dot += ai * bi + normA += ai * ai + normB += bi * bi + } + if normA == 0 || normB == 0 { + return 0 + } + return float32(float64(dot) / (math.Sqrt(float64(normA)) * math.Sqrt(float64(normB)))) +} + +var qEmbeddingsPending = dqb.Str(` + WITH pending AS ( + SELECT rowid + FROM fts + WHERE type IN ('title', 'document', 'comment') + AND length(raw_content) > 3 + EXCEPT + SELECT fts_id FROM embeddings + ) + SELECT fts.rowid, fts.raw_content + FROM fts + JOIN pending ON pending.rowid = fts.rowid + LIMIT ?; +`) + +var qEmbeddableTotalCount = dqb.Str(` + SELECT COUNT(*) FROM fts + WHERE type IN ('title', 'document', 'comment') + AND length(raw_content) > 3; +`) + +var qAlreadyEmbeddedCount = dqb.Str(` + SELECT COUNT(DISTINCT fts_id) FROM embeddings; +`) + +var qEmbeddingsInsert = dqb.Str(` + INSERT INTO embeddings (multilingual_minilm_l12_v2, fts_id) + VALUES (vec_int8(?), ?); +`) + +// qEmbeddingsSearchUnfiltered searches embeddings without IRI filtering. +// Used when iriGlob is generic (e.g., "*" or "hm://*"). +var qEmbeddingsSearchUnfiltered = dqb.Str(` +SELECT + v.fts_id, + v.distance +FROM embeddings v +JOIN fts_index fi ON fi.rowid = v.fts_id +WHERE v.multilingual_minilm_l12_v2 MATCH vec_int8(?) + AND v.distance < ? + AND k = ? + AND fi.type IN (?, ?, ?, ?) +ORDER BY v.distance +`) + +// qEmbeddingsSearchFiltered searches embeddings with IRI pre-filtering. +// Uses fts_id IN (subquery) to leverage sqlite-vec's metadata pre-filtering, +// which filters vectors BEFORE distance calculation for better performance. +// The subquery finds fts entries matching the IRI pattern via two paths: +// 1. Direct: fts_index -> structural_blobs -> resources (for documents/titles) +// 2. Indirect: fts_index -> blob_links -> structural_blobs -> resources (for comments). +var qEmbeddingsSearchFiltered = dqb.Str(` +SELECT + v.fts_id, + v.distance +FROM embeddings v +WHERE v.multilingual_minilm_l12_v2 MATCH vec_int8(?) + AND v.distance < ? + AND k = ? + AND v.fts_id IN ( + SELECT fi.rowid FROM fts_index fi + JOIN structural_blobs sb ON sb.id = fi.blob_id + JOIN resources r ON r.id = sb.resource + WHERE fi.type IN (?, ?, ?, ?) + AND r.iri GLOB ? + UNION + SELECT fi.rowid FROM fts_index fi + JOIN blob_links bl ON bl.target = fi.blob_id AND bl.type = 'ref/head' + JOIN structural_blobs sb ON sb.id = bl.source + JOIN resources r ON r.id = sb.resource + WHERE fi.type IN (?, ?, ?, ?) + AND r.iri GLOB ? + ) +ORDER BY v.distance +`) diff --git a/backend/llm/embedding_test.go b/backend/llm/embedding_test.go new file mode 100644 index 000000000..5cfbfcf27 --- /dev/null +++ b/backend/llm/embedding_test.go @@ -0,0 +1,1132 @@ +package llm + +import ( + "context" + "fmt" + "math" + "net/url" + "sync" + "testing" + "time" + + "seed/backend/daemon/taskmanager" + daemonpb "seed/backend/genproto/daemon/v1alpha" + "seed/backend/llm/backends" + "seed/backend/llm/backends/llamacpp" + "seed/backend/llm/backends/ollama" + "seed/backend/storage" + "seed/backend/testutil" + "seed/backend/util/sqlite" + "seed/backend/util/sqlite/sqlitex" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +type fakeEmbeddingBackend struct { + mu sync.Mutex + + loadCalls int + embedCalls int + retrieveSingleCalls int + + embedInputs [][]string + + contextSize int +} + +func (b *fakeEmbeddingBackend) CloseModel(ctx context.Context) error { + _ = ctx + return nil +} + +func (b *fakeEmbeddingBackend) TokenLength(ctx context.Context, input string) (int, error) { + _ = ctx + return len([]rune(input)), nil +} + +func (b *fakeEmbeddingBackend) LoadModel(ctx context.Context, model string, force bool, taskMgr *taskmanager.TaskManager) (backends.ModelInfo, error) { + _ = ctx + _ = model + _ = force + _ = taskMgr + + b.mu.Lock() + defer b.mu.Unlock() + + b.loadCalls++ + return backends.ModelInfo{Dimensions: 384, ContextSize: b.contextSize, Checksum: "fake-checksum"}, nil +} + +func (b *fakeEmbeddingBackend) RetrieveSingle(ctx context.Context, input string) ([]float32, error) { + _ = ctx + b.mu.Lock() + b.retrieveSingleCalls++ + b.mu.Unlock() + embedding := make([]float32, 384) + embedding[0] = float32(len([]rune(input))) + return embedding, nil +} + +func (b *fakeEmbeddingBackend) Embed(ctx context.Context, inputs []string) ([][]float32, error) { + _ = ctx + + b.mu.Lock() + b.embedCalls++ + b.embedInputs = append(b.embedInputs, append([]string(nil), inputs...)) + b.mu.Unlock() + + out := make([][]float32, len(inputs)) + for i := range inputs { + embedding := make([]float32, 384) + embedding[0] = float32(len([]rune(inputs[i]))) + out[i] = embedding + } + return out, nil +} + +func (b *fakeEmbeddingBackend) Version(ctx context.Context) (string, error) { + _ = ctx + return "fake", nil +} + +// Thread-safe getters for test assertions. +func (b *fakeEmbeddingBackend) getLoadCalls() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.loadCalls +} + +func (b *fakeEmbeddingBackend) getEmbedCalls() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.embedCalls +} + +func (b *fakeEmbeddingBackend) getRetrieveSingleCalls() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.retrieveSingleCalls +} + +func (b *fakeEmbeddingBackend) getEmbedInputs() [][]string { + b.mu.Lock() + defer b.mu.Unlock() + // Return a copy to avoid races after releasing the lock + result := make([][]string, len(b.embedInputs)) + for i, inputs := range b.embedInputs { + result[i] = append([]string(nil), inputs...) + } + return result +} + +func countEmbeddings(t *testing.T, conn *sqlite.Conn) int64 { + t.Helper() + + var n int64 + require.NoError(t, sqlitex.Exec(conn, "SELECT COUNT(*) FROM embeddings;", func(stmt *sqlite.Stmt) error { + n = stmt.ColumnInt64(0) + return nil + })) + return n +} + +func countEmbeddingsForFTSID(t *testing.T, conn *sqlite.Conn, ftsID int64) int64 { + t.Helper() + + var n int64 + require.NoError(t, sqlitex.Exec(conn, "SELECT COUNT(*) FROM embeddings WHERE fts_id = ?;", func(stmt *sqlite.Stmt) error { + n = stmt.ColumnInt64(0) + return nil + }, ftsID)) + return n +} + +func TestEmbedderRunOnce_IndexingBehavior(t *testing.T) { + ctx := t.Context() + + db := storage.MakeTestMemoryDB(t) + require.NoError(t, db.WithTx(ctx, func(conn *sqlite.Conn) error { + const ( + fts1 int64 = 1 + fts2 int64 = 2 + fts3 int64 = 3 + ) + + longText := "01234567890123456789" // 20 runes + alreadyEmbeddedText := "this one is already embedded" + shortText := "tiny-text" + + if err := sqlitex.Exec(conn, + `INSERT INTO fts(rowid, raw_content, type) VALUES (?, ?, ?);`, + nil, fts1, longText, "document", + ); err != nil { + return err + } + if err := sqlitex.Exec(conn, + `INSERT INTO fts(rowid, raw_content, type) VALUES (?, ?, ?);`, + nil, fts2, alreadyEmbeddedText, "document", + ); err != nil { + return err + } + if err := sqlitex.Exec(conn, + `INSERT INTO fts(rowid, raw_content, type) VALUES (?, ?, ?);`, + nil, fts3, shortText, "title", + ); err != nil { + return err + } + if err := sqlitex.SetKV(ctx, conn, kvEmbeddingModelChecksumKey, "fake-checksum", true); err != nil { + return err + } + // Mark fts2 as already embedded so it must be skipped by pending query. + return sqlitex.Exec(conn, + `INSERT INTO embeddings (multilingual_minilm_l12_v2, fts_id) VALUES (vec_int8(?), ?);`, + nil, make([]int8, 384), fts2, + ) + })) + + tm := taskmanager.NewTaskManager() + tm.UpdateGlobalState(daemonpb.State_ACTIVE) + + backend := &fakeEmbeddingBackend{contextSize: 10} // maxChunkLength=floor(10*0.9)=9 + + e, err := NewEmbedder( + db, + backend, + zap.NewNop(), + tm, + WithModel(DefaultEmbeddingModel), + WithInterval(10*time.Minute), // disable automatic runs + WithIndexPassSize(1), // force multiple passes + WithSleepPerPass(0*time.Millisecond), + ) + require.NoError(t, err) + + conn, release, err := db.Conn(ctx) + require.NoError(t, err) + beforeTotal := countEmbeddings(t, conn) + beforeFTS2 := countEmbeddingsForFTSID(t, conn, 2) + release() + + require.Equal(t, int64(1), beforeFTS2) + e.Init(t.Context()) + + //require.NoError(t, e.runOnce(ctx)) + + require.Equal(t, 1, backend.getLoadCalls()) + require.Eventually(t, func() bool { return backend.getEmbedCalls() == 2 }, + 200*time.Second, 10*time.Millisecond, "expected 2 embed call after init run") + // Wait for runOnce to fully complete (all DB writes committed). + // embedCalls==2 only means Embed() was called, but the INSERT + // transaction may still be in-flight. The task is deleted via defer + // at the end of runOnce, so 0 tasks means all writes are done. + require.Eventually(t, func() bool { return len(tm.Tasks()) == 0 }, + 10*time.Second, 10*time.Millisecond, "runOnce must complete before checking DB state") + embedInputs := backend.getEmbedInputs() + firstPassInputs := embedInputs[0] + secondPassInputs := embedInputs[1] + + expectedChunks := chunkText("01234567890123456789", 9, pctOverlap) + require.Equal(t, expectedChunks, firstPassInputs) + + expectedOverlap := int(math.Round(float64(pctOverlap) * float64(9))) + if expectedOverlap >= 9 { + expectedOverlap = 8 + } + for i := 0; i+1 < len(expectedChunks); i++ { + prev := []rune(expectedChunks[i]) + next := []rune(expectedChunks[i+1]) + if expectedOverlap == 0 { + continue + } + require.GreaterOrEqual(t, len(prev), expectedOverlap) + require.GreaterOrEqual(t, len(next), expectedOverlap) + require.Equal(t, prev[len(prev)-expectedOverlap:], next[:expectedOverlap]) + } + require.Equal(t, []string{"tiny-text"}, secondPassInputs) + + conn, release, err = db.Conn(ctx) + require.NoError(t, err) + afterTotal := countEmbeddings(t, conn) + require.Equal(t, beforeFTS2, countEmbeddingsForFTSID(t, conn, 2), "fts2 must not be duplicated") + require.Equal(t, int64(3), countEmbeddingsForFTSID(t, conn, 1), "fts1 must be chunked into 3 rows") + require.Equal(t, int64(1), countEmbeddingsForFTSID(t, conn, 3), "fts3 must produce one row") + + wantIncrease := int64(3 + 1) // chunks(fts1)=3 plus fts3=1 + require.Equal(t, beforeTotal+wantIncrease, afterTotal) + + release() + + // Second run must not embed or insert anything new. + require.NoError(t, e.runOnce(ctx)) + + require.Equal(t, 1, backend.getLoadCalls(), "model must only be loaded once") + require.Equal(t, 2, backend.getEmbedCalls(), "no new embedding calls expected") + + conn, release, err = db.Conn(ctx) + require.NoError(t, err) + require.Equal(t, afterTotal, countEmbeddings(t, conn)) + release() + + require.Len(t, tm.Tasks(), 0, "task must be deleted at the end of run") +} + +func TestEmbedderRunOnce_RequiresDaemonActive(t *testing.T) { + ctx := t.Context() + + db := storage.MakeTestMemoryDB(t) + require.NoError(t, db.WithTx(ctx, func(conn *sqlite.Conn) error { + return sqlitex.Exec(conn, + `INSERT INTO fts(rowid, raw_content, type) VALUES (?, ?, ?);`, + nil, int64(1), "hello world", "document", + ) + })) + + // Default is State_STARTING; runOnce must refuse to run. + tm := taskmanager.NewTaskManager() + backend := &fakeEmbeddingBackend{contextSize: 10} + + e, err := NewEmbedder( + db, + backend, + zap.NewNop(), + tm, + WithModel(DefaultEmbeddingModel), + WithSleepPerPass(0*time.Millisecond), + ) + require.NoError(t, err) + + err = e.runOnce(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "daemon must be fully active") +} + +func TestEmbedderInit_StartsIndexingLoop(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Use a small context size so chunking is exercised: floor(10*0.9)=9. + mockServer := testutil.NewMockOllamaServer(t, testutil.WithMockOllamaContextSize(10)) + t.Cleanup(mockServer.Server.Close) + url, err := url.Parse(mockServer.Server.URL) + require.NoError(t, err) + backend, err := ollama.NewClient(*url, ollama.WithBatchSize(1000)) + require.NoError(t, err) + + db := storage.MakeTestMemoryDB(t) + require.NoError(t, db.WithTx(ctx, func(conn *sqlite.Conn) error { + return sqlitex.Exec(conn, + `INSERT INTO fts(rowid, raw_content, type) VALUES (?, ?, ?);`, + nil, int64(1), "this is a test document", "document", + ) + })) + + tm := taskmanager.NewTaskManager() + tm.UpdateGlobalState(daemonpb.State_ACTIVE) + + e, err := NewEmbedder( + db, + backend, + zap.NewNop(), + tm, + WithModel(DefaultEmbeddingModel), + WithIndexPassSize(100), + WithSleepPerPass(0*time.Millisecond), + WithInterval(minRunInterval), + ) + require.NoError(t, err) + + e.Init(ctx) + + select { + case <-mockServer.FirstEmbedDone: + // Wait for the run to finish inserting before canceling. + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for Init() to trigger embedding") + } + + // With context size 10 -> max chunk len 9 -> 20 runes become 3 chunks. + require.Eventually(t, func() bool { + conn, release, err := db.Conn(t.Context()) + if err != nil { + return false + } + defer release() + return countEmbeddingsForFTSID(t, conn, 1) == 3 + }, 2*time.Second, 10*time.Millisecond) + + // Stop the loop quickly after the first run completes. + cancel() + + // Wait for the runOnce deferred cleanup to run. + require.Eventually(t, func() bool { + return len(tm.Tasks()) == 0 + }, 2*time.Second, 10*time.Millisecond) + + mockServer.Mu.Lock() + require.GreaterOrEqual(t, mockServer.ShowRequests, 1) + require.Equal(t, 1, mockServer.EmbedRequests) + require.Len(t, mockServer.BatchSizes, 1) + require.Equal(t, 3, mockServer.BatchSizes[0]) + mockServer.Mu.Unlock() +} + +func TestEmbedder_SemanticSearch_Manual(t *testing.T) { + // Quality checks are tight to detect any regressions on embedding model. + ctx := t.Context() + + // Use embedded GGUF model (empty URL = embedded) + backend, err := llamacpp.NewClient(url.URL{}, llamacpp.WithBatchSize(10)) + require.NoError(t, err) + t.Cleanup(func() { _ = backend.CloseModel(ctx) }) + + db := storage.MakeTestDB(t) + var allTypes = map[string]bool{"title": true, "document": true, "comment": true, "contact": true} + // Test sentences: semantically related in different languages + testSentences := []struct { + id int64 + text string + contentType string + topic string // for verification + }{ + // Technology/AI topic - English and Spanish + {1, "Machine learning is transforming how we build software", "document", "tech"}, + {2, "El aprendizaje automático está transformando cómo construimos software", "document", "tech"}, + {3, "Deep neural networks can recognize patterns in data", "document", "tech"}, + {4, "Las redes neuronales profundas pueden reconocer patrones en datos", "document", "tech"}, //nolint:misspell // "patrones" is Spanish for "patterns" + + // Food/cooking topic - English and Spanish + {5, "The best way to cook pasta is in salted boiling water", "document", "food"}, + {6, "La mejor forma de cocinar pasta es en agua hirviendo con sal", "document", "food"}, + {7, "Italian cuisine uses fresh tomatoes and olive oil", "title", "food"}, + {8, "La cocina italiana usa tomates frescos y aceite de oliva", "title", "food"}, + + // Nature/animals topic - English and Spanish + {9, "Dogs are loyal companions and love to play", "comment", "animals"}, + {10, "Los perros son compañeros leales y les encanta jugar", "comment", "animals"}, + {11, "Cats are independent animals that enjoy sleeping", "comment", "animals"}, + {12, "Los gatos son animales independientes que disfrutan dormir", "comment", "animals"}, + } + + tm := taskmanager.NewTaskManager() + tm.UpdateGlobalState(daemonpb.State_ACTIVE) + + e, err := NewEmbedder( + db, + backend, + zap.NewNop(), + tm, + WithModel(DefaultEmbeddingModel), + WithInterval(10*time.Minute), + WithSleepPerPass(0), + ) + require.NoError(t, err) + + // Insert test data and generate real embeddings + require.NoError(t, db.WithTx(ctx, func(conn *sqlite.Conn) error { + // Insert public_key for author (shared by all entries) + if err := sqlitex.Exec(conn, + `INSERT INTO public_keys(id, principal) VALUES (?, ?);`, + nil, int64(1), "test-author", + ); err != nil { + return err + } + + for _, s := range testSentences { + // Insert blob + if err := sqlitex.Exec(conn, + `INSERT INTO blobs(id, multihash, codec, size) VALUES (?, ?, ?, ?);`, + nil, s.id*100, []byte(fmt.Sprintf("hash-%d", s.id)), 0x55, len(s.text), + ); err != nil { + return err + } + // Insert resource with IRI + if err := sqlitex.Exec(conn, + `INSERT INTO resources(id, iri) VALUES (?, ?);`, + nil, s.id, fmt.Sprintf("hm://test/doc-%d", s.id), + ); err != nil { + return err + } + // Insert structural_blob linking blob to resource + if err := sqlitex.Exec(conn, + `INSERT INTO structural_blobs(id, type, resource, author) VALUES (?, ?, ?, ?);`, + nil, s.id*100, "Change", s.id, int64(1), + ); err != nil { + return err + } + // Insert FTS entry + if err := sqlitex.Exec(conn, + `INSERT INTO fts(rowid, raw_content, type, blob_id, block_id, version) VALUES (?, ?, ?, ?, ?, ?);`, + nil, s.id, s.text, s.contentType, s.id*100, fmt.Sprintf("block%d", s.id), fmt.Sprintf("v%d", s.id), + ); err != nil { + return err + } + // Insert fts_index entry + if err := sqlitex.Exec(conn, + `INSERT INTO fts_index(rowid, blob_id, block_id, version, type, ts) VALUES (?, ?, ?, ?, ?, ?);`, + nil, s.id, s.id*100, fmt.Sprintf("block%d", s.id), fmt.Sprintf("v%d", s.id), s.contentType, s.id*1000, + ); err != nil { + return err + } + } + return nil + })) + + e.Init(t.Context()) + require.Eventually(t, func() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.modelLoaded + }, 2*time.Second, 10*time.Millisecond) + + // Generate and store embeddings for all sentences + allTexts := make([]string, len(testSentences)) + for i, s := range testSentences { + allTexts[i] = s.text + } + + embeddings, err := backend.Embed(ctx, allTexts) + require.NoError(t, err) + require.Len(t, embeddings, len(testSentences)) + + // Wait for any indexing tasks to finish (The one produced by the initial indexing pass). + require.Eventually(t, func() bool { + return len(tm.Tasks()) == 0 + }, 30*time.Second, 100*time.Millisecond, "indexing tasks should complete") + + t.Run("English ML query finds tech content first", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "artificial intelligence and machine learning", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + // Top results should be about technology + t.Logf("Query: 'artificial intelligence and machine learning'") + for ftsRowid, score := range results { + t.Logf(" %d. [%.4f] %s", ftsRowid, score, "") + } + + // At least the top result should be tech-related (IDs 1-4) + topResult := results.Max() + require.GreaterOrEqual(t, topResult.RowID, int64(1), "Top result should be in the AI/Tech bucket: %d", topResult.RowID) + require.LessOrEqual(t, topResult.RowID, int64(4), "Top result should be in the AI/Tech bucket: %d", topResult.RowID) + + // Tech content (IDs 1-4) should rank higher than non-tech content (IDs 5-12) + sortedResults := results.ToList(true) + techScores := make([]float32, 0) + nonTechScores := make([]float32, 0) + for _, r := range sortedResults { + if r.RowID >= 1 && r.RowID <= 4 { + techScores = append(techScores, r.Score) + } else { + nonTechScores = append(nonTechScores, r.Score) + } + } + require.NotEmpty(t, techScores, "Should have tech results") + require.NotEmpty(t, nonTechScores, "Should have non-tech results") + require.Greater(t, techScores[0], nonTechScores[0], "Best tech result should beat best non-tech result") + }) + + t.Run("Spanish ML query finds tech content", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "inteligencia artificial y redes neuronales", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + t.Logf("Query: 'inteligencia artificial y redes neuronales'") + for ftsRowid, score := range results { + t.Logf(" %d. [%.4f] %s", ftsRowid, score, "") + } + + // At least the top result should be tech-related (IDs 1-4) + topResult := results.Max() + require.GreaterOrEqual(t, topResult.RowID, int64(1), "Top result should be in the AI/Tech bucket: %d", topResult.RowID) + require.LessOrEqual(t, topResult.RowID, int64(4), "Top result should be in the AI/Tech bucket: %d", topResult.RowID) + }) + + t.Run("Food query finds cooking content", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "how to cook Italian food with pasta", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + t.Logf("Query: 'how to cook Italian food with pasta'") + for ftsRowid, score := range results { + t.Logf(" %d. [%.4f] %s", ftsRowid, score, "") + } + + // Top result should be about food (IDs 5-8) + topResult := results.Max() + require.GreaterOrEqual(t, topResult.RowID, int64(5), "Top result should be in the food bucket: %d", topResult.RowID) + require.LessOrEqual(t, topResult.RowID, int64(8), "Top result should be in the food bucket: %d", topResult.RowID) + }) + + t.Run("Spanish food query finds cooking content", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "recetas de comida italiana con aceite", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + t.Logf("Query: 'recetas de comida italiana con aceite'") + for ftsRowid, score := range results { + t.Logf(" %d. [%.4f] %s", ftsRowid, score, "") + } + + // Top result should be about food (IDs 5-8) + topResult := results.Max() + require.GreaterOrEqual(t, topResult.RowID, int64(5), "Top result should be in the food bucket: %d", topResult.RowID) + require.LessOrEqual(t, topResult.RowID, int64(8), "Top result should be in the food bucket: %d", topResult.RowID) + }) + + t.Run("Pets query finds animal content", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "pets and domestic animals", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + t.Logf("Query: 'pets and domestic animals'") + for ftsRowid, score := range results { + t.Logf(" %d. [%.4f] %s", ftsRowid, score, "") + } + + // Top result should be about animals (IDs 9-12) + topResult := results.Max() + require.GreaterOrEqual(t, topResult.RowID, int64(9), "Top result should be in the animals bucket: %d", topResult.RowID) + require.LessOrEqual(t, topResult.RowID, int64(12), "Top result should be in the animals bucket: %d", topResult.RowID) + }) + + t.Run("Cross-language similarity works", func(t *testing.T) { + // Query in English about dogs + resultsEn, err := e.SemanticSearch(ctx, "dogs playing and having fun", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, resultsEn) + + // Query in Spanish about dogs + resultsEs, err := e.SemanticSearch(ctx, "perros jugando y divirtiéndose", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, resultsEs) + + t.Logf("English query 'dogs playing and having fun':") + for ftsRowid, score := range resultsEn { + t.Logf(" %d. [%.4f] %s", ftsRowid, score, "") + } + t.Logf("Spanish query 'perros jugando y divirtiéndose':") + for ftsRowid, score := range resultsEs { + t.Logf(" %d. [%.4f] %s", ftsRowid, score, "") + } + + // Both should return dog-related content as top result (IDs 9 or 10) + topResultEn := resultsEn.Max() + topResultEs := resultsEs.Max() + + // Dogs are in IDs 9-10, so top result should be in animals bucket (9-12) + require.GreaterOrEqual(t, topResultEn.RowID, int64(9), "English query top result should be about animals") + require.LessOrEqual(t, topResultEn.RowID, int64(12), "English query top result should be about animals") + require.GreaterOrEqual(t, topResultEs.RowID, int64(9), "Spanish query top result should be about animals") + require.LessOrEqual(t, topResultEs.RowID, int64(12), "Spanish query top result should be about animals") + + // Both should have decent scores (above 0.6) + require.Greater(t, topResultEn.Score, float32(0.6), "English query should have decent score") + require.Greater(t, topResultEs.Score, float32(0.6), "Spanish query should have decent score") + }) + + t.Run("Content type filtering works with real embeddings", func(t *testing.T) { + // Only comments (animals topic) + results, err := e.SemanticSearch(ctx, "domestic pets", 10, map[string]bool{"comment": true}, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + t.Logf("Query 'domestic pets' filtered to comments only:") + for ftsRowid, score := range results { + t.Logf(" %d. [%.4f]", ftsRowid, score) + } + + // Comments are IDs 9-12, so all results should be in that range + for rowID := range results { + require.GreaterOrEqual(t, rowID, int64(9), "Filtered result should be comment type (IDs 9-12)") + require.LessOrEqual(t, rowID, int64(12), "Filtered result should be comment type (IDs 9-12)") + } + }) + + t.Run("Scores are ordered correctly", func(t *testing.T) { + resultsMap, err := e.SemanticSearch(ctx, "software development", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, resultsMap) + + // All scores should be between 0 and 1 + maxScore := resultsMap.ToList(true)[2] + minScore := resultsMap.ToList(false)[1] + + require.GreaterOrEqual(t, maxScore.Score, float32(0.0), "Max score should be >= 0") + require.LessOrEqual(t, maxScore.Score, float32(1.0), "Max score should be <= 1") + require.GreaterOrEqual(t, minScore.Score, float32(0.0), "Min score should be >= 0") + require.LessOrEqual(t, minScore.Score, float32(1.0), "Min score should be <= 1") + require.GreaterOrEqual(t, maxScore.Score, minScore.Score, "Max score should be >= min score") + + t.Logf("Query 'software development' - max score: %.4f (rowID: %d), min score: %.4f (rowID: %d)", + maxScore.Score, maxScore.RowID, minScore.Score, minScore.RowID) + }) +} + +func TestEmbedder_SemanticSearch(t *testing.T) { + ctx := t.Context() + + db := storage.MakeTestMemoryDB(t) + allTypes := map[string]bool{"title": true, "document": true, "comment": true, "contact": true} + // Insert test data: FTS entries with corresponding embeddings + require.NoError(t, db.WithTx(ctx, func(conn *sqlite.Conn) error { + // Insert blobs (required for structural_blobs FK) + for _, blobID := range []int64{100, 101, 102} { + if err := sqlitex.Exec(conn, + `INSERT INTO blobs(id, multihash, codec, size) VALUES (?, ?, ?, ?);`, + nil, blobID, []byte(fmt.Sprintf("hash-%d", blobID)), 0x55, 0, + ); err != nil { + return err + } + } + + // Insert resources with non-null IRI + for i, resID := range []int64{1, 2, 3} { + if err := sqlitex.Exec(conn, + `INSERT INTO resources(id, iri) VALUES (?, ?);`, + nil, resID, fmt.Sprintf("hm://test/resource-%d", i+1), + ); err != nil { + return err + } + } + + // Insert public_key for author + if err := sqlitex.Exec(conn, + `INSERT INTO public_keys(id, principal) VALUES (?, ?);`, + nil, int64(1), "test-author", + ); err != nil { + return err + } + + // Insert structural_blobs linking blob_id to resources + for i, blobID := range []int64{100, 101, 102} { + if err := sqlitex.Exec(conn, + `INSERT INTO structural_blobs(id, type, resource, author) VALUES (?, ?, ?, ?);`, + nil, blobID, "Change", int64(i+1), int64(1), + ); err != nil { + return err + } + } + + // Insert FTS entries + if err := sqlitex.Exec(conn, + `INSERT INTO fts(rowid, raw_content, type, blob_id, block_id, version) VALUES (?, ?, ?, ?, ?, ?);`, + nil, int64(1), "machine learning algorithms", "document", 100, "block1", "v1", + ); err != nil { + return err + } + if err := sqlitex.Exec(conn, + `INSERT INTO fts(rowid, raw_content, type, blob_id, block_id, version) VALUES (?, ?, ?, ?, ?, ?);`, + nil, int64(2), "deep neural networks", "document", 101, "block2", "v2", + ); err != nil { + return err + } + if err := sqlitex.Exec(conn, + `INSERT INTO fts(rowid, raw_content, type, blob_id, block_id, version) VALUES (?, ?, ?, ?, ?, ?);`, + nil, int64(3), "cooking recipes for beginners", "title", 102, "block3", "v3", + ); err != nil { + return err + } + + // Insert fts_index entries (required for join) + if err := sqlitex.Exec(conn, + `INSERT INTO fts_index(rowid, blob_id, block_id, version, type, ts) VALUES (?, ?, ?, ?, ?, ?);`, + nil, int64(1), 100, "block1", "v1", "document", 1000, + ); err != nil { + return err + } + if err := sqlitex.Exec(conn, + `INSERT INTO fts_index(rowid, blob_id, block_id, version, type, ts) VALUES (?, ?, ?, ?, ?, ?);`, + nil, int64(2), 101, "block2", "v2", "document", 2000, + ); err != nil { + return err + } + if err := sqlitex.Exec(conn, + `INSERT INTO fts_index(rowid, blob_id, block_id, version, type, ts) VALUES (?, ?, ?, ?, ?, ?);`, + nil, int64(3), 102, "block3", "v3", "title", 3000, + ); err != nil { + return err + } + + // Insert embeddings - fake backend produces embedding[0] = len(input) + // "machine learning algorithms" = 28 chars + // "deep neural networks" = 20 chars + // "cooking recipes for beginners" = 29 chars + emb1 := make([]int8, 384) + emb1[0] = 28 // similar to ML query + emb2 := make([]int8, 384) + emb2[0] = 20 // similar to ML query + emb3 := make([]int8, 384) + emb3[0] = 29 // different topic + + if err := sqlitex.Exec(conn, + `INSERT INTO embeddings (multilingual_minilm_l12_v2, fts_id) VALUES (vec_int8(?), ?);`, + nil, emb1, int64(1), + ); err != nil { + return err + } + if err := sqlitex.Exec(conn, + `INSERT INTO embeddings (multilingual_minilm_l12_v2, fts_id) VALUES (vec_int8(?), ?);`, + nil, emb2, int64(2), + ); err != nil { + return err + } + if err := sqlitex.Exec(conn, + `INSERT INTO embeddings (multilingual_minilm_l12_v2, fts_id) VALUES (vec_int8(?), ?);`, + nil, emb3, int64(3), + ); err != nil { + return err + } + + return sqlitex.SetKV(ctx, conn, kvEmbeddingModelChecksumKey, "fake-checksum", true) + })) + + tm := taskmanager.NewTaskManager() + tm.UpdateGlobalState(daemonpb.State_ACTIVE) + + backend := &fakeEmbeddingBackend{contextSize: 1000} + + e, err := NewEmbedder( + db, + backend, + zap.NewNop(), + tm, + WithModel(DefaultEmbeddingModel), + ) + require.NoError(t, err) + + // Load model to enable semantic search + e.Init(ctx) + require.Eventually(t, func() bool { + e.mu.Lock() + defer e.mu.Unlock() + return e.modelLoaded + }, 2*time.Second, 10*time.Millisecond) + + t.Run("basic search returns results", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "artificial intelligence", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + // Should have called RetrieveSingle for the query + require.GreaterOrEqual(t, backend.getRetrieveSingleCalls(), 1) + }) + + t.Run("search with content type filter", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "test query", 10, map[string]bool{"document": true}, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + // Results should only include document fts rowids (1, 2 based on test data) + for rowID := range results { + require.Contains(t, []int64{1, 2}, rowID, "Filtered results should only include documents") + } + }) + + t.Run("search with title filter", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "test query", 10, map[string]bool{"title": true}, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + // Results should only include title fts rowid (3 based on test data) + for rowID := range results { + require.Equal(t, int64(3), rowID, "Filtered results should only include title") + } + }) + + t.Run("search respects limit", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "test", 1, allTypes, "*", 0.0) + require.NoError(t, err) + require.LessOrEqual(t, len(results), 1) + }) + + t.Run("results have valid scores", func(t *testing.T) { + results, err := e.SemanticSearch(ctx, "machine learning", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, results) + + // All scores should be between 0 and 1 + for _, score := range results { + require.GreaterOrEqual(t, score, float32(0.0)) + require.LessOrEqual(t, score, float32(1.0)) + } + }) + + t.Run("search fails if model not loaded", func(t *testing.T) { + uninitialized, err := NewEmbedder( + db, + backend, + zap.NewNop(), + tm, + WithModel(DefaultEmbeddingModel), + ) + require.NoError(t, err) + // Don't call Init + + _, err = uninitialized.SemanticSearch(ctx, "test", 10, allTypes, "*", 0.0) + require.Error(t, err) + require.Contains(t, err.Error(), "model not loaded") + }) + + t.Run("rejects invalid content types", func(t *testing.T) { + _, err := e.SemanticSearch(ctx, "test", 10, map[string]bool{"malicious'; DROP TABLE embeddings; --": true}, "*", 0.0) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid content type") + }) + + t.Run("rejects unknown content types", func(t *testing.T) { + _, err := e.SemanticSearch(ctx, "test", 10, map[string]bool{"unknown_type": true}, "*", 0.0) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid content type") + }) + + t.Run("threshold filters out low similarity results", func(t *testing.T) { + // Get all results without threshold + allResults, err := e.SemanticSearch(ctx, "machine learning", 10, allTypes, "*", 0.0) + require.NoError(t, err) + require.NotEmpty(t, allResults, "Should have results with no threshold") + + // Find a reasonable threshold value between min and max scores + minScore := allResults.Min().Score + maxScore := allResults.Max().Score + threshold := (minScore + maxScore) / 2 + + t.Logf("All results range: min=%.4f, max=%.4f, threshold=%.4f", minScore, maxScore, threshold) + + // Search with threshold - should only get results >= threshold + filteredResults, err := e.SemanticSearch(ctx, "machine learning", 10, allTypes, "*", threshold) + require.NoError(t, err) + + // Verify all filtered results have scores >= threshold + for rowID, score := range filteredResults { + require.GreaterOrEqual(t, score, threshold, + "Result rowID %d has score %.4f which is below threshold %.4f", + rowID, score, threshold) + } + + // Filtered results should be fewer than or equal to all results + require.LessOrEqual(t, len(filteredResults), len(allResults), + "Filtered results (%d) should be <= all results (%d)", + len(filteredResults), len(allResults)) + + // If threshold is above min, we should filter out at least one result + if threshold > minScore { + require.Less(t, len(filteredResults), len(allResults), + "With threshold %.4f > min score %.4f, should filter out some results", + threshold, minScore) + } + + t.Logf("Filtered %d results out of %d total (%.1f%% passed threshold)", + len(allResults)-len(filteredResults), + len(allResults), + float32(len(filteredResults))/float32(len(allResults))*100) + }) + + t.Run("high threshold returns only top results", func(t *testing.T) { + // Set a high threshold - should only get very similar results + highThreshold := float32(0.95) + + results, err := e.SemanticSearch(ctx, "machine learning", 10, allTypes, "*", highThreshold) + require.NoError(t, err) + + // All results must meet the threshold + for rowID, score := range results { + require.GreaterOrEqual(t, score, highThreshold, + "Result rowID %d has score %.4f which is below high threshold %.4f", + rowID, score, highThreshold) + } + + t.Logf("High threshold (%.2f) returned %d results", highThreshold, len(results)) + }) + + t.Run("threshold of 1.0 returns only perfect matches", func(t *testing.T) { + // Threshold of 1.0 should only return exact matches (if any) + results, err := e.SemanticSearch(ctx, "machine learning", 10, allTypes, "*", 1.0) + require.NoError(t, err) + + // All results must have score == 1.0 + for rowID, score := range results { + require.Equal(t, float32(1.0), score, + "Result rowID %d has score %.4f but threshold is 1.0", + rowID, score) + } + + t.Logf("Perfect match threshold (1.0) returned %d results", len(results)) + }) +} + +func TestEmbedder_SemanticRanking(t *testing.T) { + // This test verifies that the embedding model correctly ranks relevant content + // higher than irrelevant content for various query types including tricky words + // like gendered terms and rare proper nouns. + ctx := t.Context() + + backend, err := llamacpp.NewClient(url.URL{}, llamacpp.WithBatchSize(10)) + require.NoError(t, err) + t.Cleanup(func() { _ = backend.CloseModel(ctx) }) + + // Load the model before running tests. + _, err = backend.LoadModel(ctx, "embedded", false, nil) + require.NoError(t, err) + + // cosineSimilarity computes similarity between two float32 vectors. + cosineSimilarity := func(a, b []float32) float32 { + if len(a) != len(b) || len(a) == 0 { + return 0 + } + var dot, normA, normB float64 + for i := range a { + dot += float64(a[i]) * float64(b[i]) + normA += float64(a[i]) * float64(a[i]) + normB += float64(b[i]) * float64(b[i]) + } + if normA == 0 || normB == 0 { + return 0 + } + return float32(dot / (math.Sqrt(normA) * math.Sqrt(normB))) + } + + // rankingTest defines a test case for semantic ranking. + type rankingTest struct { + query string + relevant string + irrelevant string + } + + // English semantic ranking tests - includes tricky gendered terms and rare proper nouns. + englishTests := []rankingTest{ + // Gendered terms - these were broken in the old model. + {"male", "Male and female differences in biology", "Software development practices"}, + {"female", "Female athletes compete in sports", "Bitcoin cryptocurrency trading"}, + {"sex", "Sexual reproduction in mammals", "Cloud computing infrastructure"}, + {"gender", "Gender studies and social research", "Machine learning algorithms"}, + {"man", "The man walked to the store", "Quantum physics theories"}, + {"woman", "The woman won the competition", "Database optimization techniques"}, + // Rare proper nouns. + {"engelbart", "Douglas Engelbart invented the computer mouse", "Italian pizza recipes"}, + {"dijkstra", "Dijkstra's algorithm finds shortest paths", "French cooking techniques"}, + {"turing", "Alan Turing was a brilliant mathematician", "Spanish guitar music"}, + // Common words (baseline). + {"bitcoin", "Bitcoin is a decentralized digital currency", "Dogs are loyal companions"}, + {"music", "Classical music and jazz compositions", "Software development practices"}, + {"technology", "Technology is advancing rapidly in AI", "Italian cooking recipes"}, + {"dogs", "Dogs are loyal and friendly pets", "Quantum physics theories"}, + } + + // Spanish semantic ranking tests. + spanishTests := []rankingTest{ + // Gendered terms in Spanish. + {"masculino", "Diferencias entre masculino y femenino en biología", "Desarrollo de software"}, + {"femenino", "Atletas femeninas compiten en deportes", "Comercio de criptomonedas"}, + {"sexo", "Reproducción sexual en mamíferos", "Infraestructura de computación"}, + {"género", "Estudios de género e investigación social", "Algoritmos de aprendizaje"}, + {"hombre", "El hombre caminó a la tienda", "Teorías de física cuántica"}, + {"mujer", "La mujer ganó la competencia", "Técnicas de optimización"}, + // Common words in Spanish. + {"bitcoin", "Bitcoin es una moneda digital descentralizada", "Los perros son compañeros leales"}, + {"música", "Música clásica y composiciones de jazz", "Prácticas de desarrollo de software"}, + {"tecnología", "La tecnología avanza rápidamente en IA", "Recetas de cocina italiana"}, + {"perros", "Los perros son mascotas leales y amigables", "Teorías de física cuántica"}, + } + + // Cross-language tests: English query -> Spanish content. + crossLangEnEsTests := []rankingTest{ + {"male", "Diferencias entre masculino y femenino", "Recetas de cocina italiana"}, + {"female", "Atletas femeninas en competición", "Bitcoin y criptomonedas"}, + {"music", "La música clásica es hermosa", "Desarrollo de software moderno"}, + {"technology", "La tecnología está avanzando rápidamente", "Recetas de pasta italiana"}, + {"dogs", "Los perros son compañeros leales", "Algoritmos de inteligencia artificial"}, + {"bitcoin", "Bitcoin es una moneda digital", "La música clásica es relajante"}, + } + + // Cross-language tests: Spanish query -> English content. + crossLangEsEnTests := []rankingTest{ + {"masculino", "Male and female biological differences", "Italian cooking recipes"}, + {"femenino", "Female athletes in competition", "Bitcoin cryptocurrency"}, + {"música", "Classical music is beautiful", "Software development practices"}, + {"tecnología", "Technology is advancing rapidly", "Italian pasta recipes"}, + {"perros", "Dogs are loyal companions", "Artificial intelligence algorithms"}, + } + + runRankingTests := func(t *testing.T, tests []rankingTest) { + t.Helper() + for _, tc := range tests { + tc := tc + t.Run(tc.query, func(t *testing.T) { + queryEmb, err := backend.RetrieveSingle(ctx, tc.query) + require.NoError(t, err, "failed to embed query") + + relEmb, err := backend.RetrieveSingle(ctx, tc.relevant) + require.NoError(t, err, "failed to embed relevant content") + + irrEmb, err := backend.RetrieveSingle(ctx, tc.irrelevant) + require.NoError(t, err, "failed to embed irrelevant content") + + relSim := cosineSimilarity(queryEmb, relEmb) + irrSim := cosineSimilarity(queryEmb, irrEmb) + + t.Logf("query=%q relevant=%.4f irrelevant=%.4f", tc.query, relSim, irrSim) + require.Greater(t, relSim, irrSim, + "relevant content must rank higher than irrelevant: rel=%.4f, irr=%.4f", + relSim, irrSim) + }) + } + } + + t.Run("English semantic ranking", func(t *testing.T) { + runRankingTests(t, englishTests) + }) + + t.Run("Spanish semantic ranking", func(t *testing.T) { + runRankingTests(t, spanishTests) + }) + + t.Run("Cross-language EN->ES ranking", func(t *testing.T) { + runRankingTests(t, crossLangEnEsTests) + }) + + t.Run("Cross-language ES->EN ranking", func(t *testing.T) { + runRankingTests(t, crossLangEsEnTests) + }) +} + +func TestCosineSimilarityInt8(t *testing.T) { + t.Run("identical vectors have similarity 1", func(t *testing.T) { + v := []int8{10, 20, 30, 40, 50} + sim := cosineSimilarityInt8(v, v) + require.InDelta(t, 1.0, sim, 0.0001) + }) + + t.Run("opposite vectors have similarity -1", func(t *testing.T) { + v1 := []int8{10, 20, 30} + v2 := []int8{-10, -20, -30} + sim := cosineSimilarityInt8(v1, v2) + require.InDelta(t, -1.0, sim, 0.0001) + }) + + t.Run("orthogonal vectors have similarity 0", func(t *testing.T) { + v1 := []int8{10, 0, 0} + v2 := []int8{0, 10, 0} + sim := cosineSimilarityInt8(v1, v2) + require.InDelta(t, 0.0, sim, 0.0001) + }) + + t.Run("different length vectors return 0", func(t *testing.T) { + v1 := []int8{10, 20, 30} + v2 := []int8{10, 20} + sim := cosineSimilarityInt8(v1, v2) + require.Equal(t, float32(0), sim) + }) + + t.Run("empty vectors return 0", func(t *testing.T) { + sim := cosineSimilarityInt8([]int8{}, []int8{}) + require.Equal(t, float32(0), sim) + }) + + t.Run("zero vector returns 0", func(t *testing.T) { + v1 := []int8{0, 0, 0} + v2 := []int8{10, 20, 30} + sim := cosineSimilarityInt8(v1, v2) + require.Equal(t, float32(0), sim) + }) +} diff --git a/backend/storage/dbext/dbext.h b/backend/storage/dbext/dbext.h index 564084817..d9042ddbd 100644 --- a/backend/storage/dbext/dbext.h +++ b/backend/storage/dbext/dbext.h @@ -4,6 +4,7 @@ #include "./mycount/mycount.c" #include "./roaring/roaring.c" #include "./roaring/roaring_ext.c" +#include "./sqlite-vec/sqlite-vec.c" #include "./sha1/sha1.c" static void load_extensions() @@ -13,4 +14,5 @@ static void load_extensions() sqlite3_auto_extension((void (*)(void))sqlite3_carray_init); sqlite3_auto_extension((void (*)(void))sqlite3_roaring_init); sqlite3_auto_extension((void (*)(void))sqlite3_base58btc_init); + sqlite3_auto_extension((void (*)(void))sqlite3_vec_init); } diff --git a/backend/storage/dbext/sqlite-vec/sqlite-vec.c b/backend/storage/dbext/sqlite-vec/sqlite-vec.c new file mode 100644 index 000000000..058e968e2 --- /dev/null +++ b/backend/storage/dbext/sqlite-vec/sqlite-vec.c @@ -0,0 +1,9748 @@ +#include "sqlite-vec.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef SQLITE_VEC_OMIT_FS +#include +#endif + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#else +#include "sqlite3.h" +#endif + +#ifndef UINT32_TYPE +#ifdef HAVE_UINT32_T +#define UINT32_TYPE uint32_t +#else +#define UINT32_TYPE unsigned int +#endif +#endif +#ifndef UINT16_TYPE +#ifdef HAVE_UINT16_T +#define UINT16_TYPE uint16_t +#else +#define UINT16_TYPE unsigned short int +#endif +#endif +#ifndef INT16_TYPE +#ifdef HAVE_INT16_T +#define INT16_TYPE int16_t +#else +#define INT16_TYPE short int +#endif +#endif +#ifndef UINT8_TYPE +#ifdef HAVE_UINT8_T +#define UINT8_TYPE uint8_t +#else +#define UINT8_TYPE unsigned char +#endif +#endif +#ifndef INT8_TYPE +#ifdef HAVE_INT8_T +#define INT8_TYPE int8_t +#else +#define INT8_TYPE signed char +#endif +#endif +#ifndef LONGDOUBLE_TYPE +#define LONGDOUBLE_TYPE long double +#endif + +// u_int*_t types are BSD-isms available on glibc but not on musl (Alpine). +// Since uint*_t are already provided by (included via sqlite3.h), +// these re-typedefs are only needed on glibc where u_int*_t exists. +#if !defined(_WIN32) && !defined(__EMSCRIPTEN__) && !defined(__COSMOPOLITAN__) && !defined(__wasi__) && defined(__GLIBC__) +typedef u_int8_t uint8_t; +typedef u_int16_t uint16_t; +typedef u_int64_t uint64_t; +#endif + +typedef int8_t i8; +typedef uint8_t u8; +typedef int16_t i16; +typedef int32_t i32; +typedef sqlite3_int64 i64; +typedef uint32_t u32; +typedef uint64_t u64; +typedef float f32; +typedef size_t usize; + +#ifndef UNUSED_PARAMETER +#define UNUSED_PARAMETER(X) (void)(X) +#endif + +// sqlite3_vtab_in() was added in SQLite version 3.38 (2022-02-22) +// https://www.sqlite.org/changes.html#version_3_38_0 +#if SQLITE_VERSION_NUMBER >= 3038000 +#define COMPILER_SUPPORTS_VTAB_IN 1 +#endif + +#ifndef SQLITE_SUBTYPE +#define SQLITE_SUBTYPE 0x000100000 +#endif + +#ifndef SQLITE_RESULT_SUBTYPE +#define SQLITE_RESULT_SUBTYPE 0x001000000 +#endif + +#ifndef SQLITE_INDEX_CONSTRAINT_LIMIT +#define SQLITE_INDEX_CONSTRAINT_LIMIT 73 +#endif + +#ifndef SQLITE_INDEX_CONSTRAINT_OFFSET +#define SQLITE_INDEX_CONSTRAINT_OFFSET 74 +#endif + +#define countof(x) (sizeof(x) / sizeof((x)[0])) +#define min(a, b) (((a) <= (b)) ? (a) : (b)) + +enum VectorElementType { + // clang-format off + SQLITE_VEC_ELEMENT_TYPE_FLOAT32 = 223 + 0, + SQLITE_VEC_ELEMENT_TYPE_BIT = 223 + 1, + SQLITE_VEC_ELEMENT_TYPE_INT8 = 223 + 2, + // clang-format on +}; + +#ifdef SQLITE_VEC_ENABLE_AVX +#include +#define PORTABLE_ALIGN32 __attribute__((aligned(32))) +#define PORTABLE_ALIGN64 __attribute__((aligned(64))) + +static f32 l2_sqr_float_avx(const void *pVect1v, const void *pVect2v, + const void *qty_ptr) { + f32 *pVect1 = (f32 *)pVect1v; + f32 *pVect2 = (f32 *)pVect2v; + size_t qty = *((size_t *)qty_ptr); + f32 PORTABLE_ALIGN32 TmpRes[8]; + size_t qty16 = qty >> 4; + + const f32 *pEnd1 = pVect1 + (qty16 << 4); + + __m256 diff, v1, v2; + __m256 sum = _mm256_set1_ps(0); + + while (pVect1 < pEnd1) { + v1 = _mm256_loadu_ps(pVect1); + pVect1 += 8; + v2 = _mm256_loadu_ps(pVect2); + pVect2 += 8; + diff = _mm256_sub_ps(v1, v2); + sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff)); + + v1 = _mm256_loadu_ps(pVect1); + pVect1 += 8; + v2 = _mm256_loadu_ps(pVect2); + pVect2 += 8; + diff = _mm256_sub_ps(v1, v2); + sum = _mm256_add_ps(sum, _mm256_mul_ps(diff, diff)); + } + + _mm256_store_ps(TmpRes, sum); + return sqrt(TmpRes[0] + TmpRes[1] + TmpRes[2] + TmpRes[3] + TmpRes[4] + + TmpRes[5] + TmpRes[6] + TmpRes[7]); +} +#endif + +#ifdef SQLITE_VEC_ENABLE_NEON +#include + +#define PORTABLE_ALIGN32 __attribute__((aligned(32))) + +// thx https://github.com/nmslib/hnswlib/pull/299/files +static f32 l2_sqr_float_neon(const void *pVect1v, const void *pVect2v, + const void *qty_ptr) { + f32 *pVect1 = (f32 *)pVect1v; + f32 *pVect2 = (f32 *)pVect2v; + size_t qty = *((size_t *)qty_ptr); + size_t qty16 = qty >> 4; + + const f32 *pEnd1 = pVect1 + (qty16 << 4); + + float32x4_t diff, v1, v2; + float32x4_t sum0 = vdupq_n_f32(0); + float32x4_t sum1 = vdupq_n_f32(0); + float32x4_t sum2 = vdupq_n_f32(0); + float32x4_t sum3 = vdupq_n_f32(0); + + while (pVect1 < pEnd1) { + v1 = vld1q_f32(pVect1); + pVect1 += 4; + v2 = vld1q_f32(pVect2); + pVect2 += 4; + diff = vsubq_f32(v1, v2); + sum0 = vfmaq_f32(sum0, diff, diff); + + v1 = vld1q_f32(pVect1); + pVect1 += 4; + v2 = vld1q_f32(pVect2); + pVect2 += 4; + diff = vsubq_f32(v1, v2); + sum1 = vfmaq_f32(sum1, diff, diff); + + v1 = vld1q_f32(pVect1); + pVect1 += 4; + v2 = vld1q_f32(pVect2); + pVect2 += 4; + diff = vsubq_f32(v1, v2); + sum2 = vfmaq_f32(sum2, diff, diff); + + v1 = vld1q_f32(pVect1); + pVect1 += 4; + v2 = vld1q_f32(pVect2); + pVect2 += 4; + diff = vsubq_f32(v1, v2); + sum3 = vfmaq_f32(sum3, diff, diff); + } + + f32 sum_scalar = + vaddvq_f32(vaddq_f32(vaddq_f32(sum0, sum1), vaddq_f32(sum2, sum3))); + const f32 *pEnd2 = pVect1 + (qty - (qty16 << 4)); + while (pVect1 < pEnd2) { + f32 diff = *pVect1 - *pVect2; + sum_scalar += diff * diff; + pVect1++; + pVect2++; + } + + return sqrt(sum_scalar); +} + +static f32 l2_sqr_int8_neon(const void *pVect1v, const void *pVect2v, + const void *qty_ptr) { + i8 *pVect1 = (i8 *)pVect1v; + i8 *pVect2 = (i8 *)pVect2v; + size_t qty = *((size_t *)qty_ptr); + + const i8 *pEnd1 = pVect1 + qty; + i32 sum_scalar = 0; + + while (pVect1 < pEnd1 - 7) { + // loading 8 at a time + int8x8_t v1 = vld1_s8(pVect1); + int8x8_t v2 = vld1_s8(pVect2); + pVect1 += 8; + pVect2 += 8; + + // widen to protect against overflow + int16x8_t v1_wide = vmovl_s8(v1); + int16x8_t v2_wide = vmovl_s8(v2); + + int16x8_t diff = vsubq_s16(v1_wide, v2_wide); + int16x8_t squared_diff = vmulq_s16(diff, diff); + int32x4_t sum = vpaddlq_s16(squared_diff); + + sum_scalar += vgetq_lane_s32(sum, 0) + vgetq_lane_s32(sum, 1) + + vgetq_lane_s32(sum, 2) + vgetq_lane_s32(sum, 3); + } + + // handle leftovers + while (pVect1 < pEnd1) { + i16 diff = (i16)*pVect1 - (i16)*pVect2; + sum_scalar += diff * diff; + pVect1++; + pVect2++; + } + + return sqrtf(sum_scalar); +} + +static i32 l1_int8_neon(const void *pVect1v, const void *pVect2v, + const void *qty_ptr) { + i8 *pVect1 = (i8 *)pVect1v; + i8 *pVect2 = (i8 *)pVect2v; + size_t qty = *((size_t *)qty_ptr); + + const int8_t *pEnd1 = pVect1 + qty; + + int32x4_t acc1 = vdupq_n_s32(0); + int32x4_t acc2 = vdupq_n_s32(0); + int32x4_t acc3 = vdupq_n_s32(0); + int32x4_t acc4 = vdupq_n_s32(0); + + while (pVect1 < pEnd1 - 63) { + int8x16_t v1 = vld1q_s8(pVect1); + int8x16_t v2 = vld1q_s8(pVect2); + int8x16_t diff1 = vabdq_s8(v1, v2); + acc1 = vaddq_s32(acc1, vpaddlq_u16(vpaddlq_u8(diff1))); + + v1 = vld1q_s8(pVect1 + 16); + v2 = vld1q_s8(pVect2 + 16); + int8x16_t diff2 = vabdq_s8(v1, v2); + acc2 = vaddq_s32(acc2, vpaddlq_u16(vpaddlq_u8(diff2))); + + v1 = vld1q_s8(pVect1 + 32); + v2 = vld1q_s8(pVect2 + 32); + int8x16_t diff3 = vabdq_s8(v1, v2); + acc3 = vaddq_s32(acc3, vpaddlq_u16(vpaddlq_u8(diff3))); + + v1 = vld1q_s8(pVect1 + 48); + v2 = vld1q_s8(pVect2 + 48); + int8x16_t diff4 = vabdq_s8(v1, v2); + acc4 = vaddq_s32(acc4, vpaddlq_u16(vpaddlq_u8(diff4))); + + pVect1 += 64; + pVect2 += 64; + } + + while (pVect1 < pEnd1 - 15) { + int8x16_t v1 = vld1q_s8(pVect1); + int8x16_t v2 = vld1q_s8(pVect2); + int8x16_t diff = vabdq_s8(v1, v2); + acc1 = vaddq_s32(acc1, vpaddlq_u16(vpaddlq_u8(diff))); + pVect1 += 16; + pVect2 += 16; + } + + int32x4_t acc = vaddq_s32(vaddq_s32(acc1, acc2), vaddq_s32(acc3, acc4)); + + int32_t sum = 0; + while (pVect1 < pEnd1) { + int32_t diff = abs((int32_t)*pVect1 - (int32_t)*pVect2); + sum += diff; + pVect1++; + pVect2++; + } + + return vaddvq_s32(acc) + sum; +} + +static double l1_f32_neon(const void *pVect1v, const void *pVect2v, + const void *qty_ptr) { + f32 *pVect1 = (f32 *)pVect1v; + f32 *pVect2 = (f32 *)pVect2v; + size_t qty = *((size_t *)qty_ptr); + + const f32 *pEnd1 = pVect1 + qty; + float64x2_t acc = vdupq_n_f64(0); + + while (pVect1 < pEnd1 - 3) { + float32x4_t v1 = vld1q_f32(pVect1); + float32x4_t v2 = vld1q_f32(pVect2); + pVect1 += 4; + pVect2 += 4; + + // f32x4 -> f64x2 pad for overflow + float64x2_t low_diff = vabdq_f64(vcvt_f64_f32(vget_low_f32(v1)), + vcvt_f64_f32(vget_low_f32(v2))); + float64x2_t high_diff = + vabdq_f64(vcvt_high_f64_f32(v1), vcvt_high_f64_f32(v2)); + + acc = vaddq_f64(acc, vaddq_f64(low_diff, high_diff)); + } + + double sum = 0; + while (pVect1 < pEnd1) { + sum += fabs((double)*pVect1 - (double)*pVect2); + pVect1++; + pVect2++; + } + + return vaddvq_f64(acc) + sum; +} +#endif + +static f32 l2_sqr_float(const void *pVect1v, const void *pVect2v, + const void *qty_ptr) { + f32 *pVect1 = (f32 *)pVect1v; + f32 *pVect2 = (f32 *)pVect2v; + size_t qty = *((size_t *)qty_ptr); + + f32 res = 0; + for (size_t i = 0; i < qty; i++) { + f32 t = *pVect1 - *pVect2; + pVect1++; + pVect2++; + res += t * t; + } + return sqrt(res); +} + +static f32 l2_sqr_int8(const void *pA, const void *pB, const void *pD) { + i8 *a = (i8 *)pA; + i8 *b = (i8 *)pB; + size_t d = *((size_t *)pD); + + f32 res = 0; + for (size_t i = 0; i < d; i++) { + f32 t = *a - *b; + a++; + b++; + res += t * t; + } + return sqrt(res); +} + +static f32 distance_l2_sqr_float(const void *a, const void *b, const void *d) { +#ifdef SQLITE_VEC_ENABLE_NEON + if ((*(const size_t *)d) > 16) { + return l2_sqr_float_neon(a, b, d); + } +#endif +#ifdef SQLITE_VEC_ENABLE_AVX + if (((*(const size_t *)d) % 16 == 0)) { + return l2_sqr_float_avx(a, b, d); + } +#endif + return l2_sqr_float(a, b, d); +} + +static f32 distance_l2_sqr_int8(const void *a, const void *b, const void *d) { +#ifdef SQLITE_VEC_ENABLE_NEON + if ((*(const size_t *)d) > 7) { + return l2_sqr_int8_neon(a, b, d); + } +#endif + return l2_sqr_int8(a, b, d); +} + +static i32 l1_int8(const void *pA, const void *pB, const void *pD) { + i8 *a = (i8 *)pA; + i8 *b = (i8 *)pB; + size_t d = *((size_t *)pD); + + i32 res = 0; + for (size_t i = 0; i < d; i++) { + res += abs(*a - *b); + a++; + b++; + } + + return res; +} + +static i32 distance_l1_int8(const void *a, const void *b, const void *d) { +#ifdef SQLITE_VEC_ENABLE_NEON + if ((*(const size_t *)d) > 15) { + return l1_int8_neon(a, b, d); + } +#endif + return l1_int8(a, b, d); +} + +static double l1_f32(const void *pA, const void *pB, const void *pD) { + f32 *a = (f32 *)pA; + f32 *b = (f32 *)pB; + size_t d = *((size_t *)pD); + + double res = 0; + for (size_t i = 0; i < d; i++) { + res += fabs((double)*a - (double)*b); + a++; + b++; + } + + return res; +} + +static double distance_l1_f32(const void *a, const void *b, const void *d) { +#ifdef SQLITE_VEC_ENABLE_NEON + if ((*(const size_t *)d) > 3) { + return l1_f32_neon(a, b, d); + } +#endif + return l1_f32(a, b, d); +} + +static f32 distance_cosine_float(const void *pVect1v, const void *pVect2v, + const void *qty_ptr) { + f32 *pVect1 = (f32 *)pVect1v; + f32 *pVect2 = (f32 *)pVect2v; + size_t qty = *((size_t *)qty_ptr); + + f32 dot = 0; + f32 aMag = 0; + f32 bMag = 0; + for (size_t i = 0; i < qty; i++) { + dot += *pVect1 * *pVect2; + aMag += *pVect1 * *pVect1; + bMag += *pVect2 * *pVect2; + pVect1++; + pVect2++; + } + return 1 - (dot / (sqrt(aMag) * sqrt(bMag))); +} +static f32 distance_cosine_int8(const void *pA, const void *pB, + const void *pD) { + i8 *a = (i8 *)pA; + i8 *b = (i8 *)pB; + size_t d = *((size_t *)pD); + + f32 dot = 0; + f32 aMag = 0; + f32 bMag = 0; + for (size_t i = 0; i < d; i++) { + dot += *a * *b; + aMag += *a * *a; + bMag += *b * *b; + a++; + b++; + } + return 1 - (dot / (sqrt(aMag) * sqrt(bMag))); +} + +// https://github.com/facebookresearch/faiss/blob/77e2e79cd0a680adc343b9840dd865da724c579e/faiss/utils/hamming_distance/common.h#L34 +static u8 hamdist_table[256] = { + 0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4, + 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, + 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, + 4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, + 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8}; + +static f32 distance_hamming_u8(u8 *a, u8 *b, size_t n) { + int same = 0; + for (unsigned long i = 0; i < n; i++) { + same += hamdist_table[a[i] ^ b[i]]; + } + return (f32)same; +} + +#ifdef _MSC_VER +#if !defined(__clang__) && (defined(_M_ARM) || defined(_M_ARM64)) +// From +// https://github.com/ngtcp2/ngtcp2/blob/b64f1e77b5e0d880b93d31f474147fae4a1d17cc/lib/ngtcp2_ringbuf.c, +// line 34-43 +static unsigned int __builtin_popcountl(unsigned int x) { + unsigned int c = 0; + for (; x; ++c) { + x &= x - 1; + } + return c; +} +#else +#include +#define __builtin_popcountl __popcnt64 +#endif +#endif + +static f32 distance_hamming_u64(u64 *a, u64 *b, size_t n) { + int same = 0; + for (unsigned long i = 0; i < n; i++) { + same += __builtin_popcountl(a[i] ^ b[i]); + } + return (f32)same; +} + +/** + * @brief Calculate the hamming distance between two bitvectors. + * + * @param a - first bitvector, MUST have d dimensions + * @param b - second bitvector, MUST have d dimensions + * @param d - pointer to size_t, MUST be divisible by CHAR_BIT + * @return f32 + */ +static f32 distance_hamming(const void *a, const void *b, const void *d) { + size_t dimensions = *((size_t *)d); + + if ((dimensions % 64) == 0) { + return distance_hamming_u64((u64 *)a, (u64 *)b, dimensions / 8 / CHAR_BIT); + } + return distance_hamming_u8((u8 *)a, (u8 *)b, dimensions / CHAR_BIT); +} + +// from SQLite source: +// https://github.com/sqlite/sqlite/blob/a509a90958ddb234d1785ed7801880ccb18b497e/src/json.c#L153 +static const char vecJsonIsSpaceX[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; + +#define vecJsonIsspace(x) (vecJsonIsSpaceX[(unsigned char)x]) + +typedef void (*vector_cleanup)(void *p); + +void vector_cleanup_noop(void *_) { UNUSED_PARAMETER(_); } + +#define JSON_SUBTYPE 74 + +void vtab_set_error(sqlite3_vtab *pVTab, const char *zFormat, ...) { + va_list args; + sqlite3_free(pVTab->zErrMsg); + va_start(args, zFormat); + pVTab->zErrMsg = sqlite3_vmprintf(zFormat, args); + va_end(args); +} +struct Array { + size_t element_size; + size_t length; + size_t capacity; + void *z; +}; + +/** + * @brief Initial an array with the given element size and capacity. + * + * @param array + * @param element_size + * @param init_capacity + * @return SQLITE_OK on success, error code on failure. Only error is + * SQLITE_NOMEM + */ +int array_init(struct Array *array, size_t element_size, size_t init_capacity) { + int sz = element_size * init_capacity; + void *z = sqlite3_malloc(sz); + if (!z) { + return SQLITE_NOMEM; + } + memset(z, 0, sz); + + array->element_size = element_size; + array->length = 0; + array->capacity = init_capacity; + array->z = z; + return SQLITE_OK; +} + +int array_append(struct Array *array, const void *element) { + if (array->length == array->capacity) { + size_t new_capacity = array->capacity * 2 + 100; + void *z = sqlite3_realloc64(array->z, array->element_size * new_capacity); + if (z) { + array->capacity = new_capacity; + array->z = z; + } else { + return SQLITE_NOMEM; + } + } + memcpy(&((unsigned char *)array->z)[array->length * array->element_size], + element, array->element_size); + array->length++; + return SQLITE_OK; +} + +void array_cleanup(struct Array *array) { + if (!array) + return; + array->element_size = 0; + array->length = 0; + array->capacity = 0; + sqlite3_free(array->z); + array->z = NULL; +} + +char *vector_subtype_name(int subtype) { + switch (subtype) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: + return "float32"; + case SQLITE_VEC_ELEMENT_TYPE_INT8: + return "int8"; + case SQLITE_VEC_ELEMENT_TYPE_BIT: + return "bit"; + } + return ""; +} +char *type_name(int type) { + switch (type) { + case SQLITE_INTEGER: + return "INTEGER"; + case SQLITE_BLOB: + return "BLOB"; + case SQLITE_TEXT: + return "TEXT"; + case SQLITE_FLOAT: + return "FLOAT"; + case SQLITE_NULL: + return "NULL"; + } + return ""; +} + +typedef void (*fvec_cleanup)(f32 *vector); + +void fvec_cleanup_noop(f32 *_) { UNUSED_PARAMETER(_); } + +static int fvec_from_value(sqlite3_value *value, f32 **vector, + size_t *dimensions, fvec_cleanup *cleanup, + char **pzErr) { + int value_type = sqlite3_value_type(value); + + if (value_type == SQLITE_BLOB) { + const void *blob = sqlite3_value_blob(value); + int bytes = sqlite3_value_bytes(value); + if (bytes == 0) { + *pzErr = sqlite3_mprintf("zero-length vectors are not supported."); + return SQLITE_ERROR; + } + if ((bytes % sizeof(f32)) != 0) { + *pzErr = sqlite3_mprintf("invalid float32 vector BLOB length. Must be " + "divisible by %d, found %d", + sizeof(f32), bytes); + return SQLITE_ERROR; + } + *vector = (f32 *)blob; + *dimensions = bytes / sizeof(f32); + *cleanup = fvec_cleanup_noop; + return SQLITE_OK; + } + + if (value_type == SQLITE_TEXT) { + const char *source = (const char *)sqlite3_value_text(value); + int source_len = sqlite3_value_bytes(value); + if (source_len == 0) { + *pzErr = sqlite3_mprintf("zero-length vectors are not supported."); + return SQLITE_ERROR; + } + int i = 0; + + struct Array x; + int rc = array_init(&x, sizeof(f32), ceil(source_len / 2.0)); + if (rc != SQLITE_OK) { + return rc; + } + + // advance leading whitespace to first '[' + while (i < source_len) { + if (vecJsonIsspace(source[i])) { + i++; + continue; + } + if (source[i] == '[') { + break; + } + + *pzErr = sqlite3_mprintf( + "JSON array parsing error: Input does not start with '['"); + array_cleanup(&x); + return SQLITE_ERROR; + } + if (source[i] != '[') { + *pzErr = sqlite3_mprintf( + "JSON array parsing error: Input does not start with '['"); + array_cleanup(&x); + return SQLITE_ERROR; + } + int offset = i + 1; + + while (offset < source_len) { + char *ptr = (char *)&source[offset]; + char *endptr; + + errno = 0; + double result = strtod(ptr, &endptr); + if ((errno != 0 && result == 0) // some interval error? + || (errno == ERANGE && + (result == HUGE_VAL || result == -HUGE_VAL)) // too big / smalls + ) { + sqlite3_free(x.z); + *pzErr = sqlite3_mprintf("JSON parsing error"); + return SQLITE_ERROR; + } + + if (endptr == ptr) { + if (*ptr != ']') { + sqlite3_free(x.z); + *pzErr = sqlite3_mprintf("JSON parsing error"); + return SQLITE_ERROR; + } + goto done; + } + + f32 res = (f32)result; + array_append(&x, (const void *)&res); + + offset += (endptr - ptr); + while (offset < source_len) { + if (vecJsonIsspace(source[offset])) { + offset++; + continue; + } + if (source[offset] == ',') { + offset++; + continue; + } + if (source[offset] == ']') + goto done; + break; + } + } + + done: + + if (x.length > 0) { + *vector = (f32 *)x.z; + *dimensions = x.length; + *cleanup = (fvec_cleanup)sqlite3_free; + return SQLITE_OK; + } + sqlite3_free(x.z); + *pzErr = sqlite3_mprintf("zero-length vectors are not supported."); + return SQLITE_ERROR; + } + + *pzErr = sqlite3_mprintf( + "Input must have type BLOB (compact format) or TEXT (JSON), found %s", + type_name(value_type)); + return SQLITE_ERROR; +} + +static int bitvec_from_value(sqlite3_value *value, u8 **vector, + size_t *dimensions, vector_cleanup *cleanup, + char **pzErr) { + int value_type = sqlite3_value_type(value); + if (value_type == SQLITE_BLOB) { + const void *blob = sqlite3_value_blob(value); + int bytes = sqlite3_value_bytes(value); + if (bytes == 0) { + *pzErr = sqlite3_mprintf("zero-length vectors are not supported."); + return SQLITE_ERROR; + } + *vector = (u8 *)blob; + *dimensions = bytes * CHAR_BIT; + *cleanup = vector_cleanup_noop; + return SQLITE_OK; + } + *pzErr = sqlite3_mprintf("Unknown type for bitvector."); + return SQLITE_ERROR; +} + +static int int8_vec_from_value(sqlite3_value *value, i8 **vector, + size_t *dimensions, vector_cleanup *cleanup, + char **pzErr) { + int value_type = sqlite3_value_type(value); + if (value_type == SQLITE_BLOB) { + const void *blob = sqlite3_value_blob(value); + int bytes = sqlite3_value_bytes(value); + if (bytes == 0) { + *pzErr = sqlite3_mprintf("zero-length vectors are not supported."); + return SQLITE_ERROR; + } + *vector = (i8 *)blob; + *dimensions = bytes; + *cleanup = vector_cleanup_noop; + return SQLITE_OK; + } + + if (value_type == SQLITE_TEXT) { + const char *source = (const char *)sqlite3_value_text(value); + int source_len = sqlite3_value_bytes(value); + int i = 0; + + if (source_len == 0) { + *pzErr = sqlite3_mprintf("zero-length vectors are not supported."); + return SQLITE_ERROR; + } + + struct Array x; + int rc = array_init(&x, sizeof(i8), ceil(source_len / 2.0)); + if (rc != SQLITE_OK) { + return rc; + } + + // advance leading whitespace to first '[' + while (i < source_len) { + if (vecJsonIsspace(source[i])) { + i++; + continue; + } + if (source[i] == '[') { + break; + } + + *pzErr = sqlite3_mprintf( + "JSON array parsing error: Input does not start with '['"); + array_cleanup(&x); + return SQLITE_ERROR; + } + if (source[i] != '[') { + *pzErr = sqlite3_mprintf( + "JSON array parsing error: Input does not start with '['"); + array_cleanup(&x); + return SQLITE_ERROR; + } + int offset = i + 1; + + while (offset < source_len) { + char *ptr = (char *)&source[offset]; + char *endptr; + + errno = 0; + long result = strtol(ptr, &endptr, 10); + if ((errno != 0 && result == 0) || + (errno == ERANGE && (result == LONG_MAX || result == LONG_MIN))) { + sqlite3_free(x.z); + *pzErr = sqlite3_mprintf("JSON parsing error"); + return SQLITE_ERROR; + } + + if (endptr == ptr) { + if (*ptr != ']') { + sqlite3_free(x.z); + *pzErr = sqlite3_mprintf("JSON parsing error"); + return SQLITE_ERROR; + } + goto done; + } + + if (result < INT8_MIN || result > INT8_MAX) { + sqlite3_free(x.z); + *pzErr = + sqlite3_mprintf("JSON parsing error: value out of range for int8"); + return SQLITE_ERROR; + } + + i8 res = (i8)result; + array_append(&x, (const void *)&res); + + offset += (endptr - ptr); + while (offset < source_len) { + if (vecJsonIsspace(source[offset])) { + offset++; + continue; + } + if (source[offset] == ',') { + offset++; + continue; + } + if (source[offset] == ']') + goto done; + break; + } + } + + done: + + if (x.length > 0) { + *vector = (i8 *)x.z; + *dimensions = x.length; + *cleanup = (vector_cleanup)sqlite3_free; + return SQLITE_OK; + } + sqlite3_free(x.z); + *pzErr = sqlite3_mprintf("zero-length vectors are not supported."); + return SQLITE_ERROR; + } + + *pzErr = sqlite3_mprintf("Unknown type for int8 vector."); + return SQLITE_ERROR; +} + +/** + * @brief Extract a vector from a sqlite3_value. Can be a float32, int8, or bit + * vector. + * + * @param value: the sqlite3_value to read from. + * @param vector: Output pointer to vector data. + * @param dimensions: Output number of dimensions + * @param dimensions: Output vector element type + * @param cleanup + * @param pzErrorMessage + * @return int SQLITE_OK on success, error code otherwise + */ +int vector_from_value(sqlite3_value *value, void **vector, size_t *dimensions, + enum VectorElementType *element_type, + vector_cleanup *cleanup, char **pzErrorMessage) { + int subtype = sqlite3_value_subtype(value); + if (!subtype || (subtype == SQLITE_VEC_ELEMENT_TYPE_FLOAT32) || + (subtype == JSON_SUBTYPE)) { + int rc = fvec_from_value(value, (f32 **)vector, dimensions, + (fvec_cleanup *)cleanup, pzErrorMessage); + if (rc == SQLITE_OK) { + *element_type = SQLITE_VEC_ELEMENT_TYPE_FLOAT32; + } + return rc; + } + + if (subtype == SQLITE_VEC_ELEMENT_TYPE_BIT) { + int rc = bitvec_from_value(value, (u8 **)vector, dimensions, cleanup, + pzErrorMessage); + if (rc == SQLITE_OK) { + *element_type = SQLITE_VEC_ELEMENT_TYPE_BIT; + } + return rc; + } + if (subtype == SQLITE_VEC_ELEMENT_TYPE_INT8) { + int rc = int8_vec_from_value(value, (i8 **)vector, dimensions, cleanup, + pzErrorMessage); + if (rc == SQLITE_OK) { + *element_type = SQLITE_VEC_ELEMENT_TYPE_INT8; + } + return rc; + } + *pzErrorMessage = sqlite3_mprintf("Unknown subtype: %d", subtype); + return SQLITE_ERROR; +} + +int ensure_vector_match(sqlite3_value *aValue, sqlite3_value *bValue, void **a, + void **b, enum VectorElementType *element_type, + size_t *dimensions, vector_cleanup *outACleanup, + vector_cleanup *outBCleanup, char **outError) { + int rc; + enum VectorElementType aType, bType; + size_t aDims, bDims; + char *error = NULL; + vector_cleanup aCleanup, bCleanup; + + rc = vector_from_value(aValue, a, &aDims, &aType, &aCleanup, &error); + if (rc != SQLITE_OK) { + *outError = sqlite3_mprintf("Error reading 1st vector: %s", error); + sqlite3_free(error); + return SQLITE_ERROR; + } + + rc = vector_from_value(bValue, b, &bDims, &bType, &bCleanup, &error); + if (rc != SQLITE_OK) { + *outError = sqlite3_mprintf("Error reading 2nd vector: %s", error); + sqlite3_free(error); + aCleanup(a); + return SQLITE_ERROR; + } + + if (aType != bType) { + *outError = + sqlite3_mprintf("Vector type mistmatch. First vector has type %s, " + "while the second has type %s.", + vector_subtype_name(aType), vector_subtype_name(bType)); + aCleanup(*a); + bCleanup(*b); + return SQLITE_ERROR; + } + if (aDims != bDims) { + *outError = sqlite3_mprintf( + "Vector dimension mistmatch. First vector has %ld dimensions, " + "while the second has %ld dimensions.", + aDims, bDims); + aCleanup(*a); + bCleanup(*b); + return SQLITE_ERROR; + } + *element_type = aType; + *dimensions = aDims; + *outACleanup = aCleanup; + *outBCleanup = bCleanup; + return SQLITE_OK; +} + +int _cmp(const void *a, const void *b) { return (*(i64 *)a - *(i64 *)b); } + +struct VecNpyFile { + char *path; + size_t pathLength; +}; +#define SQLITE_VEC_NPY_FILE_NAME "vec0-npy-file" + +#ifndef SQLITE_VEC_OMIT_FS +static void vec_npy_file(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 1); + char *path = (char *)sqlite3_value_text(argv[0]); + size_t pathLength = sqlite3_value_bytes(argv[0]); + struct VecNpyFile *f; + + f = sqlite3_malloc(sizeof(*f)); + if (!f) { + sqlite3_result_error_nomem(context); + return; + } + memset(f, 0, sizeof(*f)); + + f->path = path; + f->pathLength = pathLength; + sqlite3_result_pointer(context, f, SQLITE_VEC_NPY_FILE_NAME, sqlite3_free); +} +#endif + +#pragma region scalar functions +static void vec_f32(sqlite3_context *context, int argc, sqlite3_value **argv) { + assert(argc == 1); + int rc; + f32 *vector = NULL; + size_t dimensions; + fvec_cleanup cleanup; + char *errmsg; + rc = fvec_from_value(argv[0], &vector, &dimensions, &cleanup, &errmsg); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, errmsg, -1); + sqlite3_free(errmsg); + return; + } + sqlite3_result_blob(context, vector, dimensions * sizeof(f32), + (void (*)(void *))cleanup); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32); +} + +static void vec_bit(sqlite3_context *context, int argc, sqlite3_value **argv) { + assert(argc == 1); + int rc; + u8 *vector; + size_t dimensions; + vector_cleanup cleanup; + char *errmsg; + rc = bitvec_from_value(argv[0], &vector, &dimensions, &cleanup, &errmsg); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, errmsg, -1); + sqlite3_free(errmsg); + return; + } + sqlite3_result_blob(context, vector, dimensions / CHAR_BIT, SQLITE_TRANSIENT); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_BIT); + cleanup(vector); +} +static void vec_int8(sqlite3_context *context, int argc, sqlite3_value **argv) { + assert(argc == 1); + int rc; + i8 *vector; + size_t dimensions; + vector_cleanup cleanup; + char *errmsg; + rc = int8_vec_from_value(argv[0], &vector, &dimensions, &cleanup, &errmsg); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, errmsg, -1); + sqlite3_free(errmsg); + return; + } + sqlite3_result_blob(context, vector, dimensions, SQLITE_TRANSIENT); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8); + cleanup(vector); +} + +static void vec_length(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 1); + int rc; + void *vector; + size_t dimensions; + vector_cleanup cleanup; + char *errmsg; + enum VectorElementType elementType; + rc = vector_from_value(argv[0], &vector, &dimensions, &elementType, &cleanup, + &errmsg); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, errmsg, -1); + sqlite3_free(errmsg); + return; + } + sqlite3_result_int64(context, dimensions); + cleanup(vector); +} + +static void vec_distance_cosine(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 2); + int rc; + void *a = NULL, *b = NULL; + size_t dimensions; + vector_cleanup aCleanup, bCleanup; + char *error; + enum VectorElementType elementType; + rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions, + &aCleanup, &bCleanup, &error); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, error, -1); + sqlite3_free(error); + return; + } + + switch (elementType) { + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + sqlite3_result_error( + context, "Cannot calculate cosine distance between two bitvectors.", + -1); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + f32 result = distance_cosine_float(a, b, &dimensions); + sqlite3_result_double(context, result); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + f32 result = distance_cosine_int8(a, b, &dimensions); + sqlite3_result_double(context, result); + goto finish; + } + } + +finish: + aCleanup(a); + bCleanup(b); + return; +} + +static void vec_distance_l2(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 2); + int rc; + void *a = NULL, *b = NULL; + size_t dimensions; + vector_cleanup aCleanup, bCleanup; + char *error; + enum VectorElementType elementType; + rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions, + &aCleanup, &bCleanup, &error); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, error, -1); + sqlite3_free(error); + return; + } + + switch (elementType) { + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + sqlite3_result_error( + context, "Cannot calculate L2 distance between two bitvectors.", -1); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + f32 result = distance_l2_sqr_float(a, b, &dimensions); + sqlite3_result_double(context, result); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + f32 result = distance_l2_sqr_int8(a, b, &dimensions); + sqlite3_result_double(context, result); + goto finish; + } + } + +finish: + aCleanup(a); + bCleanup(b); + return; +} + +static void vec_distance_l1(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 2); + int rc; + void *a, *b; + size_t dimensions; + vector_cleanup aCleanup, bCleanup; + char *error; + enum VectorElementType elementType; + rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions, + &aCleanup, &bCleanup, &error); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, error, -1); + sqlite3_free(error); + return; + } + + switch (elementType) { + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + sqlite3_result_error( + context, "Cannot calculate L1 distance between two bitvectors.", -1); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + double result = distance_l1_f32(a, b, &dimensions); + sqlite3_result_double(context, result); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + i64 result = distance_l1_int8(a, b, &dimensions); + sqlite3_result_int(context, result); + goto finish; + } + } + +finish: + aCleanup(a); + bCleanup(b); + return; +} + +static void vec_distance_hamming(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 2); + int rc; + void *a = NULL, *b = NULL; + size_t dimensions; + vector_cleanup aCleanup, bCleanup; + char *error; + enum VectorElementType elementType; + rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions, + &aCleanup, &bCleanup, &error); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, error, -1); + sqlite3_free(error); + return; + } + + switch (elementType) { + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + sqlite3_result_double(context, distance_hamming(a, b, &dimensions)); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + sqlite3_result_error( + context, + "Cannot calculate hamming distance between two float32 vectors.", -1); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + sqlite3_result_error( + context, "Cannot calculate hamming distance between two int8 vectors.", + -1); + goto finish; + } + } + +finish: + aCleanup(a); + bCleanup(b); + return; +} + +char *vec_type_name(enum VectorElementType elementType) { + switch (elementType) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: + return "float32"; + case SQLITE_VEC_ELEMENT_TYPE_INT8: + return "int8"; + case SQLITE_VEC_ELEMENT_TYPE_BIT: + return "bit"; + } + return ""; +} + +static void vec_type(sqlite3_context *context, int argc, sqlite3_value **argv) { + assert(argc == 1); + void *vector; + size_t dimensions; + vector_cleanup cleanup; + char *pzError; + enum VectorElementType elementType; + int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType, + &cleanup, &pzError); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, pzError, -1); + sqlite3_free(pzError); + return; + } + sqlite3_result_text(context, vec_type_name(elementType), -1, SQLITE_STATIC); + cleanup(vector); +} +static void vec_quantize_binary(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 1); + void *vector; + size_t dimensions; + vector_cleanup vectorCleanup; + char *pzError; + enum VectorElementType elementType; + int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType, + &vectorCleanup, &pzError); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, pzError, -1); + sqlite3_free(pzError); + return; + } + + if (dimensions <= 0) { + sqlite3_result_error(context, "Zero length vectors are not supported.", -1); + goto cleanup; + return; + } + if ((dimensions % CHAR_BIT) != 0) { + sqlite3_result_error( + context, + "Binary quantization requires vectors with a length divisible by 8", + -1); + goto cleanup; + return; + } + + int sz = dimensions / CHAR_BIT; + u8 *out = sqlite3_malloc(sz); + if (!out) { + sqlite3_result_error_code(context, SQLITE_NOMEM); + goto cleanup; + return; + } + memset(out, 0, sz); + + switch (elementType) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + + for (size_t i = 0; i < dimensions; i++) { + int res = ((f32 *)vector)[i] > 0.0; + out[i / 8] |= (res << (i % 8)); + } + break; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + for (size_t i = 0; i < dimensions; i++) { + int res = ((i8 *)vector)[i] > 0; + out[i / 8] |= (res << (i % 8)); + } + break; + } + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + sqlite3_result_error(context, + "Can only binary quantize float or int8 vectors", -1); + sqlite3_free(out); + return; + } + } + sqlite3_result_blob(context, out, sz, sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_BIT); + +cleanup: + vectorCleanup(vector); +} + +static void vec_quantize_int8(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 2); + f32 *srcVector; + size_t dimensions; + fvec_cleanup srcCleanup; + char *err; + i8 *out = NULL; + int rc = fvec_from_value(argv[0], &srcVector, &dimensions, &srcCleanup, &err); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, err, -1); + sqlite3_free(err); + return; + } + + int sz = dimensions * sizeof(i8); + out = sqlite3_malloc(sz); + if (!out) { + sqlite3_result_error_nomem(context); + goto cleanup; + } + memset(out, 0, sz); + + if ((sqlite3_value_type(argv[1]) != SQLITE_TEXT) || + (sqlite3_value_bytes(argv[1]) != strlen("unit")) || + (sqlite3_stricmp((const char *)sqlite3_value_text(argv[1]), "unit") != + 0)) { + sqlite3_result_error( + context, "2nd argument to vec_quantize_int8() must be 'unit'.", -1); + sqlite3_free(out); + goto cleanup; + } + f32 step = (1.0 - (-1.0)) / 255; + for (size_t i = 0; i < dimensions; i++) { + out[i] = ((srcVector[i] - (-1.0)) / step) - 128; + } + + sqlite3_result_blob(context, out, dimensions * sizeof(i8), sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8); + +cleanup: + srcCleanup(srcVector); +} + +static void vec_add(sqlite3_context *context, int argc, sqlite3_value **argv) { + assert(argc == 2); + int rc; + void *a = NULL, *b = NULL; + size_t dimensions; + vector_cleanup aCleanup, bCleanup; + char *error; + enum VectorElementType elementType; + rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions, + &aCleanup, &bCleanup, &error); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, error, -1); + sqlite3_free(error); + return; + } + + switch (elementType) { + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + sqlite3_result_error(context, "Cannot add two bitvectors together.", -1); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + size_t outSize = dimensions * sizeof(f32); + f32 *out = sqlite3_malloc(outSize); + if (!out) { + sqlite3_result_error_nomem(context); + goto finish; + } + memset(out, 0, outSize); + for (size_t i = 0; i < dimensions; i++) { + out[i] = ((f32 *)a)[i] + ((f32 *)b)[i]; + } + sqlite3_result_blob(context, out, outSize, sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + size_t outSize = dimensions * sizeof(i8); + i8 *out = sqlite3_malloc(outSize); + if (!out) { + sqlite3_result_error_nomem(context); + goto finish; + } + memset(out, 0, outSize); + for (size_t i = 0; i < dimensions; i++) { + out[i] = ((i8 *)a)[i] + ((i8 *)b)[i]; + } + sqlite3_result_blob(context, out, outSize, sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8); + goto finish; + } + } +finish: + aCleanup(a); + bCleanup(b); + return; +} +static void vec_sub(sqlite3_context *context, int argc, sqlite3_value **argv) { + assert(argc == 2); + int rc; + void *a = NULL, *b = NULL; + size_t dimensions; + vector_cleanup aCleanup, bCleanup; + char *error; + enum VectorElementType elementType; + rc = ensure_vector_match(argv[0], argv[1], &a, &b, &elementType, &dimensions, + &aCleanup, &bCleanup, &error); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, error, -1); + sqlite3_free(error); + return; + } + + switch (elementType) { + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + sqlite3_result_error(context, "Cannot subtract two bitvectors together.", + -1); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + size_t outSize = dimensions * sizeof(f32); + f32 *out = sqlite3_malloc(outSize); + if (!out) { + sqlite3_result_error_nomem(context); + goto finish; + } + memset(out, 0, outSize); + for (size_t i = 0; i < dimensions; i++) { + out[i] = ((f32 *)a)[i] - ((f32 *)b)[i]; + } + sqlite3_result_blob(context, out, outSize, sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32); + goto finish; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + size_t outSize = dimensions * sizeof(i8); + i8 *out = sqlite3_malloc(outSize); + if (!out) { + sqlite3_result_error_nomem(context); + goto finish; + } + memset(out, 0, outSize); + for (size_t i = 0; i < dimensions; i++) { + out[i] = ((i8 *)a)[i] - ((i8 *)b)[i]; + } + sqlite3_result_blob(context, out, outSize, sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8); + goto finish; + } + } +finish: + aCleanup(a); + bCleanup(b); + return; +} +static void vec_slice(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 3); + + void *vector; + size_t dimensions; + vector_cleanup cleanup; + char *err; + enum VectorElementType elementType; + + int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType, + &cleanup, &err); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, err, -1); + sqlite3_free(err); + return; + } + + int start = sqlite3_value_int(argv[1]); + int end = sqlite3_value_int(argv[2]); + + if (start < 0) { + sqlite3_result_error(context, + "slice 'start' index must be a postive number.", -1); + goto done; + } + if (end < 0) { + sqlite3_result_error(context, "slice 'end' index must be a postive number.", + -1); + goto done; + } + if (((size_t)start) > dimensions) { + sqlite3_result_error( + context, "slice 'start' index is greater than the number of dimensions", + -1); + goto done; + } + if (((size_t)end) > dimensions) { + sqlite3_result_error( + context, "slice 'end' index is greater than the number of dimensions", + -1); + goto done; + } + if (start > end) { + sqlite3_result_error(context, + "slice 'start' index is greater than 'end' index", -1); + goto done; + } + if (start == end) { + sqlite3_result_error(context, + "slice 'start' index is equal to the 'end' index, " + "vectors must have non-zero length", + -1); + goto done; + } + size_t n = end - start; + + switch (elementType) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + int outSize = n * sizeof(f32); + f32 *out = sqlite3_malloc(outSize); + if (!out) { + sqlite3_result_error_nomem(context); + goto done; + } + memset(out, 0, outSize); + for (size_t i = 0; i < n; i++) { + out[i] = ((f32 *)vector)[start + i]; + } + sqlite3_result_blob(context, out, outSize, sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32); + goto done; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + int outSize = n * sizeof(i8); + i8 *out = sqlite3_malloc(outSize); + if (!out) { + sqlite3_result_error_nomem(context); + return; + } + memset(out, 0, outSize); + for (size_t i = 0; i < n; i++) { + out[i] = ((i8 *)vector)[start + i]; + } + sqlite3_result_blob(context, out, outSize, sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_INT8); + goto done; + } + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + if ((start % CHAR_BIT) != 0) { + sqlite3_result_error(context, "start index must be divisible by 8.", -1); + goto done; + } + if ((end % CHAR_BIT) != 0) { + sqlite3_result_error(context, "end index must be divisible by 8.", -1); + goto done; + } + int outSize = n / CHAR_BIT; + u8 *out = sqlite3_malloc(outSize); + if (!out) { + sqlite3_result_error_nomem(context); + return; + } + memset(out, 0, outSize); + for (size_t i = 0; i < n / CHAR_BIT; i++) { + out[i] = ((u8 *)vector)[(start / CHAR_BIT) + i]; + } + sqlite3_result_blob(context, out, outSize, sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_BIT); + goto done; + } + } +done: + cleanup(vector); +} + +static void vec_to_json(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 1); + void *vector; + size_t dimensions; + vector_cleanup cleanup; + char *err; + enum VectorElementType elementType; + + int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType, + &cleanup, &err); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, err, -1); + sqlite3_free(err); + return; + } + + sqlite3_str *str = sqlite3_str_new(sqlite3_context_db_handle(context)); + sqlite3_str_appendall(str, "["); + for (size_t i = 0; i < dimensions; i++) { + if (i != 0) { + sqlite3_str_appendall(str, ","); + } + if (elementType == SQLITE_VEC_ELEMENT_TYPE_FLOAT32) { + f32 value = ((f32 *)vector)[i]; + if (isnan(value)) { + sqlite3_str_appendall(str, "null"); + } else { + sqlite3_str_appendf(str, "%f", value); + } + + } else if (elementType == SQLITE_VEC_ELEMENT_TYPE_INT8) { + sqlite3_str_appendf(str, "%d", ((i8 *)vector)[i]); + } else if (elementType == SQLITE_VEC_ELEMENT_TYPE_BIT) { + u8 b = (((u8 *)vector)[i / 8] >> (i % CHAR_BIT)) & 1; + sqlite3_str_appendf(str, "%d", b); + } + } + sqlite3_str_appendall(str, "]"); + int len = sqlite3_str_length(str); + char *s = sqlite3_str_finish(str); + if (s) { + sqlite3_result_text(context, s, len, sqlite3_free); + sqlite3_result_subtype(context, JSON_SUBTYPE); + } else { + sqlite3_result_error_nomem(context); + } + cleanup(vector); +} + +static void vec_normalize(sqlite3_context *context, int argc, + sqlite3_value **argv) { + assert(argc == 1); + void *vector; + size_t dimensions; + vector_cleanup cleanup; + char *err; + enum VectorElementType elementType; + + int rc = vector_from_value(argv[0], &vector, &dimensions, &elementType, + &cleanup, &err); + if (rc != SQLITE_OK) { + sqlite3_result_error(context, err, -1); + sqlite3_free(err); + return; + } + + if (elementType != SQLITE_VEC_ELEMENT_TYPE_FLOAT32) { + sqlite3_result_error( + context, "only float32 vectors are supported when normalizing", -1); + cleanup(vector); + return; + } + + int outSize = dimensions * sizeof(f32); + f32 *out = sqlite3_malloc(outSize); + if (!out) { + cleanup(vector); + sqlite3_result_error_code(context, SQLITE_NOMEM); + return; + } + memset(out, 0, outSize); + + f32 *v = (f32 *)vector; + + f32 norm = 0; + for (size_t i = 0; i < dimensions; i++) { + norm += v[i] * v[i]; + } + norm = sqrt(norm); + for (size_t i = 0; i < dimensions; i++) { + out[i] = v[i] / norm; + } + + sqlite3_result_blob(context, out, dimensions * sizeof(f32), sqlite3_free); + sqlite3_result_subtype(context, SQLITE_VEC_ELEMENT_TYPE_FLOAT32); + cleanup(vector); +} + +static void _static_text_func(sqlite3_context *context, int argc, + sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + sqlite3_result_text(context, sqlite3_user_data(context), -1, SQLITE_STATIC); +} + +#pragma endregion + +enum Vec0TokenType { + TOKEN_TYPE_IDENTIFIER, + TOKEN_TYPE_DIGIT, + TOKEN_TYPE_LBRACKET, + TOKEN_TYPE_RBRACKET, + TOKEN_TYPE_PLUS, + TOKEN_TYPE_EQ, +}; +struct Vec0Token { + enum Vec0TokenType token_type; + char *start; + char *end; +}; + +int is_alpha(char x) { + return (x >= 'a' && x <= 'z') || (x >= 'A' && x <= 'Z'); +} +int is_digit(char x) { return (x >= '0' && x <= '9'); } +int is_whitespace(char x) { + return x == ' ' || x == '\t' || x == '\n' || x == '\r'; +} + +#define VEC0_TOKEN_RESULT_EOF 1 +#define VEC0_TOKEN_RESULT_SOME 2 +#define VEC0_TOKEN_RESULT_ERROR 3 + +int vec0_token_next(char *start, char *end, struct Vec0Token *out) { + char *ptr = start; + while (ptr < end) { + char curr = *ptr; + if (is_whitespace(curr)) { + ptr++; + continue; + } else if (curr == '+') { + ptr++; + out->start = ptr; + out->end = ptr; + out->token_type = TOKEN_TYPE_PLUS; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == '[') { + ptr++; + out->start = ptr; + out->end = ptr; + out->token_type = TOKEN_TYPE_LBRACKET; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == ']') { + ptr++; + out->start = ptr; + out->end = ptr; + out->token_type = TOKEN_TYPE_RBRACKET; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == '=') { + ptr++; + out->start = ptr; + out->end = ptr; + out->token_type = TOKEN_TYPE_EQ; + return VEC0_TOKEN_RESULT_SOME; + } else if (is_alpha(curr)) { + char *start = ptr; + while (ptr < end && (is_alpha(*ptr) || is_digit(*ptr) || *ptr == '_')) { + ptr++; + } + out->start = start; + out->end = ptr; + out->token_type = TOKEN_TYPE_IDENTIFIER; + return VEC0_TOKEN_RESULT_SOME; + } else if (is_digit(curr)) { + char *start = ptr; + while (ptr < end && (is_digit(*ptr))) { + ptr++; + } + out->start = start; + out->end = ptr; + out->token_type = TOKEN_TYPE_DIGIT; + return VEC0_TOKEN_RESULT_SOME; + } else { + return VEC0_TOKEN_RESULT_ERROR; + } + } + return VEC0_TOKEN_RESULT_EOF; +} + +struct Vec0Scanner { + char *start; + char *end; + char *ptr; +}; + +void vec0_scanner_init(struct Vec0Scanner *scanner, const char *source, + int source_length) { + scanner->start = (char *)source; + scanner->end = (char *)source + source_length; + scanner->ptr = (char *)source; +} +int vec0_scanner_next(struct Vec0Scanner *scanner, struct Vec0Token *out) { + int rc = vec0_token_next(scanner->start, scanner->end, out); + if (rc == VEC0_TOKEN_RESULT_SOME) { + scanner->start = out->end; + } + return rc; +} + +int vec0_parse_table_option(const char *source, int source_length, + char **out_key, int *out_key_length, + char **out_value, int *out_value_length) { + int rc; + struct Vec0Scanner scanner; + struct Vec0Token token; + char *key; + char *value; + int keyLength, valueLength; + + vec0_scanner_init(&scanner, source, source_length); + + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + key = token.start; + keyLength = token.end - token.start; + + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_EQ) { + return SQLITE_EMPTY; + } + + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + !((token.token_type == TOKEN_TYPE_IDENTIFIER) || + (token.token_type == TOKEN_TYPE_DIGIT))) { + return SQLITE_ERROR; + } + value = token.start; + valueLength = token.end - token.start; + + rc = vec0_scanner_next(&scanner, &token); + if (rc == VEC0_TOKEN_RESULT_EOF) { + *out_key = key; + *out_key_length = keyLength; + *out_value = value; + *out_value_length = valueLength; + return SQLITE_OK; + } + return SQLITE_ERROR; +} +/** + * @brief Parse an argv[i] entry of a vec0 virtual table definition, and see if + * it's a PARTITION KEY definition. + * + * @param source: argv[i] source string + * @param source_length: length of the source string + * @param out_column_name: If it is a partition key, the output column name. Same lifetime + * as source, points to specific char * + * @param out_column_name_length: Length of out_column_name in bytes + * @param out_column_type: SQLITE_TEXT or SQLITE_INTEGER. + * @return int: SQLITE_EMPTY if not a PK, SQLITE_OK if it is. + */ +int vec0_parse_partition_key_definition(const char *source, int source_length, + char **out_column_name, + int *out_column_name_length, + int *out_column_type) { + struct Vec0Scanner scanner; + struct Vec0Token token; + char *column_name; + int column_name_length; + int column_type; + vec0_scanner_init(&scanner, source, source_length); + + // Check first token is identifier, will be the column name + int rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + + column_name = token.start; + column_name_length = token.end - token.start; + + // Check the next token matches "text" or "integer", as column type + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + if (sqlite3_strnicmp(token.start, "text", token.end - token.start) == 0) { + column_type = SQLITE_TEXT; + } else if (sqlite3_strnicmp(token.start, "int", token.end - token.start) == + 0 || + sqlite3_strnicmp(token.start, "integer", + token.end - token.start) == 0) { + column_type = SQLITE_INTEGER; + } else { + return SQLITE_EMPTY; + } + + // Check the next token is identifier and matches "partition" + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + if (sqlite3_strnicmp(token.start, "partition", token.end - token.start) != 0) { + return SQLITE_EMPTY; + } + + // Check the next token is identifier and matches "key" + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + if (sqlite3_strnicmp(token.start, "key", token.end - token.start) != 0) { + return SQLITE_EMPTY; + } + + *out_column_name = column_name; + *out_column_name_length = column_name_length; + *out_column_type = column_type; + + return SQLITE_OK; +} + +/** + * @brief Parse an argv[i] entry of a vec0 virtual table definition, and see if + * it's an auxiliar column definition, ie `+[name] [type]` like `+contents text` + * + * @param source: argv[i] source string + * @param source_length: length of the source string + * @param out_column_name: If it is a partition key, the output column name. Same lifetime + * as source, points to specific char * + * @param out_column_name_length: Length of out_column_name in bytes + * @param out_column_type: SQLITE_TEXT, SQLITE_INTEGER, SQLITE_FLOAT, or SQLITE_BLOB. + * @return int: SQLITE_EMPTY if not an aux column, SQLITE_OK if it is. + */ +int vec0_parse_auxiliary_column_definition(const char *source, int source_length, + char **out_column_name, + int *out_column_name_length, + int *out_column_type) { + struct Vec0Scanner scanner; + struct Vec0Token token; + char *column_name; + int column_name_length; + int column_type; + vec0_scanner_init(&scanner, source, source_length); + + // Check first token is '+', which denotes aux columns + int rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME || + token.token_type != TOKEN_TYPE_PLUS) { + return SQLITE_EMPTY; + } + + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + + column_name = token.start; + column_name_length = token.end - token.start; + + // Check the next token matches "text" or "integer", as column type + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + if (sqlite3_strnicmp(token.start, "text", token.end - token.start) == 0) { + column_type = SQLITE_TEXT; + } else if (sqlite3_strnicmp(token.start, "int", token.end - token.start) == + 0 || + sqlite3_strnicmp(token.start, "integer", + token.end - token.start) == 0) { + column_type = SQLITE_INTEGER; + } else if (sqlite3_strnicmp(token.start, "float", token.end - token.start) == + 0 || + sqlite3_strnicmp(token.start, "double", + token.end - token.start) == 0) { + column_type = SQLITE_FLOAT; + } else if (sqlite3_strnicmp(token.start, "blob", token.end - token.start) ==0) { + column_type = SQLITE_BLOB; + } else { + return SQLITE_EMPTY; + } + + *out_column_name = column_name; + *out_column_name_length = column_name_length; + *out_column_type = column_type; + + return SQLITE_OK; +} + +typedef enum { + VEC0_METADATA_COLUMN_KIND_BOOLEAN, + VEC0_METADATA_COLUMN_KIND_INTEGER, + VEC0_METADATA_COLUMN_KIND_FLOAT, + VEC0_METADATA_COLUMN_KIND_TEXT, + // future: blob, date, datetime +} vec0_metadata_column_kind; + +/** + * @brief Parse an argv[i] entry of a vec0 virtual table definition, and see if + * it's an metadata column definition, ie `[name] [type]` like `is_released boolean` + * + * @param source: argv[i] source string + * @param source_length: length of the source string + * @param out_column_name: If it is a metadata column, the output column name. Same lifetime + * as source, points to specific char * + * @param out_column_name_length: Length of out_column_name in bytes + * @param out_column_type: one of vec0_metadata_column_kind + * @return int: SQLITE_EMPTY if not an metadata column, SQLITE_OK if it is. + */ +int vec0_parse_metadata_column_definition(const char *source, int source_length, + char **out_column_name, + int *out_column_name_length, + vec0_metadata_column_kind *out_column_type) { + struct Vec0Scanner scanner; + struct Vec0Token token; + char *column_name; + int column_name_length; + vec0_metadata_column_kind column_type; + int rc; + vec0_scanner_init(&scanner, source, source_length); + + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME || + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + + column_name = token.start; + column_name_length = token.end - token.start; + + // Check the next token matches a valid metadata type + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME || + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + char * t = token.start; + int n = token.end - token.start; + if (sqlite3_strnicmp(t, "boolean", n) == 0 || sqlite3_strnicmp(t, "bool", n) == 0) { + column_type = VEC0_METADATA_COLUMN_KIND_BOOLEAN; + }else if (sqlite3_strnicmp(t, "int64", n) == 0 || sqlite3_strnicmp(t, "integer64", n) == 0 || sqlite3_strnicmp(t, "integer", n) == 0 || sqlite3_strnicmp(t, "int", n) == 0) { + column_type = VEC0_METADATA_COLUMN_KIND_INTEGER; + }else if (sqlite3_strnicmp(t, "float", n) == 0 || sqlite3_strnicmp(t, "double", n) == 0 || sqlite3_strnicmp(t, "float64", n) == 0 || sqlite3_strnicmp(t, "f64", n) == 0) { + column_type = VEC0_METADATA_COLUMN_KIND_FLOAT; + } else if (sqlite3_strnicmp(t, "text", n) == 0) { + column_type = VEC0_METADATA_COLUMN_KIND_TEXT; + } else { + return SQLITE_EMPTY; + } + + *out_column_name = column_name; + *out_column_name_length = column_name_length; + *out_column_type = column_type; + + return SQLITE_OK; +} + +/** + * @brief Parse an argv[i] entry of a vec0 virtual table definition, and see if + * it's a PRIMARY KEY definition. + * + * @param source: argv[i] source string + * @param source_length: length of the source string + * @param out_column_name: If it is a PK, the output column name. Same lifetime + * as source, points to specific char * + * @param out_column_name_length: Length of out_column_name in bytes + * @param out_column_type: SQLITE_TEXT or SQLITE_INTEGER. + * @return int: SQLITE_EMPTY if not a PK, SQLITE_OK if it is. + */ +int vec0_parse_primary_key_definition(const char *source, int source_length, + char **out_column_name, + int *out_column_name_length, + int *out_column_type) { + struct Vec0Scanner scanner; + struct Vec0Token token; + char *column_name; + int column_name_length; + int column_type; + vec0_scanner_init(&scanner, source, source_length); + + // Check first token is identifier, will be the column name + int rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + + column_name = token.start; + column_name_length = token.end - token.start; + + // Check the next token matches "text" or "integer", as column type + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + if (sqlite3_strnicmp(token.start, "text", token.end - token.start) == 0) { + column_type = SQLITE_TEXT; + } else if (sqlite3_strnicmp(token.start, "int", token.end - token.start) == + 0 || + sqlite3_strnicmp(token.start, "integer", + token.end - token.start) == 0) { + column_type = SQLITE_INTEGER; + } else { + return SQLITE_EMPTY; + } + + // Check the next token is identifier and matches "primary" + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + if (sqlite3_strnicmp(token.start, "primary", token.end - token.start) != 0) { + return SQLITE_EMPTY; + } + + // Check the next token is identifier and matches "key" + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + if (sqlite3_strnicmp(token.start, "key", token.end - token.start) != 0) { + return SQLITE_EMPTY; + } + + *out_column_name = column_name; + *out_column_name_length = column_name_length; + *out_column_type = column_type; + + return SQLITE_OK; +} + +enum Vec0DistanceMetrics { + VEC0_DISTANCE_METRIC_L2 = 1, + VEC0_DISTANCE_METRIC_COSINE = 2, + VEC0_DISTANCE_METRIC_L1 = 3, +}; + +struct VectorColumnDefinition { + char *name; + int name_length; + size_t dimensions; + enum VectorElementType element_type; + enum Vec0DistanceMetrics distance_metric; +}; + +struct Vec0PartitionColumnDefinition { + int type; + char * name; + int name_length; +}; + +struct Vec0AuxiliaryColumnDefinition { + int type; + char * name; + int name_length; +}; +struct Vec0MetadataColumnDefinition { + vec0_metadata_column_kind kind; + char * name; + int name_length; +}; + +size_t vector_byte_size(enum VectorElementType element_type, + size_t dimensions) { + switch (element_type) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: + return dimensions * sizeof(f32); + case SQLITE_VEC_ELEMENT_TYPE_INT8: + return dimensions * sizeof(i8); + case SQLITE_VEC_ELEMENT_TYPE_BIT: + return dimensions / CHAR_BIT; + } + return 0; +} + +size_t vector_column_byte_size(struct VectorColumnDefinition column) { + return vector_byte_size(column.element_type, column.dimensions); +} + +/** + * @brief Parse an vec0 vtab argv[i] column definition and see if + * it's a vector column defintion, ex `contents_embedding float[768]`. + * + * @param source vec0 argv[i] item + * @param source_length length of source in bytes + * @param outColumn Output the parse vector column to this struct, if success + * @return int SQLITE_OK on success, SQLITE_EMPTY is it's not a vector column + * definition, SQLITE_ERROR on error. + */ +int vec0_parse_vector_column(const char *source, int source_length, + struct VectorColumnDefinition *outColumn) { + // parses a vector column definition like so: + // "abc float[123]", "abc_123 bit[1234]", eetc. + // https://github.com/asg017/sqlite-vec/issues/46 + int rc; + struct Vec0Scanner scanner; + struct Vec0Token token; + + char *name; + int nameLength; + enum VectorElementType elementType; + enum Vec0DistanceMetrics distanceMetric = VEC0_DISTANCE_METRIC_L2; + int dimensions; + + vec0_scanner_init(&scanner, source, source_length); + + // starts with an identifier + rc = vec0_scanner_next(&scanner, &token); + + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + + name = token.start; + nameLength = token.end - token.start; + + // vector column type comes next: float, int, or bit + rc = vec0_scanner_next(&scanner, &token); + + if (rc != VEC0_TOKEN_RESULT_SOME || + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_EMPTY; + } + if (sqlite3_strnicmp(token.start, "float", 5) == 0 || + sqlite3_strnicmp(token.start, "f32", 3) == 0) { + elementType = SQLITE_VEC_ELEMENT_TYPE_FLOAT32; + } else if (sqlite3_strnicmp(token.start, "int8", 4) == 0 || + sqlite3_strnicmp(token.start, "i8", 2) == 0) { + elementType = SQLITE_VEC_ELEMENT_TYPE_INT8; + } else if (sqlite3_strnicmp(token.start, "bit", 3) == 0) { + elementType = SQLITE_VEC_ELEMENT_TYPE_BIT; + } else { + return SQLITE_EMPTY; + } + + // left '[' bracket + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_LBRACKET) { + return SQLITE_EMPTY; + } + + // digit, for vector dimension length + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_DIGIT) { + return SQLITE_ERROR; + } + dimensions = atoi(token.start); + if (dimensions <= 0) { + return SQLITE_ERROR; + } + + // // right ']' bracket + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_RBRACKET) { + return SQLITE_ERROR; + } + + // any other tokens left should be column-level options , ex `key=value` + // ex `distance_metric=L2 distance_metric=cosine` should error + while (1) { + // should be EOF or identifier (option key) + rc = vec0_scanner_next(&scanner, &token); + if (rc == VEC0_TOKEN_RESULT_EOF) { + break; + } + + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_ERROR; + } + + char *key = token.start; + int keyLength = token.end - token.start; + + if (sqlite3_strnicmp(key, "distance_metric", keyLength) == 0) { + + if (elementType == SQLITE_VEC_ELEMENT_TYPE_BIT) { + return SQLITE_ERROR; + } + // ensure equal sign after distance_metric + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && token.token_type != TOKEN_TYPE_EQ) { + return SQLITE_ERROR; + } + + // distance_metric value, an identifier (L2, cosine, etc) + rc = vec0_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME && + token.token_type != TOKEN_TYPE_IDENTIFIER) { + return SQLITE_ERROR; + } + + char *value = token.start; + int valueLength = token.end - token.start; + if (sqlite3_strnicmp(value, "l2", valueLength) == 0) { + distanceMetric = VEC0_DISTANCE_METRIC_L2; + } else if (sqlite3_strnicmp(value, "l1", valueLength) == 0) { + distanceMetric = VEC0_DISTANCE_METRIC_L1; + } else if (sqlite3_strnicmp(value, "cosine", valueLength) == 0) { + distanceMetric = VEC0_DISTANCE_METRIC_COSINE; + } else { + return SQLITE_ERROR; + } + } + // unknown key + else { + return SQLITE_ERROR; + } + } + + outColumn->name = sqlite3_mprintf("%.*s", nameLength, name); + if (!outColumn->name) { + return SQLITE_ERROR; + } + outColumn->name_length = nameLength; + outColumn->distance_metric = distanceMetric; + outColumn->element_type = elementType; + outColumn->dimensions = dimensions; + return SQLITE_OK; +} + +#pragma region vec_each table function + +typedef struct vec_each_vtab vec_each_vtab; +struct vec_each_vtab { + sqlite3_vtab base; +}; + +typedef struct vec_each_cursor vec_each_cursor; +struct vec_each_cursor { + sqlite3_vtab_cursor base; + i64 iRowid; + enum VectorElementType vector_type; + void *vector; + size_t dimensions; + vector_cleanup cleanup; +}; + +static int vec_eachConnect(sqlite3 *db, void *pAux, int argc, + const char *const *argv, sqlite3_vtab **ppVtab, + char **pzErr) { + UNUSED_PARAMETER(pAux); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + UNUSED_PARAMETER(pzErr); + vec_each_vtab *pNew; + int rc; + + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(value, vector hidden)"); +#define VEC_EACH_COLUMN_VALUE 0 +#define VEC_EACH_COLUMN_VECTOR 1 + if (rc == SQLITE_OK) { + pNew = sqlite3_malloc(sizeof(*pNew)); + *ppVtab = (sqlite3_vtab *)pNew; + if (pNew == 0) + return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +static int vec_eachDisconnect(sqlite3_vtab *pVtab) { + vec_each_vtab *p = (vec_each_vtab *)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +static int vec_eachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor) { + UNUSED_PARAMETER(p); + vec_each_cursor *pCur; + pCur = sqlite3_malloc(sizeof(*pCur)); + if (pCur == 0) + return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +static int vec_eachClose(sqlite3_vtab_cursor *cur) { + vec_each_cursor *pCur = (vec_each_cursor *)cur; + if(pCur->vector) { + pCur->cleanup(pCur->vector); + } + sqlite3_free(pCur); + return SQLITE_OK; +} + +static int vec_eachBestIndex(sqlite3_vtab *pVTab, + sqlite3_index_info *pIdxInfo) { + UNUSED_PARAMETER(pVTab); + int hasVector = 0; + for (int i = 0; i < pIdxInfo->nConstraint; i++) { + const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; + // printf("i=%d iColumn=%d, op=%d, usable=%d\n", i, pCons->iColumn, + // pCons->op, pCons->usable); + switch (pCons->iColumn) { + case VEC_EACH_COLUMN_VECTOR: { + if (pCons->op == SQLITE_INDEX_CONSTRAINT_EQ && pCons->usable) { + hasVector = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + break; + } + } + } + if (!hasVector) { + return SQLITE_CONSTRAINT; + } + + pIdxInfo->estimatedCost = (double)100000; + pIdxInfo->estimatedRows = 100000; + + return SQLITE_OK; +} + +static int vec_eachFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum, + const char *idxStr, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + assert(argc == 1); + vec_each_cursor *pCur = (vec_each_cursor *)pVtabCursor; + + if (pCur->vector) { + pCur->cleanup(pCur->vector); + pCur->vector = NULL; + } + + char *pzErrMsg; + int rc = vector_from_value(argv[0], &pCur->vector, &pCur->dimensions, + &pCur->vector_type, &pCur->cleanup, &pzErrMsg); + if (rc != SQLITE_OK) { + return SQLITE_ERROR; + } + pCur->iRowid = 0; + return SQLITE_OK; +} + +static int vec_eachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) { + vec_each_cursor *pCur = (vec_each_cursor *)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +static int vec_eachEof(sqlite3_vtab_cursor *cur) { + vec_each_cursor *pCur = (vec_each_cursor *)cur; + return pCur->iRowid >= (i64)pCur->dimensions; +} + +static int vec_eachNext(sqlite3_vtab_cursor *cur) { + vec_each_cursor *pCur = (vec_each_cursor *)cur; + pCur->iRowid++; + return SQLITE_OK; +} + +static int vec_eachColumn(sqlite3_vtab_cursor *cur, sqlite3_context *context, + int i) { + vec_each_cursor *pCur = (vec_each_cursor *)cur; + switch (i) { + case VEC_EACH_COLUMN_VALUE: + switch (pCur->vector_type) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + sqlite3_result_double(context, ((f32 *)pCur->vector)[pCur->iRowid]); + break; + } + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + u8 x = ((u8 *)pCur->vector)[pCur->iRowid / CHAR_BIT]; + sqlite3_result_int(context, + (x & (0b10000000 >> ((pCur->iRowid % CHAR_BIT)))) > 0); + break; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + sqlite3_result_int(context, ((i8 *)pCur->vector)[pCur->iRowid]); + break; + } + } + + break; + } + return SQLITE_OK; +} + +static sqlite3_module vec_eachModule = { + /* iVersion */ 0, + /* xCreate */ 0, + /* xConnect */ vec_eachConnect, + /* xBestIndex */ vec_eachBestIndex, + /* xDisconnect */ vec_eachDisconnect, + /* xDestroy */ 0, + /* xOpen */ vec_eachOpen, + /* xClose */ vec_eachClose, + /* xFilter */ vec_eachFilter, + /* xNext */ vec_eachNext, + /* xEof */ vec_eachEof, + /* xColumn */ vec_eachColumn, + /* xRowid */ vec_eachRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, +#if SQLITE_VERSION_NUMBER >= 3044000 + /* xIntegrity */ 0 +#endif +}; + +#pragma endregion + +#pragma region vec_npy_each table function + +enum NpyTokenType { + NPY_TOKEN_TYPE_IDENTIFIER, + NPY_TOKEN_TYPE_NUMBER, + NPY_TOKEN_TYPE_LPAREN, + NPY_TOKEN_TYPE_RPAREN, + NPY_TOKEN_TYPE_LBRACE, + NPY_TOKEN_TYPE_RBRACE, + NPY_TOKEN_TYPE_COLON, + NPY_TOKEN_TYPE_COMMA, + NPY_TOKEN_TYPE_STRING, + NPY_TOKEN_TYPE_FALSE, +}; + +struct NpyToken { + enum NpyTokenType token_type; + unsigned char *start; + unsigned char *end; +}; + +int npy_token_next(unsigned char *start, unsigned char *end, + struct NpyToken *out) { + unsigned char *ptr = start; + while (ptr < end) { + unsigned char curr = *ptr; + if (is_whitespace(curr)) { + ptr++; + continue; + } else if (curr == '(') { + out->start = ptr++; + out->end = ptr; + out->token_type = NPY_TOKEN_TYPE_LPAREN; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == ')') { + out->start = ptr++; + out->end = ptr; + out->token_type = NPY_TOKEN_TYPE_RPAREN; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == '{') { + out->start = ptr++; + out->end = ptr; + out->token_type = NPY_TOKEN_TYPE_LBRACE; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == '}') { + out->start = ptr++; + out->end = ptr; + out->token_type = NPY_TOKEN_TYPE_RBRACE; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == ':') { + out->start = ptr++; + out->end = ptr; + out->token_type = NPY_TOKEN_TYPE_COLON; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == ',') { + out->start = ptr++; + out->end = ptr; + out->token_type = NPY_TOKEN_TYPE_COMMA; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == '\'') { + unsigned char *start = ptr; + ptr++; + while (ptr < end) { + if ((*ptr) == '\'') { + break; + } + ptr++; + } + if ((*ptr) != '\'') { + return VEC0_TOKEN_RESULT_ERROR; + } + out->start = start; + out->end = ++ptr; + out->token_type = NPY_TOKEN_TYPE_STRING; + return VEC0_TOKEN_RESULT_SOME; + } else if (curr == 'F' && + strncmp((char *)ptr, "False", strlen("False")) == 0) { + out->start = ptr; + out->end = (ptr + (int)strlen("False")); + ptr = out->end; + out->token_type = NPY_TOKEN_TYPE_FALSE; + return VEC0_TOKEN_RESULT_SOME; + } else if (is_digit(curr)) { + unsigned char *start = ptr; + while (ptr < end && (is_digit(*ptr))) { + ptr++; + } + out->start = start; + out->end = ptr; + out->token_type = NPY_TOKEN_TYPE_NUMBER; + return VEC0_TOKEN_RESULT_SOME; + } else { + return VEC0_TOKEN_RESULT_ERROR; + } + } + return VEC0_TOKEN_RESULT_ERROR; +} + +struct NpyScanner { + unsigned char *start; + unsigned char *end; + unsigned char *ptr; +}; + +void npy_scanner_init(struct NpyScanner *scanner, const unsigned char *source, + int source_length) { + scanner->start = (unsigned char *)source; + scanner->end = (unsigned char *)source + source_length; + scanner->ptr = (unsigned char *)source; +} + +int npy_scanner_next(struct NpyScanner *scanner, struct NpyToken *out) { + int rc = npy_token_next(scanner->start, scanner->end, out); + if (rc == VEC0_TOKEN_RESULT_SOME) { + scanner->start = out->end; + } + return rc; +} + +#define NPY_PARSE_ERROR "Error parsing numpy array: " +int parse_npy_header(sqlite3_vtab *pVTab, const unsigned char *header, + size_t headerLength, + enum VectorElementType *out_element_type, + int *fortran_order, size_t *numElements, + size_t *numDimensions) { + + struct NpyScanner scanner; + struct NpyToken token; + int rc; + npy_scanner_init(&scanner, header, headerLength); + + if (npy_scanner_next(&scanner, &token) != VEC0_TOKEN_RESULT_SOME && + token.token_type != NPY_TOKEN_TYPE_LBRACE) { + vtab_set_error(pVTab, + NPY_PARSE_ERROR "numpy header did not start with '{'"); + return SQLITE_ERROR; + } + while (1) { + rc = npy_scanner_next(&scanner, &token); + if (rc != VEC0_TOKEN_RESULT_SOME) { + vtab_set_error(pVTab, NPY_PARSE_ERROR "expected key in numpy header"); + return SQLITE_ERROR; + } + + if (token.token_type == NPY_TOKEN_TYPE_RBRACE) { + break; + } + if (token.token_type != NPY_TOKEN_TYPE_STRING) { + vtab_set_error(pVTab, NPY_PARSE_ERROR + "expected a string as key in numpy header"); + return SQLITE_ERROR; + } + unsigned char *key = token.start; + + rc = npy_scanner_next(&scanner, &token); + if ((rc != VEC0_TOKEN_RESULT_SOME) || + (token.token_type != NPY_TOKEN_TYPE_COLON)) { + vtab_set_error(pVTab, NPY_PARSE_ERROR + "expected a ':' after key in numpy header"); + return SQLITE_ERROR; + } + + if (strncmp((char *)key, "'descr'", strlen("'descr'")) == 0) { + rc = npy_scanner_next(&scanner, &token); + if ((rc != VEC0_TOKEN_RESULT_SOME) || + (token.token_type != NPY_TOKEN_TYPE_STRING)) { + vtab_set_error(pVTab, NPY_PARSE_ERROR + "expected a string value after 'descr' key"); + return SQLITE_ERROR; + } + if (strncmp((char *)token.start, "'maxChunks = 1024; + pCur->chunksBufferSize = + (vector_byte_size(element_type, numDimensions)) * pCur->maxChunks; + pCur->chunksBuffer = sqlite3_malloc(pCur->chunksBufferSize); + if (pCur->chunksBufferSize && !pCur->chunksBuffer) { + return SQLITE_NOMEM; + } + + pCur->currentChunkSize = + fread(pCur->chunksBuffer, vector_byte_size(element_type, numDimensions), + pCur->maxChunks, file); + + pCur->currentChunkIndex = 0; + pCur->elementType = element_type; + pCur->nElements = numElements; + pCur->nDimensions = numDimensions; + pCur->input_type = VEC_NPY_EACH_INPUT_FILE; + + pCur->eof = pCur->currentChunkSize == 0; + pCur->file = file; + return SQLITE_OK; +} +#endif + +int parse_npy_buffer(sqlite3_vtab *pVTab, const unsigned char *buffer, + int bufferLength, void **data, size_t *numElements, + size_t *numDimensions, + enum VectorElementType *element_type) { + + if (bufferLength < 10) { + // IMP: V03312_20150 + vtab_set_error(pVTab, "numpy array too short"); + return SQLITE_ERROR; + } + if (memcmp(NPY_MAGIC, buffer, sizeof(NPY_MAGIC)) != 0) { + // V11954_28792 + vtab_set_error(pVTab, "numpy array does not contain the 'magic' header"); + return SQLITE_ERROR; + } + + u8 major = buffer[6]; + u8 minor = buffer[7]; + uint16_t headerLength = 0; + memcpy(&headerLength, &buffer[8], sizeof(uint16_t)); + + i32 totalHeaderLength = sizeof(NPY_MAGIC) + sizeof(major) + sizeof(minor) + + sizeof(headerLength) + headerLength; + i32 dataSize = bufferLength - totalHeaderLength; + + if (dataSize < 0) { + vtab_set_error(pVTab, "numpy array header length is invalid"); + return SQLITE_ERROR; + } + + const unsigned char *header = &buffer[10]; + int fortran_order; + + int rc = parse_npy_header(pVTab, header, headerLength, element_type, + &fortran_order, numElements, numDimensions); + if (rc != SQLITE_OK) { + return rc; + } + + i32 expectedDataSize = + (*numElements * vector_byte_size(*element_type, *numDimensions)); + if (expectedDataSize != dataSize) { + vtab_set_error(pVTab, + "numpy array error: Expected a data size of %d, found %d", + expectedDataSize, dataSize); + return SQLITE_ERROR; + } + + *data = (void *)&buffer[totalHeaderLength]; + return SQLITE_OK; +} + +static int vec_npy_eachConnect(sqlite3 *db, void *pAux, int argc, + const char *const *argv, sqlite3_vtab **ppVtab, + char **pzErr) { + UNUSED_PARAMETER(pAux); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + UNUSED_PARAMETER(pzErr); + vec_npy_each_vtab *pNew; + int rc; + + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(vector, input hidden)"); +#define VEC_NPY_EACH_COLUMN_VECTOR 0 +#define VEC_NPY_EACH_COLUMN_INPUT 1 + if (rc == SQLITE_OK) { + pNew = sqlite3_malloc(sizeof(*pNew)); + *ppVtab = (sqlite3_vtab *)pNew; + if (pNew == 0) + return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +static int vec_npy_eachDisconnect(sqlite3_vtab *pVtab) { + vec_npy_each_vtab *p = (vec_npy_each_vtab *)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +static int vec_npy_eachOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor) { + UNUSED_PARAMETER(p); + vec_npy_each_cursor *pCur; + pCur = sqlite3_malloc(sizeof(*pCur)); + if (pCur == 0) + return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +static int vec_npy_eachClose(sqlite3_vtab_cursor *cur) { + vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur; +#ifndef SQLITE_VEC_OMIT_FS + if (pCur->file) { + fclose(pCur->file); + pCur->file = NULL; + } +#endif + if (pCur->chunksBuffer) { + sqlite3_free(pCur->chunksBuffer); + pCur->chunksBuffer = NULL; + } + if (pCur->vector) { + pCur->vector = NULL; + } + sqlite3_free(pCur); + return SQLITE_OK; +} + +static int vec_npy_eachBestIndex(sqlite3_vtab *pVTab, + sqlite3_index_info *pIdxInfo) { + int hasInput; + for (int i = 0; i < pIdxInfo->nConstraint; i++) { + const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i]; + // printf("i=%d iColumn=%d, op=%d, usable=%d\n", i, pCons->iColumn, + // pCons->op, pCons->usable); + switch (pCons->iColumn) { + case VEC_NPY_EACH_COLUMN_INPUT: { + if (pCons->op == SQLITE_INDEX_CONSTRAINT_EQ && pCons->usable) { + hasInput = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + break; + } + } + } + if (!hasInput) { + pVTab->zErrMsg = sqlite3_mprintf("input argument is required"); + return SQLITE_ERROR; + } + + pIdxInfo->estimatedCost = (double)100000; + pIdxInfo->estimatedRows = 100000; + + return SQLITE_OK; +} + +static int vec_npy_eachFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum, + const char *idxStr, int argc, + sqlite3_value **argv) { + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + assert(argc == 1); + int rc; + + vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)pVtabCursor; + +#ifndef SQLITE_VEC_OMIT_FS + if (pCur->file) { + fclose(pCur->file); + pCur->file = NULL; + } +#endif + if (pCur->chunksBuffer) { + sqlite3_free(pCur->chunksBuffer); + pCur->chunksBuffer = NULL; + } + if (pCur->vector) { + pCur->vector = NULL; + } + +#ifndef SQLITE_VEC_OMIT_FS + struct VecNpyFile *f = NULL; + if ((f = sqlite3_value_pointer(argv[0], SQLITE_VEC_NPY_FILE_NAME))) { + FILE *file = fopen(f->path, "r"); + if (!file) { + vtab_set_error(pVtabCursor->pVtab, "Could not open numpy file"); + return SQLITE_ERROR; + } + + rc = parse_npy_file(pVtabCursor->pVtab, file, pCur); + if (rc != SQLITE_OK) { +#ifndef SQLITE_VEC_OMIT_FS + fclose(file); +#endif + return rc; + } + + } else +#endif + { + + const unsigned char *input = sqlite3_value_blob(argv[0]); + int inputLength = sqlite3_value_bytes(argv[0]); + void *data; + size_t numElements; + size_t numDimensions; + enum VectorElementType element_type; + + rc = parse_npy_buffer(pVtabCursor->pVtab, input, inputLength, &data, + &numElements, &numDimensions, &element_type); + if (rc != SQLITE_OK) { + return rc; + } + + pCur->vector = data; + pCur->elementType = element_type; + pCur->nElements = numElements; + pCur->nDimensions = numDimensions; + pCur->input_type = VEC_NPY_EACH_INPUT_BUFFER; + } + + pCur->iRowid = 0; + return SQLITE_OK; +} + +static int vec_npy_eachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) { + vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +static int vec_npy_eachEof(sqlite3_vtab_cursor *cur) { + vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur; + if (pCur->input_type == VEC_NPY_EACH_INPUT_BUFFER) { + return (!pCur->nElements) || (size_t)pCur->iRowid >= pCur->nElements; + } + return pCur->eof; +} + +static int vec_npy_eachNext(sqlite3_vtab_cursor *cur) { + vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur; + pCur->iRowid++; + if (pCur->input_type == VEC_NPY_EACH_INPUT_BUFFER) { + return SQLITE_OK; + } + +#ifndef SQLITE_VEC_OMIT_FS + // else: input is a file + pCur->currentChunkIndex++; + if (pCur->currentChunkIndex >= pCur->currentChunkSize) { + pCur->currentChunkSize = + fread(pCur->chunksBuffer, + vector_byte_size(pCur->elementType, pCur->nDimensions), + pCur->maxChunks, pCur->file); + if (!pCur->currentChunkSize) { + pCur->eof = 1; + } + pCur->currentChunkIndex = 0; + } +#endif + return SQLITE_OK; +} + +static int vec_npy_eachColumnBuffer(vec_npy_each_cursor *pCur, + sqlite3_context *context, int i) { + switch (i) { + case VEC_NPY_EACH_COLUMN_VECTOR: { + sqlite3_result_subtype(context, pCur->elementType); + switch (pCur->elementType) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + sqlite3_result_blob( + context, + &((unsigned char *) + pCur->vector)[pCur->iRowid * pCur->nDimensions * sizeof(f32)], + pCur->nDimensions * sizeof(f32), SQLITE_TRANSIENT); + + break; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + // https://github.com/asg017/sqlite-vec/issues/42 + sqlite3_result_error(context, + "vec_npy_each only supports float32 vectors", -1); + break; + } + } + + break; + } + } + return SQLITE_OK; +} +static int vec_npy_eachColumnFile(vec_npy_each_cursor *pCur, + sqlite3_context *context, int i) { + switch (i) { + case VEC_NPY_EACH_COLUMN_VECTOR: { + switch (pCur->elementType) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + sqlite3_result_blob( + context, + &((unsigned char *) + pCur->chunksBuffer)[pCur->currentChunkIndex * + pCur->nDimensions * sizeof(f32)], + pCur->nDimensions * sizeof(f32), SQLITE_TRANSIENT); + break; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + // https://github.com/asg017/sqlite-vec/issues/42 + sqlite3_result_error(context, + "vec_npy_each only supports float32 vectors", -1); + break; + } + } + break; + } + } + return SQLITE_OK; +} +static int vec_npy_eachColumn(sqlite3_vtab_cursor *cur, + sqlite3_context *context, int i) { + vec_npy_each_cursor *pCur = (vec_npy_each_cursor *)cur; + switch (pCur->input_type) { + case VEC_NPY_EACH_INPUT_BUFFER: + return vec_npy_eachColumnBuffer(pCur, context, i); + case VEC_NPY_EACH_INPUT_FILE: + return vec_npy_eachColumnFile(pCur, context, i); + } + return SQLITE_ERROR; +} + +static sqlite3_module vec_npy_eachModule = { + /* iVersion */ 0, + /* xCreate */ 0, + /* xConnect */ vec_npy_eachConnect, + /* xBestIndex */ vec_npy_eachBestIndex, + /* xDisconnect */ vec_npy_eachDisconnect, + /* xDestroy */ 0, + /* xOpen */ vec_npy_eachOpen, + /* xClose */ vec_npy_eachClose, + /* xFilter */ vec_npy_eachFilter, + /* xNext */ vec_npy_eachNext, + /* xEof */ vec_npy_eachEof, + /* xColumn */ vec_npy_eachColumn, + /* xRowid */ vec_npy_eachRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, +#if SQLITE_VERSION_NUMBER >= 3044000 + /* xIntegrity */ 0, +#endif +}; + +#pragma endregion + +#pragma region vec0 virtual table + +#define VEC0_COLUMN_ID 0 +#define VEC0_COLUMN_USERN_START 1 +#define VEC0_COLUMN_OFFSET_DISTANCE 1 +#define VEC0_COLUMN_OFFSET_K 2 + +#define VEC0_SHADOW_INFO_NAME "\"%w\".\"%w_info\"" + +#define VEC0_SHADOW_CHUNKS_NAME "\"%w\".\"%w_chunks\"" +/// 1) schema, 2) original vtab table name +#define VEC0_SHADOW_CHUNKS_CREATE \ + "CREATE TABLE " VEC0_SHADOW_CHUNKS_NAME "(" \ + "chunk_id INTEGER PRIMARY KEY AUTOINCREMENT," \ + "size INTEGER NOT NULL," \ + "validity BLOB NOT NULL," \ + "rowids BLOB NOT NULL" \ + ");" + +#define VEC0_SHADOW_ROWIDS_NAME "\"%w\".\"%w_rowids\"" +/// 1) schema, 2) original vtab table name +#define VEC0_SHADOW_ROWIDS_CREATE_BASIC \ + "CREATE TABLE " VEC0_SHADOW_ROWIDS_NAME "(" \ + "rowid INTEGER PRIMARY KEY AUTOINCREMENT," \ + "id," \ + "chunk_id INTEGER," \ + "chunk_offset INTEGER" \ + ");" + +// vec0 tables with a text primary keys are still backed by int64 primary keys, +// since a fixed-length rowid is required for vec0 chunks. But we add a new 'id +// text unique' column to emulate a text primary key interface. +#define VEC0_SHADOW_ROWIDS_CREATE_PK_TEXT \ + "CREATE TABLE " VEC0_SHADOW_ROWIDS_NAME "(" \ + "rowid INTEGER PRIMARY KEY AUTOINCREMENT," \ + "id TEXT UNIQUE NOT NULL," \ + "chunk_id INTEGER," \ + "chunk_offset INTEGER" \ + ");" + +/// 1) schema, 2) original vtab table name +#define VEC0_SHADOW_VECTOR_N_NAME "\"%w\".\"%w_vector_chunks%02d\"" + +/// 1) schema, 2) original vtab table name +#define VEC0_SHADOW_VECTOR_N_CREATE \ + "CREATE TABLE " VEC0_SHADOW_VECTOR_N_NAME "(" \ + "rowid PRIMARY KEY," \ + "vectors BLOB NOT NULL" \ + ");" + +#define VEC0_SHADOW_AUXILIARY_NAME "\"%w\".\"%w_auxiliary\"" + +#define VEC0_SHADOW_METADATA_N_NAME "\"%w\".\"%w_metadatachunks%02d\"" +#define VEC0_SHADOW_METADATA_TEXT_DATA_NAME "\"%w\".\"%w_metadatatext%02d\"" + +#define VEC_INTERAL_ERROR "Internal sqlite-vec error: " +#define REPORT_URL "https://github.com/asg017/sqlite-vec/issues/new" + +typedef struct vec0_vtab vec0_vtab; + +#define VEC0_MAX_VECTOR_COLUMNS 16 +#define VEC0_MAX_PARTITION_COLUMNS 4 +#define VEC0_MAX_AUXILIARY_COLUMNS 16 +#define VEC0_MAX_METADATA_COLUMNS 16 + +#define SQLITE_VEC_VEC0_MAX_DIMENSIONS 8192 +#define VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH 16 +#define VEC0_METADATA_TEXT_VIEW_DATA_LENGTH 12 + +typedef enum { + // vector column, ie "contents_embedding float[1024]" + SQLITE_VEC0_USER_COLUMN_KIND_VECTOR = 1, + + // partition key column, ie "user_id integer partition key" + SQLITE_VEC0_USER_COLUMN_KIND_PARTITION = 2, + + // + SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY = 3, + + // metadata column that can be filtered, ie "genre text" + SQLITE_VEC0_USER_COLUMN_KIND_METADATA = 4, +} vec0_user_column_kind; + +struct vec0_vtab { + sqlite3_vtab base; + + // the SQLite connection of the host database + sqlite3 *db; + + // True if the primary key of the vec0 table has a column type TEXT. + // Will change the schema of the _rowids table, and insert/query logic. + int pkIsText; + + // number of defined vector columns. + int numVectorColumns; + + // number of defined PARTITION KEY columns. + int numPartitionColumns; + + // number of defined auxiliary columns + int numAuxiliaryColumns; + + // number of defined metadata columns + int numMetadataColumns; + + + // Name of the schema the table exists on. + // Must be freed with sqlite3_free() + char *schemaName; + + // Name of the table the table exists on. + // Must be freed with sqlite3_free() + char *tableName; + + // Name of the _rowids shadow table. + // Must be freed with sqlite3_free() + char *shadowRowidsName; + + // Name of the _chunks shadow table. + // Must be freed with sqlite3_free() + char *shadowChunksName; + + // contains enum vec0_user_column_kind values for up to + // numVectorColumns + numPartitionColumns entries + vec0_user_column_kind user_column_kinds[VEC0_MAX_VECTOR_COLUMNS + VEC0_MAX_PARTITION_COLUMNS + VEC0_MAX_AUXILIARY_COLUMNS + VEC0_MAX_METADATA_COLUMNS]; + + uint8_t user_column_idxs[VEC0_MAX_VECTOR_COLUMNS + VEC0_MAX_PARTITION_COLUMNS + VEC0_MAX_AUXILIARY_COLUMNS + VEC0_MAX_METADATA_COLUMNS]; + + + // Name of all the vector chunk shadow tables. + // Ex '_vector_chunks00' + // Only the first numVectorColumns entries will be available. + // The first numVectorColumns entries must be freed with sqlite3_free() + char *shadowVectorChunksNames[VEC0_MAX_VECTOR_COLUMNS]; + + // Name of all metadata chunk shadow tables, ie `_metadatachunks00` + // Only the first numMetadataColumns entries will be available. + // The first numMetadataColumns entries must be freed with sqlite3_free() + char *shadowMetadataChunksNames[VEC0_MAX_METADATA_COLUMNS]; + + struct VectorColumnDefinition vector_columns[VEC0_MAX_VECTOR_COLUMNS]; + struct Vec0PartitionColumnDefinition paritition_columns[VEC0_MAX_PARTITION_COLUMNS]; + struct Vec0AuxiliaryColumnDefinition auxiliary_columns[VEC0_MAX_AUXILIARY_COLUMNS]; + struct Vec0MetadataColumnDefinition metadata_columns[VEC0_MAX_METADATA_COLUMNS]; + + int chunk_size; + + // select latest chunk from _chunks, getting chunk_id + sqlite3_stmt *stmtLatestChunk; + + /** + * Statement to insert a row into the _rowids table, with a rowid. + * Parameters: + * 1: int64, rowid to insert + * Result columns: none + * SQL: "INSERT INTO _rowids(rowid) VALUES (?)" + * + * Must be cleaned up with sqlite3_finalize(). + */ + sqlite3_stmt *stmtRowidsInsertRowid; + + /** + * Statement to insert a row into the _rowids table, with an id. + * The id column isn't a tradition primary key, but instead a unique + * column to handle "text primary key" vec0 tables. The true int64 rowid + * can be retrieved after inserting with sqlite3_last_rowid(). + * + * Parameters: + * 1: text or null, id to insert + * Result columns: none + * + * Must be cleaned up with sqlite3_finalize(). + */ + sqlite3_stmt *stmtRowidsInsertId; + + /** + * Statement to update the "position" columns chunk_id and chunk_offset for + * a given _rowids row. Used when the "next available" chunk position is found + * for a vector. + * + * Parameters: + * 1: int64, chunk_id value + * 2: int64, chunk_offset value + * 3: int64, rowid value + * Result columns: none + * + * Must be cleaned up with sqlite3_finalize(). + */ + sqlite3_stmt *stmtRowidsUpdatePosition; + + /** + * Statement to quickly find the chunk_id + chunk_offset of a given row. + * Parameters: + * 1: rowid of the row/vector to lookup + * Result columns: + * 0: chunk_id (i64) + * 1: chunk_offset (i64) + * SQL: "SELECT id, chunk_id, chunk_offset FROM _rowids WHERE rowid = ?"" + * + * Must be cleaned up with sqlite3_finalize(). + */ + sqlite3_stmt *stmtRowidsGetChunkPosition; +}; + +/** + * @brief Finalize all the sqlite3_stmt members in a vec0_vtab. + * + * @param p vec0_vtab pointer + */ +void vec0_free_resources(vec0_vtab *p) { + sqlite3_finalize(p->stmtLatestChunk); + p->stmtLatestChunk = NULL; + sqlite3_finalize(p->stmtRowidsInsertRowid); + p->stmtRowidsInsertRowid = NULL; + sqlite3_finalize(p->stmtRowidsInsertId); + p->stmtRowidsInsertId = NULL; + sqlite3_finalize(p->stmtRowidsUpdatePosition); + p->stmtRowidsUpdatePosition = NULL; + sqlite3_finalize(p->stmtRowidsGetChunkPosition); + p->stmtRowidsGetChunkPosition = NULL; +} + +/** + * @brief Free all memory and sqlite3_stmt members of a vec0_vtab + * + * @param p vec0_vtab pointer + */ +void vec0_free(vec0_vtab *p) { + vec0_free_resources(p); + + sqlite3_free(p->schemaName); + p->schemaName = NULL; + sqlite3_free(p->tableName); + p->tableName = NULL; + sqlite3_free(p->shadowChunksName); + p->shadowChunksName = NULL; + sqlite3_free(p->shadowRowidsName); + p->shadowRowidsName = NULL; + + for (int i = 0; i < p->numVectorColumns; i++) { + sqlite3_free(p->shadowVectorChunksNames[i]); + p->shadowVectorChunksNames[i] = NULL; + + sqlite3_free(p->vector_columns[i].name); + p->vector_columns[i].name = NULL; + } +} + +int vec0_num_defined_user_columns(vec0_vtab *p) { + return p->numVectorColumns + p->numPartitionColumns + p->numAuxiliaryColumns + p->numMetadataColumns; +} + +/** + * @brief Returns the index of the distance hidden column for the given vec0 + * table. + * + * @param p vec0 table + * @return int + */ +int vec0_column_distance_idx(vec0_vtab *p) { + return VEC0_COLUMN_USERN_START + (vec0_num_defined_user_columns(p) - 1) + + VEC0_COLUMN_OFFSET_DISTANCE; +} + +/** + * @brief Returns the index of the k hidden column for the given vec0 table. + * + * @param p vec0 table + * @return int k column index + */ +int vec0_column_k_idx(vec0_vtab *p) { + return VEC0_COLUMN_USERN_START + (vec0_num_defined_user_columns(p) - 1) + + VEC0_COLUMN_OFFSET_K; +} + +/** + * Returns 1 if the given column-based index is a valid vector column, + * 0 otherwise. + */ +int vec0_column_idx_is_vector(vec0_vtab *pVtab, int column_idx) { + return column_idx >= VEC0_COLUMN_USERN_START && + column_idx <= (VEC0_COLUMN_USERN_START + vec0_num_defined_user_columns(pVtab) - 1) && + pVtab->user_column_kinds[column_idx - VEC0_COLUMN_USERN_START] == SQLITE_VEC0_USER_COLUMN_KIND_VECTOR; +} + +/** + * Returns the vector index of the given user column index. + * ONLY call if validated with vec0_column_idx_is_vector before + */ +int vec0_column_idx_to_vector_idx(vec0_vtab *pVtab, int column_idx) { + UNUSED_PARAMETER(pVtab); + return pVtab->user_column_idxs[column_idx - VEC0_COLUMN_USERN_START]; +} +/** + * Returns 1 if the given column-based index is a "partition key" column, + * 0 otherwise. + */ +int vec0_column_idx_is_partition(vec0_vtab *pVtab, int column_idx) { + return column_idx >= VEC0_COLUMN_USERN_START && + column_idx <= (VEC0_COLUMN_USERN_START + vec0_num_defined_user_columns(pVtab) - 1) && + pVtab->user_column_kinds[column_idx - VEC0_COLUMN_USERN_START] == SQLITE_VEC0_USER_COLUMN_KIND_PARTITION; +} + +/** + * Returns the partition column index of the given user column index. + * ONLY call if validated with vec0_column_idx_is_vector before + */ +int vec0_column_idx_to_partition_idx(vec0_vtab *pVtab, int column_idx) { + UNUSED_PARAMETER(pVtab); + return pVtab->user_column_idxs[column_idx - VEC0_COLUMN_USERN_START]; +} + +/** + * Returns 1 if the given column-based index is a auxiliary column, + * 0 otherwise. + */ +int vec0_column_idx_is_auxiliary(vec0_vtab *pVtab, int column_idx) { + return column_idx >= VEC0_COLUMN_USERN_START && + column_idx <= (VEC0_COLUMN_USERN_START + vec0_num_defined_user_columns(pVtab) - 1) && + pVtab->user_column_kinds[column_idx - VEC0_COLUMN_USERN_START] == SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY; +} + +/** + * Returns the auxiliary column index of the given user column index. + * ONLY call if validated with vec0_column_idx_to_partition_idx before + */ +int vec0_column_idx_to_auxiliary_idx(vec0_vtab *pVtab, int column_idx) { + UNUSED_PARAMETER(pVtab); + return pVtab->user_column_idxs[column_idx - VEC0_COLUMN_USERN_START]; +} + +/** + * Returns 1 if the given column-based index is a metadata column, + * 0 otherwise. + */ +int vec0_column_idx_is_metadata(vec0_vtab *pVtab, int column_idx) { + return column_idx >= VEC0_COLUMN_USERN_START && + column_idx <= (VEC0_COLUMN_USERN_START + vec0_num_defined_user_columns(pVtab) - 1) && + pVtab->user_column_kinds[column_idx - VEC0_COLUMN_USERN_START] == SQLITE_VEC0_USER_COLUMN_KIND_METADATA; +} + +/** + * Returns the metadata column index of the given user column index. + * ONLY call if validated with vec0_column_idx_is_metadata before + */ +int vec0_column_idx_to_metadata_idx(vec0_vtab *pVtab, int column_idx) { + UNUSED_PARAMETER(pVtab); + return pVtab->user_column_idxs[column_idx - VEC0_COLUMN_USERN_START]; +} + +/** + * @brief Retrieve the chunk_id, chunk_offset, and possible "id" value + * of a vec0_vtab row with the provided rowid + * + * @param p vec0_vtab + * @param rowid the rowid of the row to query + * @param id output, optional sqlite3_value to provide the id. + * Useful for text PK rows. Must be freed with sqlite3_value_free() + * @param chunk_id output, the chunk_id the row belongs to + * @param chunk_offset output, the offset within the chunk the row belongs to + * @return SQLITE_ROW on success, error code otherwise. SQLITE_EMPTY if row DNE + */ +int vec0_get_chunk_position(vec0_vtab *p, i64 rowid, sqlite3_value **id, + i64 *chunk_id, i64 *chunk_offset) { + int rc; + + if (!p->stmtRowidsGetChunkPosition) { + const char *zSql = + sqlite3_mprintf("SELECT id, chunk_id, chunk_offset " + "FROM " VEC0_SHADOW_ROWIDS_NAME " WHERE rowid = ?", + p->schemaName, p->tableName); + if (!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtRowidsGetChunkPosition, 0); + sqlite3_free((void *)zSql); + if (rc != SQLITE_OK) { + vtab_set_error( + &p->base, VEC_INTERAL_ERROR + "could not initialize 'rowids get chunk position' statement"); + goto cleanup; + } + } + + sqlite3_bind_int64(p->stmtRowidsGetChunkPosition, 1, rowid); + rc = sqlite3_step(p->stmtRowidsGetChunkPosition); + // special case: when no results, return SQLITE_EMPTY to convey "that chunk + // position doesnt exist" + if (rc == SQLITE_DONE) { + rc = SQLITE_EMPTY; + goto cleanup; + } + if (rc != SQLITE_ROW) { + goto cleanup; + } + + if (id) { + sqlite3_value *value = + sqlite3_column_value(p->stmtRowidsGetChunkPosition, 0); + *id = sqlite3_value_dup(value); + if (!*id) { + rc = SQLITE_NOMEM; + goto cleanup; + } + } + + if (chunk_id) { + *chunk_id = sqlite3_column_int64(p->stmtRowidsGetChunkPosition, 1); + } + if (chunk_offset) { + *chunk_offset = sqlite3_column_int64(p->stmtRowidsGetChunkPosition, 2); + } + + rc = SQLITE_OK; + +cleanup: + sqlite3_reset(p->stmtRowidsGetChunkPosition); + sqlite3_clear_bindings(p->stmtRowidsGetChunkPosition); + return rc; +} + +/** + * @brief Return the id value from the _rowids table where _rowids.rowid = + * rowid. + * + * @param pVtab: vec0 table to query + * @param rowid: rowid of the row to query. + * @param out: A dup'ed sqlite3_value of the id column. Might be null. + * Must be cleaned up with sqlite3_value_free(). + * @returns SQLITE_OK on success, error code on failure + */ +int vec0_get_id_value_from_rowid(vec0_vtab *pVtab, i64 rowid, + sqlite3_value **out) { + // PERF: different strategy than get_chunk_position? + return vec0_get_chunk_position((vec0_vtab *)pVtab, rowid, out, NULL, NULL); +} + +int vec0_rowid_from_id(vec0_vtab *p, sqlite3_value *valueId, i64 *rowid) { + sqlite3_stmt *stmt = NULL; + int rc; + char *zSql; + zSql = sqlite3_mprintf("SELECT rowid" + " FROM " VEC0_SHADOW_ROWIDS_NAME " WHERE id = ?", + p->schemaName, p->tableName); + if (!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + goto cleanup; + } + sqlite3_bind_value(stmt, 1, valueId); + rc = sqlite3_step(stmt); + if (rc == SQLITE_DONE) { + rc = SQLITE_EMPTY; + goto cleanup; + } + if (rc != SQLITE_ROW) { + goto cleanup; + } + *rowid = sqlite3_column_int64(stmt, 0); + rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + goto cleanup; + } + + rc = SQLITE_OK; + +cleanup: + sqlite3_finalize(stmt); + return rc; +} + +int vec0_result_id(vec0_vtab *p, sqlite3_context *context, i64 rowid) { + if (!p->pkIsText) { + sqlite3_result_int64(context, rowid); + return SQLITE_OK; + } + sqlite3_value *valueId; + int rc = vec0_get_id_value_from_rowid(p, rowid, &valueId); + if (rc != SQLITE_OK) { + return rc; + } + if (!valueId) { + sqlite3_result_error_nomem(context); + } else { + sqlite3_result_value(context, valueId); + sqlite3_value_free(valueId); + } + return SQLITE_OK; +} + +/** + * @brief + * + * @param pVtab: virtual table to query + * @param rowid: row to lookup + * @param vector_column_idx: which vector column to query + * @param outVector: Output pointer to the vector buffer. + * Must be sqlite3_free()'ed. + * @param outVectorSize: Pointer to a int where the size of outVector + * will be stored. + * @return int SQLITE_OK on success. + */ +int vec0_get_vector_data(vec0_vtab *pVtab, i64 rowid, int vector_column_idx, + void **outVector, int *outVectorSize) { + vec0_vtab *p = pVtab; + int rc, brc; + i64 chunk_id; + i64 chunk_offset; + size_t size; + void *buf = NULL; + int blobOffset; + sqlite3_blob *vectorBlob = NULL; + assert((vector_column_idx >= 0) && + (vector_column_idx < pVtab->numVectorColumns)); + + rc = vec0_get_chunk_position(pVtab, rowid, NULL, &chunk_id, &chunk_offset); + if (rc == SQLITE_EMPTY) { + vtab_set_error(&pVtab->base, "Could not find a row with rowid %lld", rowid); + goto cleanup; + } + if (rc != SQLITE_OK) { + goto cleanup; + } + + rc = sqlite3_blob_open(p->db, p->schemaName, + p->shadowVectorChunksNames[vector_column_idx], + "vectors", chunk_id, 0, &vectorBlob); + + if (rc != SQLITE_OK) { + vtab_set_error(&pVtab->base, + "Could not fetch vector data for %lld, opening blob failed", + rowid); + rc = SQLITE_ERROR; + goto cleanup; + } + + size = vector_column_byte_size(pVtab->vector_columns[vector_column_idx]); + blobOffset = chunk_offset * size; + + buf = sqlite3_malloc(size); + if (!buf) { + rc = SQLITE_NOMEM; + goto cleanup; + } + + rc = sqlite3_blob_read(vectorBlob, buf, size, blobOffset); + if (rc != SQLITE_OK) { + sqlite3_free(buf); + buf = NULL; + vtab_set_error( + &pVtab->base, + "Could not fetch vector data for %lld, reading from blob failed", + rowid); + rc = SQLITE_ERROR; + goto cleanup; + } + + *outVector = buf; + if (outVectorSize) { + *outVectorSize = size; + } + rc = SQLITE_OK; + +cleanup: + brc = sqlite3_blob_close(vectorBlob); + if ((rc == SQLITE_OK) && (brc != SQLITE_OK)) { + vtab_set_error( + &p->base, VEC_INTERAL_ERROR + "unknown error, could not close vector blob, please file an issue"); + return brc; + } + + return rc; +} + +/** + * @brief Retrieve the sqlite3_value of the i'th partition value for the given row. + * + * @param pVtab - the vec0_vtab in questions + * @param rowid - rowid of target row + * @param partition_idx - which partition column to retrieve + * @param outValue - output sqlite3_value + * @return int - SQLITE_OK on success, otherwise error code + */ +int vec0_get_partition_value_for_rowid(vec0_vtab *pVtab, i64 rowid, int partition_idx, sqlite3_value ** outValue) { + int rc; + i64 chunk_id; + i64 chunk_offset; + rc = vec0_get_chunk_position(pVtab, rowid, NULL, &chunk_id, &chunk_offset); + if(rc != SQLITE_OK) { + return rc; + } + sqlite3_stmt * stmt = NULL; + char * zSql = sqlite3_mprintf("SELECT partition%02d FROM " VEC0_SHADOW_CHUNKS_NAME " WHERE chunk_id = ?", partition_idx, pVtab->schemaName, pVtab->tableName); + if(!zSql) { + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(pVtab->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if(rc != SQLITE_OK) { + return rc; + } + sqlite3_bind_int64(stmt, 1, chunk_id); + rc = sqlite3_step(stmt); + if(rc != SQLITE_ROW) { + rc = SQLITE_ERROR; + goto done; + } + *outValue = sqlite3_value_dup(sqlite3_column_value(stmt, 0)); + if(!*outValue) { + rc = SQLITE_NOMEM; + goto done; + } + rc = SQLITE_OK; + + done: + sqlite3_finalize(stmt); + return rc; + +} + +/** + * @brief Get the value of an auxiliary column for the given rowid + * + * @param pVtab vec0_vtab + * @param rowid the rowid of the row to lookup + * @param auxiliary_idx aux index of the column we care about + * @param outValue Output sqlite3_value to store + * @return int SQLITE_OK on success, error code otherwise + */ +int vec0_get_auxiliary_value_for_rowid(vec0_vtab *pVtab, i64 rowid, int auxiliary_idx, sqlite3_value ** outValue) { + int rc; + sqlite3_stmt * stmt = NULL; + char * zSql = sqlite3_mprintf("SELECT value%02d FROM " VEC0_SHADOW_AUXILIARY_NAME " WHERE rowid = ?", auxiliary_idx, pVtab->schemaName, pVtab->tableName); + if(!zSql) { + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(pVtab->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if(rc != SQLITE_OK) { + return rc; + } + sqlite3_bind_int64(stmt, 1, rowid); + rc = sqlite3_step(stmt); + if(rc != SQLITE_ROW) { + rc = SQLITE_ERROR; + goto done; + } + *outValue = sqlite3_value_dup(sqlite3_column_value(stmt, 0)); + if(!*outValue) { + rc = SQLITE_NOMEM; + goto done; + } + rc = SQLITE_OK; + + done: + sqlite3_finalize(stmt); + return rc; +} + +/** + * @brief Result the given metadata value for the given row and metadata column index. + * Will traverse the metadatachunksNN table with BLOB I/0 for the given rowid. + * + * @param p + * @param rowid + * @param metadata_idx + * @param context + * @return int + */ +int vec0_result_metadata_value_for_rowid(vec0_vtab *p, i64 rowid, int metadata_idx, sqlite3_context * context) { + int rc; + i64 chunk_id; + i64 chunk_offset; + rc = vec0_get_chunk_position(p, rowid, NULL, &chunk_id, &chunk_offset); + if(rc != SQLITE_OK) { + return rc; + } + sqlite3_blob * blobValue; + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowMetadataChunksNames[metadata_idx], "data", chunk_id, 0, &blobValue); + if(rc != SQLITE_OK) { + return rc; + } + + switch(p->metadata_columns[metadata_idx].kind) { + case VEC0_METADATA_COLUMN_KIND_BOOLEAN: { + u8 block; + rc = sqlite3_blob_read(blobValue, &block, sizeof(block), chunk_offset / CHAR_BIT); + if(rc != SQLITE_OK) { + goto done; + } + int value = block >> ((chunk_offset % CHAR_BIT)) & 1; + sqlite3_result_int(context, value); + break; + } + case VEC0_METADATA_COLUMN_KIND_INTEGER: { + i64 value; + rc = sqlite3_blob_read(blobValue, &value, sizeof(value), chunk_offset * sizeof(i64)); + if(rc != SQLITE_OK) { + goto done; + } + sqlite3_result_int64(context, value); + break; + } + case VEC0_METADATA_COLUMN_KIND_FLOAT: { + double value; + rc = sqlite3_blob_read(blobValue, &value, sizeof(value), chunk_offset * sizeof(double)); + if(rc != SQLITE_OK) { + goto done; + } + sqlite3_result_double(context, value); + break; + } + case VEC0_METADATA_COLUMN_KIND_TEXT: { + u8 view[VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + rc = sqlite3_blob_read(blobValue, &view, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH, chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH); + if(rc != SQLITE_OK) { + goto done; + } + int length = ((int *)view)[0]; + if(length <= VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + sqlite3_result_text(context, (const char*) (view + 4), length, SQLITE_TRANSIENT); + } + else { + sqlite3_stmt * stmt; + const char * zSql = sqlite3_mprintf("SELECT data FROM " VEC0_SHADOW_METADATA_TEXT_DATA_NAME " WHERE rowid = ?", p->schemaName, p->tableName, metadata_idx); + if(!zSql) { + rc = SQLITE_ERROR; + goto done; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free((void *) zSql); + if(rc != SQLITE_OK) { + goto done; + } + sqlite3_bind_int64(stmt, 1, rowid); + rc = sqlite3_step(stmt); + if(rc != SQLITE_ROW) { + sqlite3_finalize(stmt); + rc = SQLITE_ERROR; + goto done; + } + sqlite3_result_value(context, sqlite3_column_value(stmt, 0)); + sqlite3_finalize(stmt); + rc = SQLITE_OK; + } + break; + } + } + done: + // blobValue is read-only, will not fail on close + sqlite3_blob_close(blobValue); + return rc; + +} + +int vec0_get_latest_chunk_rowid(vec0_vtab *p, i64 *chunk_rowid, sqlite3_value ** partitionKeyValues) { + int rc; + const char *zSql; + // lazy initialize stmtLatestChunk when needed. May be cleared during xSync() + if (!p->stmtLatestChunk) { + if(p->numPartitionColumns > 0) { + sqlite3_str * s = sqlite3_str_new(NULL); + sqlite3_str_appendf(s, "SELECT max(rowid) FROM " VEC0_SHADOW_CHUNKS_NAME " WHERE ", + p->schemaName, p->tableName); + + for(int i = 0; i < p->numPartitionColumns; i++) { + if(i != 0) { + sqlite3_str_appendall(s, " AND "); + } + sqlite3_str_appendf(s, " partition%02d = ? ", i); + } + zSql = sqlite3_str_finish(s); + }else { + zSql = sqlite3_mprintf("SELECT max(rowid) FROM " VEC0_SHADOW_CHUNKS_NAME, + p->schemaName, p->tableName); + } + + if (!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtLatestChunk, 0); + sqlite3_free((void *)zSql); + if (rc != SQLITE_OK) { + // IMP: V21406_05476 + vtab_set_error(&p->base, VEC_INTERAL_ERROR + "could not initialize 'latest chunk' statement"); + goto cleanup; + } + } + + for(int i = 0; i < p->numPartitionColumns; i++) { + sqlite3_bind_value(p->stmtLatestChunk, i+1, (partitionKeyValues[i])); + } + + rc = sqlite3_step(p->stmtLatestChunk); + if (rc != SQLITE_ROW) { + // IMP: V31559_15629 + vtab_set_error(&p->base, VEC_INTERAL_ERROR "Could not find latest chunk"); + rc = SQLITE_ERROR; + goto cleanup; + } + if(sqlite3_column_type(p->stmtLatestChunk, 0) == SQLITE_NULL){ + rc = SQLITE_EMPTY; + goto cleanup; + } + *chunk_rowid = sqlite3_column_int64(p->stmtLatestChunk, 0); + rc = sqlite3_step(p->stmtLatestChunk); + if (rc != SQLITE_DONE) { + vtab_set_error(&p->base, + VEC_INTERAL_ERROR + "unknown result code when closing out stmtLatestChunk. " + "Please file an issue: " REPORT_URL, + p->schemaName, p->shadowChunksName); + goto cleanup; + } + rc = SQLITE_OK; + +cleanup: + if (p->stmtLatestChunk) { + sqlite3_reset(p->stmtLatestChunk); + sqlite3_clear_bindings(p->stmtLatestChunk); + } + return rc; +} + +int vec0_rowids_insert_rowid(vec0_vtab *p, i64 rowid) { + int rc = SQLITE_OK; + int entered = 0; + UNUSED_PARAMETER(entered); // temporary + if (!p->stmtRowidsInsertRowid) { + const char *zSql = + sqlite3_mprintf("INSERT INTO " VEC0_SHADOW_ROWIDS_NAME "(rowid)" + "VALUES (?);", + p->schemaName, p->tableName); + if (!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtRowidsInsertRowid, 0); + sqlite3_free((void *)zSql); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, VEC_INTERAL_ERROR + "could not initialize 'insert rowids' statement"); + goto cleanup; + } + } + +#if SQLITE_THREADSAFE + if (sqlite3_mutex_enter) { + sqlite3_mutex_enter(sqlite3_db_mutex(p->db)); + entered = 1; + } +#endif + sqlite3_bind_int64(p->stmtRowidsInsertRowid, 1, rowid); + rc = sqlite3_step(p->stmtRowidsInsertRowid); + + if (rc != SQLITE_DONE) { + if (sqlite3_extended_errcode(p->db) == SQLITE_CONSTRAINT_PRIMARYKEY) { + // IMP: V17090_01160 + vtab_set_error(&p->base, "UNIQUE constraint failed on %s primary key", + p->tableName); + } else { + // IMP: V04679_21517 + vtab_set_error(&p->base, + "Error inserting rowid into rowids shadow table: %s", + sqlite3_errmsg(sqlite3_db_handle(p->stmtRowidsInsertId))); + } + rc = SQLITE_ERROR; + goto cleanup; + } + + rc = SQLITE_OK; + +cleanup: + if (p->stmtRowidsInsertRowid) { + sqlite3_reset(p->stmtRowidsInsertRowid); + sqlite3_clear_bindings(p->stmtRowidsInsertRowid); + } + +#if SQLITE_THREADSAFE + if (sqlite3_mutex_leave && entered) { + sqlite3_mutex_leave(sqlite3_db_mutex(p->db)); + } +#endif + return rc; +} + +int vec0_rowids_insert_id(vec0_vtab *p, sqlite3_value *idValue, i64 *rowid) { + int rc = SQLITE_OK; + int entered = 0; + UNUSED_PARAMETER(entered); // temporary + if (!p->stmtRowidsInsertId) { + const char *zSql = + sqlite3_mprintf("INSERT INTO " VEC0_SHADOW_ROWIDS_NAME "(id)" + "VALUES (?);", + p->schemaName, p->tableName); + if (!zSql) { + rc = SQLITE_NOMEM; + goto complete; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtRowidsInsertId, 0); + sqlite3_free((void *)zSql); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, VEC_INTERAL_ERROR + "could not initialize 'insert rowids id' statement"); + goto complete; + } + } + +#if SQLITE_THREADSAFE + if (sqlite3_mutex_enter) { + sqlite3_mutex_enter(sqlite3_db_mutex(p->db)); + entered = 1; + } +#endif + + if (idValue) { + sqlite3_bind_value(p->stmtRowidsInsertId, 1, idValue); + } + rc = sqlite3_step(p->stmtRowidsInsertId); + + if (rc != SQLITE_DONE) { + if (sqlite3_extended_errcode(p->db) == SQLITE_CONSTRAINT_UNIQUE) { + // IMP: V20497_04568 + vtab_set_error(&p->base, "UNIQUE constraint failed on %s primary key", + p->tableName); + } else { + // IMP: V24016_08086 + // IMP: V15177_32015 + vtab_set_error(&p->base, + "Error inserting id into rowids shadow table: %s", + sqlite3_errmsg(sqlite3_db_handle(p->stmtRowidsInsertId))); + } + rc = SQLITE_ERROR; + goto complete; + } + + *rowid = sqlite3_last_insert_rowid(p->db); + rc = SQLITE_OK; + +complete: + if (p->stmtRowidsInsertId) { + sqlite3_reset(p->stmtRowidsInsertId); + sqlite3_clear_bindings(p->stmtRowidsInsertId); + } + +#if SQLITE_THREADSAFE + if (sqlite3_mutex_leave && entered) { + sqlite3_mutex_leave(sqlite3_db_mutex(p->db)); + } +#endif + return rc; +} + +int vec0_metadata_chunk_size(vec0_metadata_column_kind kind, int chunk_size) { + switch(kind) { + case VEC0_METADATA_COLUMN_KIND_BOOLEAN: + return chunk_size / 8; + case VEC0_METADATA_COLUMN_KIND_INTEGER: + return chunk_size * sizeof(i64); + case VEC0_METADATA_COLUMN_KIND_FLOAT: + return chunk_size * sizeof(double); + case VEC0_METADATA_COLUMN_KIND_TEXT: + return chunk_size * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH; + } + return 0; +} + +int vec0_rowids_update_position(vec0_vtab *p, i64 rowid, i64 chunk_rowid, + i64 chunk_offset) { + int rc = SQLITE_OK; + + if (!p->stmtRowidsUpdatePosition) { + const char *zSql = sqlite3_mprintf(" UPDATE " VEC0_SHADOW_ROWIDS_NAME + " SET chunk_id = ?, chunk_offset = ?" + " WHERE rowid = ?", + p->schemaName, p->tableName); + if (!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->stmtRowidsUpdatePosition, 0); + sqlite3_free((void *)zSql); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, VEC_INTERAL_ERROR + "could not initialize 'update rowids position' statement"); + goto cleanup; + } + } + + sqlite3_bind_int64(p->stmtRowidsUpdatePosition, 1, chunk_rowid); + sqlite3_bind_int64(p->stmtRowidsUpdatePosition, 2, chunk_offset); + sqlite3_bind_int64(p->stmtRowidsUpdatePosition, 3, rowid); + + rc = sqlite3_step(p->stmtRowidsUpdatePosition); + if (rc != SQLITE_DONE) { + // IMP: V21925_05995 + vtab_set_error(&p->base, + VEC_INTERAL_ERROR + "could not update rowids position for rowid=%lld, " + "chunk_rowid=%lld, chunk_offset=%lld", + rowid, chunk_rowid, chunk_offset); + rc = SQLITE_ERROR; + goto cleanup; + } + rc = SQLITE_OK; + +cleanup: + if (p->stmtRowidsUpdatePosition) { + sqlite3_reset(p->stmtRowidsUpdatePosition); + sqlite3_clear_bindings(p->stmtRowidsUpdatePosition); + } + + return rc; +} + +/** + * @brief Adds a new chunk for the vec0 table, and the corresponding vector + * chunks. + * + * Inserts a new row into the _chunks table, with blank data, and uses that new + * rowid to insert new blank rows into _vector_chunksXX tables. + * + * @param p: vec0 table to add new chunk + * @param paritionKeyValues: Array of partition key valeus for the new chunk, if available + * @param chunk_rowid: Output pointer, if not NULL, then will be filled with the + * new chunk rowid. + * @return int SQLITE_OK on success, error code otherwise. + */ +int vec0_new_chunk(vec0_vtab *p, sqlite3_value ** partitionKeyValues, i64 *chunk_rowid) { + int rc; + char *zSql; + sqlite3_stmt *stmt; + i64 rowid; + + // Step 1: Insert a new row in _chunks, capture that new rowid + if(p->numPartitionColumns > 0) { + sqlite3_str * s = sqlite3_str_new(NULL); + sqlite3_str_appendf(s, "INSERT INTO " VEC0_SHADOW_CHUNKS_NAME, p->schemaName, p->tableName); + sqlite3_str_appendall(s, "(size, validity, rowids"); + for(int i = 0; i < p->numPartitionColumns; i++) { + sqlite3_str_appendf(s, ", partition%02d", i); + } + sqlite3_str_appendall(s, ") VALUES (?, ?, ?"); + for(int i = 0; i < p->numPartitionColumns; i++) { + sqlite3_str_appendall(s, ", ?"); + } + sqlite3_str_appendall(s, ")"); + + zSql = sqlite3_str_finish(s); + }else { + zSql = sqlite3_mprintf("INSERT INTO " VEC0_SHADOW_CHUNKS_NAME + "(size, validity, rowids) " + "VALUES (?, ?, ?);", + p->schemaName, p->tableName); + } + + if (!zSql) { + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + sqlite3_finalize(stmt); + return rc; + } + +#if SQLITE_THREADSAFE + if (sqlite3_mutex_enter) { + sqlite3_mutex_enter(sqlite3_db_mutex(p->db)); + } +#endif + + sqlite3_bind_int64(stmt, 1, p->chunk_size); // size + sqlite3_bind_zeroblob(stmt, 2, p->chunk_size / CHAR_BIT); // validity bitmap + sqlite3_bind_zeroblob(stmt, 3, p->chunk_size * sizeof(i64)); // rowids + + for(int i = 0; i < p->numPartitionColumns; i++) { + sqlite3_bind_value(stmt, 4 + i, partitionKeyValues[i]); + } + + rc = sqlite3_step(stmt); + int failed = rc != SQLITE_DONE; + rowid = sqlite3_last_insert_rowid(p->db); +#if SQLITE_THREADSAFE + if (sqlite3_mutex_leave) { + sqlite3_mutex_leave(sqlite3_db_mutex(p->db)); + } +#endif + sqlite3_finalize(stmt); + if (failed) { + return SQLITE_ERROR; + } + + // Step 2: Create new vector chunks for each vector column, with + // that new chunk_rowid. + + for (int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_VECTOR) { + continue; + } + int vector_column_idx = p->user_column_idxs[i]; + i64 vectorsSize = + p->chunk_size * vector_column_byte_size(p->vector_columns[vector_column_idx]); + + zSql = sqlite3_mprintf("INSERT INTO " VEC0_SHADOW_VECTOR_N_NAME + "(rowid, vectors)" + "VALUES (?, ?)", + p->schemaName, p->tableName, vector_column_idx); + if (!zSql) { + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + + if (rc != SQLITE_OK) { + sqlite3_finalize(stmt); + return rc; + } + + sqlite3_bind_int64(stmt, 1, rowid); + sqlite3_bind_zeroblob64(stmt, 2, vectorsSize); + + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { + return rc; + } + } + + // Step 3: Create new metadata chunks for each metadata column + for (int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_METADATA) { + continue; + } + int metadata_column_idx = p->user_column_idxs[i]; + zSql = sqlite3_mprintf("INSERT INTO " VEC0_SHADOW_METADATA_N_NAME + "(rowid, data)" + "VALUES (?, ?)", + p->schemaName, p->tableName, metadata_column_idx); + if (!zSql) { + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + + if (rc != SQLITE_OK) { + sqlite3_finalize(stmt); + return rc; + } + + sqlite3_bind_int64(stmt, 1, rowid); + sqlite3_bind_zeroblob64(stmt, 2, vec0_metadata_chunk_size(p->metadata_columns[metadata_column_idx].kind, p->chunk_size)); + + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + if (rc != SQLITE_DONE) { + return rc; + } + } + + + if (chunk_rowid) { + *chunk_rowid = rowid; + } + + return SQLITE_OK; +} + +struct vec0_query_fullscan_data { + sqlite3_stmt *rowids_stmt; + i8 done; +}; +void vec0_query_fullscan_data_clear( + struct vec0_query_fullscan_data *fullscan_data) { + if (!fullscan_data) + return; + + if (fullscan_data->rowids_stmt) { + sqlite3_finalize(fullscan_data->rowids_stmt); + fullscan_data->rowids_stmt = NULL; + } +} + +struct vec0_query_knn_data { + i64 k; + i64 k_used; + // Array of rowids of size k. Must be freed with sqlite3_free(). + i64 *rowids; + // Array of distances of size k. Must be freed with sqlite3_free(). + f32 *distances; + i64 current_idx; +}; +void vec0_query_knn_data_clear(struct vec0_query_knn_data *knn_data) { + if (!knn_data) + return; + + if (knn_data->rowids) { + sqlite3_free(knn_data->rowids); + knn_data->rowids = NULL; + } + if (knn_data->distances) { + sqlite3_free(knn_data->distances); + knn_data->distances = NULL; + } +} + +struct vec0_query_point_data { + i64 rowid; + void *vectors[VEC0_MAX_VECTOR_COLUMNS]; + int done; +}; +void vec0_query_point_data_clear(struct vec0_query_point_data *point_data) { + if (!point_data) + return; + for (int i = 0; i < VEC0_MAX_VECTOR_COLUMNS; i++) { + sqlite3_free(point_data->vectors[i]); + point_data->vectors[i] = NULL; + } +} + +typedef enum { + // If any values are updated, please update the ARCHITECTURE.md docs accordingly! + + VEC0_QUERY_PLAN_FULLSCAN = '1', + VEC0_QUERY_PLAN_POINT = '2', + VEC0_QUERY_PLAN_KNN = '3', +} vec0_query_plan; + +typedef struct vec0_cursor vec0_cursor; +struct vec0_cursor { + sqlite3_vtab_cursor base; + + vec0_query_plan query_plan; + struct vec0_query_fullscan_data *fullscan_data; + struct vec0_query_knn_data *knn_data; + struct vec0_query_point_data *point_data; +}; + +void vec0_cursor_clear(vec0_cursor *pCur) { + if (pCur->fullscan_data) { + vec0_query_fullscan_data_clear(pCur->fullscan_data); + sqlite3_free(pCur->fullscan_data); + pCur->fullscan_data = NULL; + } + if (pCur->knn_data) { + vec0_query_knn_data_clear(pCur->knn_data); + sqlite3_free(pCur->knn_data); + pCur->knn_data = NULL; + } + if (pCur->point_data) { + vec0_query_point_data_clear(pCur->point_data); + sqlite3_free(pCur->point_data); + pCur->point_data = NULL; + } +} + +#define VEC_CONSTRUCTOR_ERROR "vec0 constructor error: " +static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv, + sqlite3_vtab **ppVtab, char **pzErr, bool isCreate) { + UNUSED_PARAMETER(pAux); + vec0_vtab *pNew; + int rc; + const char *zSql; + + pNew = sqlite3_malloc(sizeof(*pNew)); + if (pNew == 0) + return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + + // Declared chunk_size=N for entire table. + // -1 to use the defualt, otherwise will get re-assigned on `chunk_size=N` + // option + int chunk_size = -1; + int numVectorColumns = 0; + int numPartitionColumns = 0; + int numAuxiliaryColumns = 0; + int numMetadataColumns = 0; + int user_column_idx = 0; + + // track if a "primary key" column is defined + char *pkColumnName = NULL; + int pkColumnNameLength; + int pkColumnType = SQLITE_INTEGER; + + for (int i = 3; i < argc; i++) { + struct VectorColumnDefinition vecColumn; + struct Vec0PartitionColumnDefinition partitionColumn; + struct Vec0AuxiliaryColumnDefinition auxColumn; + struct Vec0MetadataColumnDefinition metadataColumn; + char *cName = NULL; + int cNameLength; + int cType; + + // Scenario #1: Constructor argument is a vector column definition, ie `foo float[1024]` + rc = vec0_parse_vector_column(argv[i], strlen(argv[i]), &vecColumn); + if (rc == SQLITE_ERROR) { + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR "could not parse vector column '%s'", argv[i]); + goto error; + } + if (rc == SQLITE_OK) { + if (numVectorColumns >= VEC0_MAX_VECTOR_COLUMNS) { + sqlite3_free(vecColumn.name); + *pzErr = sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR + "Too many provided vector columns, maximum %d", + VEC0_MAX_VECTOR_COLUMNS); + goto error; + } + + if (vecColumn.dimensions > SQLITE_VEC_VEC0_MAX_DIMENSIONS) { + sqlite3_free(vecColumn.name); + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR + "Dimension on vector column too large, provided %lld, maximum %lld", + (i64)vecColumn.dimensions, SQLITE_VEC_VEC0_MAX_DIMENSIONS); + goto error; + } + pNew->user_column_kinds[user_column_idx] = SQLITE_VEC0_USER_COLUMN_KIND_VECTOR; + pNew->user_column_idxs[user_column_idx] = numVectorColumns; + memcpy(&pNew->vector_columns[numVectorColumns], &vecColumn, sizeof(vecColumn)); + numVectorColumns++; + user_column_idx++; + + continue; + } + + // Scenario #2: Constructor argument is a partition key column definition, ie `user_id text partition key` + rc = vec0_parse_partition_key_definition(argv[i], strlen(argv[i]), &cName, + &cNameLength, &cType); + if (rc == SQLITE_OK) { + if (numPartitionColumns >= VEC0_MAX_PARTITION_COLUMNS) { + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR + "More than %d partition key columns were provided", + VEC0_MAX_PARTITION_COLUMNS); + goto error; + } + partitionColumn.type = cType; + partitionColumn.name_length = cNameLength; + partitionColumn.name = sqlite3_mprintf("%.*s", cNameLength, cName); + if(!partitionColumn.name) { + rc = SQLITE_NOMEM; + goto error; + } + + pNew->user_column_kinds[user_column_idx] = SQLITE_VEC0_USER_COLUMN_KIND_PARTITION; + pNew->user_column_idxs[user_column_idx] = numPartitionColumns; + memcpy(&pNew->paritition_columns[numPartitionColumns], &partitionColumn, sizeof(partitionColumn)); + numPartitionColumns++; + user_column_idx++; + continue; + } + + // Scenario #3: Constructor argument is a primary key column definition, ie `article_id text primary key` + rc = vec0_parse_primary_key_definition(argv[i], strlen(argv[i]), &cName, + &cNameLength, &cType); + if (rc == SQLITE_OK) { + if (pkColumnName) { + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR + "More than one primary key definition was provided, vec0 only " + "suports a single primary key column", + argv[i]); + goto error; + } + pkColumnName = cName; + pkColumnNameLength = cNameLength; + pkColumnType = cType; + continue; + } + + // Scenario #4: Constructor argument is a auxiliary column definition, ie `+contents text` + rc = vec0_parse_auxiliary_column_definition(argv[i], strlen(argv[i]), &cName, + &cNameLength, &cType); + if(rc == SQLITE_OK) { + if (numAuxiliaryColumns >= VEC0_MAX_AUXILIARY_COLUMNS) { + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR + "More than %d auxiliary columns were provided", + VEC0_MAX_AUXILIARY_COLUMNS); + goto error; + } + auxColumn.type = cType; + auxColumn.name_length = cNameLength; + auxColumn.name = sqlite3_mprintf("%.*s", cNameLength, cName); + if(!auxColumn.name) { + rc = SQLITE_NOMEM; + goto error; + } + + pNew->user_column_kinds[user_column_idx] = SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY; + pNew->user_column_idxs[user_column_idx] = numAuxiliaryColumns; + memcpy(&pNew->auxiliary_columns[numAuxiliaryColumns], &auxColumn, sizeof(auxColumn)); + numAuxiliaryColumns++; + user_column_idx++; + continue; + } + + vec0_metadata_column_kind kind; + rc = vec0_parse_metadata_column_definition(argv[i], strlen(argv[i]), &cName, + &cNameLength, &kind); + if(rc == SQLITE_OK) { + if (numMetadataColumns >= VEC0_MAX_METADATA_COLUMNS) { + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR + "More than %d metadata columns were provided", + VEC0_MAX_METADATA_COLUMNS); + goto error; + } + metadataColumn.kind = kind; + metadataColumn.name_length = cNameLength; + metadataColumn.name = sqlite3_mprintf("%.*s", cNameLength, cName); + if(!metadataColumn.name) { + rc = SQLITE_NOMEM; + goto error; + } + + pNew->user_column_kinds[user_column_idx] = SQLITE_VEC0_USER_COLUMN_KIND_METADATA; + pNew->user_column_idxs[user_column_idx] = numMetadataColumns; + memcpy(&pNew->metadata_columns[numMetadataColumns], &metadataColumn, sizeof(metadataColumn)); + numMetadataColumns++; + user_column_idx++; + continue; + } + + // Scenario #4: Constructor argument is a table-level option, ie `chunk_size` + + char *key; + char *value; + int keyLength, valueLength; + rc = vec0_parse_table_option(argv[i], strlen(argv[i]), &key, &keyLength, + &value, &valueLength); + if (rc == SQLITE_ERROR) { + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR "could not parse table option '%s'", argv[i]); + goto error; + } + if (rc == SQLITE_OK) { + if (sqlite3_strnicmp(key, "chunk_size", keyLength) == 0) { + chunk_size = atoi(value); + if (chunk_size <= 0) { + // IMP: V01931_18769 + *pzErr = + sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR + "chunk_size must be a non-zero positive integer"); + goto error; + } + if ((chunk_size % 8) != 0) { + // IMP: V14110_30948 + *pzErr = sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR + "chunk_size must be divisible by 8"); + goto error; + } +#define SQLITE_VEC_CHUNK_SIZE_MAX 4096 + if (chunk_size > SQLITE_VEC_CHUNK_SIZE_MAX) { + *pzErr = + sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR "chunk_size too large"); + goto error; + } + } else { + // IMP: V27642_11712 + *pzErr = sqlite3_mprintf( + VEC_CONSTRUCTOR_ERROR "Unknown table option: %.*s", keyLength, key); + goto error; + } + continue; + } + + // Scenario #5: Unknown constructor argument + *pzErr = + sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR "Could not parse '%s'", argv[i]); + goto error; + } + + if (chunk_size < 0) { + chunk_size = 1024; + } + + if (numVectorColumns <= 0) { + *pzErr = sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR + "At least one vector column is required"); + goto error; + } + + sqlite3_str *createStr = sqlite3_str_new(NULL); + sqlite3_str_appendall(createStr, "CREATE TABLE x("); + if (pkColumnName) { + sqlite3_str_appendf(createStr, "\"%.*w\" primary key, ", pkColumnNameLength, + pkColumnName); + } else { + sqlite3_str_appendall(createStr, "rowid, "); + } + for (int i = 0; i < numVectorColumns + numPartitionColumns + numAuxiliaryColumns + numMetadataColumns; i++) { + switch(pNew->user_column_kinds[i]) { + case SQLITE_VEC0_USER_COLUMN_KIND_VECTOR: { + int vector_idx = pNew->user_column_idxs[i]; + sqlite3_str_appendf(createStr, "\"%.*w\", ", + pNew->vector_columns[vector_idx].name_length, + pNew->vector_columns[vector_idx].name); + break; + } + case SQLITE_VEC0_USER_COLUMN_KIND_PARTITION: { + int partition_idx = pNew->user_column_idxs[i]; + sqlite3_str_appendf(createStr, "\"%.*w\", ", + pNew->paritition_columns[partition_idx].name_length, + pNew->paritition_columns[partition_idx].name); + break; + } + case SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY: { + int auxiliary_idx = pNew->user_column_idxs[i]; + sqlite3_str_appendf(createStr, "\"%.*w\", ", + pNew->auxiliary_columns[auxiliary_idx].name_length, + pNew->auxiliary_columns[auxiliary_idx].name); + break; + } + case SQLITE_VEC0_USER_COLUMN_KIND_METADATA: { + int metadata_idx = pNew->user_column_idxs[i]; + sqlite3_str_appendf(createStr, "\"%.*w\", ", + pNew->metadata_columns[metadata_idx].name_length, + pNew->metadata_columns[metadata_idx].name); + break; + } + } + + } + sqlite3_str_appendall(createStr, " distance hidden, k hidden) "); + if (pkColumnName) { + sqlite3_str_appendall(createStr, "without rowid "); + } + zSql = sqlite3_str_finish(createStr); + if (!zSql) { + goto error; + } + rc = sqlite3_declare_vtab(db, zSql); + sqlite3_free((void *)zSql); + if (rc != SQLITE_OK) { + *pzErr = sqlite3_mprintf(VEC_CONSTRUCTOR_ERROR + "could not declare virtual table, '%s'", + sqlite3_errmsg(db)); + goto error; + } + + const char *schemaName = argv[1]; + const char *tableName = argv[2]; + + pNew->db = db; + pNew->pkIsText = pkColumnType == SQLITE_TEXT; + pNew->schemaName = sqlite3_mprintf("%s", schemaName); + if (!pNew->schemaName) { + goto error; + } + pNew->tableName = sqlite3_mprintf("%s", tableName); + if (!pNew->tableName) { + goto error; + } + pNew->shadowRowidsName = sqlite3_mprintf("%s_rowids", tableName); + if (!pNew->shadowRowidsName) { + goto error; + } + pNew->shadowChunksName = sqlite3_mprintf("%s_chunks", tableName); + if (!pNew->shadowChunksName) { + goto error; + } + pNew->numVectorColumns = numVectorColumns; + pNew->numPartitionColumns = numPartitionColumns; + pNew->numAuxiliaryColumns = numAuxiliaryColumns; + pNew->numMetadataColumns = numMetadataColumns; + + for (int i = 0; i < pNew->numVectorColumns; i++) { + pNew->shadowVectorChunksNames[i] = + sqlite3_mprintf("%s_vector_chunks%02d", tableName, i); + if (!pNew->shadowVectorChunksNames[i]) { + goto error; + } + } + for (int i = 0; i < pNew->numMetadataColumns; i++) { + pNew->shadowMetadataChunksNames[i] = + sqlite3_mprintf("%s_metadatachunks%02d", tableName, i); + if (!pNew->shadowMetadataChunksNames[i]) { + goto error; + } + } + pNew->chunk_size = chunk_size; + + // if xCreate, then create the necessary shadow tables + if (isCreate) { + sqlite3_stmt *stmt; + int rc; + + char * zCreateInfo = sqlite3_mprintf("CREATE TABLE "VEC0_SHADOW_INFO_NAME " (key text primary key, value any)", pNew->schemaName, pNew->tableName); + if(!zCreateInfo) { + goto error; + } + rc = sqlite3_prepare_v2(db, zCreateInfo, -1, &stmt, NULL); + + sqlite3_free((void *) zCreateInfo); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + // TODO(IMP) + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not create '_info' shadow table: %s", + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + + char * zSeedInfo = sqlite3_mprintf( + "INSERT INTO "VEC0_SHADOW_INFO_NAME "(key, value) VALUES " + "(?1, ?2), (?3, ?4), (?5, ?6), (?7, ?8) ", + pNew->schemaName, pNew->tableName + ); + if(!zSeedInfo) { + goto error; + } + rc = sqlite3_prepare_v2(db, zSeedInfo, -1, &stmt, NULL); + sqlite3_free((void *) zSeedInfo); + if (rc != SQLITE_OK) { + // TODO(IMP) + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not seed '_info' shadow table: %s", + sqlite3_errmsg(db)); + goto error; + } + sqlite3_bind_text(stmt, 1, "CREATE_VERSION", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, SQLITE_VEC_VERSION, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 3, "CREATE_VERSION_MAJOR", -1, SQLITE_STATIC); + sqlite3_bind_int(stmt, 4, SQLITE_VEC_VERSION_MAJOR); + sqlite3_bind_text(stmt, 5, "CREATE_VERSION_MINOR", -1, SQLITE_STATIC); + sqlite3_bind_int(stmt, 6, SQLITE_VEC_VERSION_MINOR); + sqlite3_bind_text(stmt, 7, "CREATE_VERSION_PATCH", -1, SQLITE_STATIC); + sqlite3_bind_int(stmt, 8, SQLITE_VEC_VERSION_PATCH); + + if(sqlite3_step(stmt) != SQLITE_DONE) { + // TODO(IMP) + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not seed '_info' shadow table: %s", + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + + + + // create the _chunks shadow table + char *zCreateShadowChunks = NULL; + if(pNew->numPartitionColumns) { + sqlite3_str * s = sqlite3_str_new(NULL); + sqlite3_str_appendf(s, "CREATE TABLE " VEC0_SHADOW_CHUNKS_NAME "(", pNew->schemaName, pNew->tableName); + sqlite3_str_appendall(s, "chunk_id INTEGER PRIMARY KEY AUTOINCREMENT," "size INTEGER NOT NULL,"); + sqlite3_str_appendall(s, "sequence_id integer,"); + for(int i = 0; i < pNew->numPartitionColumns;i++) { + sqlite3_str_appendf(s, "partition%02d,", i); + } + sqlite3_str_appendall(s, "validity BLOB NOT NULL, rowids BLOB NOT NULL);"); + zCreateShadowChunks = sqlite3_str_finish(s); + }else { + zCreateShadowChunks = sqlite3_mprintf(VEC0_SHADOW_CHUNKS_CREATE, + pNew->schemaName, pNew->tableName); + } + if (!zCreateShadowChunks) { + goto error; + } + rc = sqlite3_prepare_v2(db, zCreateShadowChunks, -1, &stmt, 0); + sqlite3_free((void *)zCreateShadowChunks); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + // IMP: V17740_01811 + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not create '_chunks' shadow table: %s", + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + + // create the _rowids shadow table + char *zCreateShadowRowids; + if (pNew->pkIsText) { + // adds a "text unique not null" constraint to the id column + zCreateShadowRowids = sqlite3_mprintf(VEC0_SHADOW_ROWIDS_CREATE_PK_TEXT, + pNew->schemaName, pNew->tableName); + } else { + zCreateShadowRowids = sqlite3_mprintf(VEC0_SHADOW_ROWIDS_CREATE_BASIC, + pNew->schemaName, pNew->tableName); + } + if (!zCreateShadowRowids) { + goto error; + } + rc = sqlite3_prepare_v2(db, zCreateShadowRowids, -1, &stmt, 0); + sqlite3_free((void *)zCreateShadowRowids); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + // IMP: V11631_28470 + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not create '_rowids' shadow table: %s", + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + + for (int i = 0; i < pNew->numVectorColumns; i++) { + char *zSql = sqlite3_mprintf(VEC0_SHADOW_VECTOR_N_CREATE, + pNew->schemaName, pNew->tableName, i); + if (!zSql) { + goto error; + } + rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + // IMP: V25919_09989 + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf( + "Could not create '_vector_chunks%02d' shadow table: %s", i, + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + } + + for (int i = 0; i < pNew->numMetadataColumns; i++) { + char *zSql = sqlite3_mprintf("CREATE TABLE " VEC0_SHADOW_METADATA_N_NAME "(rowid PRIMARY KEY, data BLOB NOT NULL);", + pNew->schemaName, pNew->tableName, i); + if (!zSql) { + goto error; + } + rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf( + "Could not create '_metata_chunks%02d' shadow table: %s", i, + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + + if(pNew->metadata_columns[i].kind == VEC0_METADATA_COLUMN_KIND_TEXT) { + char *zSql = sqlite3_mprintf("CREATE TABLE " VEC0_SHADOW_METADATA_TEXT_DATA_NAME "(rowid PRIMARY KEY, data TEXT);", + pNew->schemaName, pNew->tableName, i); + if (!zSql) { + goto error; + } + rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf( + "Could not create '_metadatatext%02d' shadow table: %s", i, + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + + } + } + + if(pNew->numAuxiliaryColumns > 0) { + sqlite3_stmt * stmt; + sqlite3_str * s = sqlite3_str_new(NULL); + sqlite3_str_appendf(s, "CREATE TABLE " VEC0_SHADOW_AUXILIARY_NAME "( rowid integer PRIMARY KEY ", pNew->schemaName, pNew->tableName); + for(int i = 0; i < pNew->numAuxiliaryColumns; i++) { + sqlite3_str_appendf(s, ", value%02d", i); + } + sqlite3_str_appendall(s, ")"); + char *zSql = sqlite3_str_finish(s); + if(!zSql) { + goto error; + } + rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, NULL); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf( + "Could not create auxiliary shadow table: %s", + sqlite3_errmsg(db)); + + goto error; + } + sqlite3_finalize(stmt); + } + } + + *ppVtab = (sqlite3_vtab *)pNew; + return SQLITE_OK; + +error: + vec0_free(pNew); + return SQLITE_ERROR; +} + +static int vec0Create(sqlite3 *db, void *pAux, int argc, + const char *const *argv, sqlite3_vtab **ppVtab, + char **pzErr) { + return vec0_init(db, pAux, argc, argv, ppVtab, pzErr, true); +} +static int vec0Connect(sqlite3 *db, void *pAux, int argc, + const char *const *argv, sqlite3_vtab **ppVtab, + char **pzErr) { + return vec0_init(db, pAux, argc, argv, ppVtab, pzErr, false); +} + +static int vec0Disconnect(sqlite3_vtab *pVtab) { + vec0_vtab *p = (vec0_vtab *)pVtab; + vec0_free(p); + sqlite3_free(p); + return SQLITE_OK; +} +static int vec0Destroy(sqlite3_vtab *pVtab) { + vec0_vtab *p = (vec0_vtab *)pVtab; + sqlite3_stmt *stmt; + int rc; + const char *zSql; + + // Free up any sqlite3_stmt, otherwise DROPs on those tables will fail + vec0_free_resources(p); + + // TODO(test) later: can't evidence-of here, bc always gives "SQL logic error" instead of + // provided error + zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_CHUNKS_NAME, p->schemaName, + p->tableName); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + rc = SQLITE_ERROR; + vtab_set_error(pVtab, "could not drop chunks shadow table"); + goto done; + } + sqlite3_finalize(stmt); + + zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_INFO_NAME, p->schemaName, + p->tableName); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + rc = SQLITE_ERROR; + vtab_set_error(pVtab, "could not drop info shadow table"); + goto done; + } + sqlite3_finalize(stmt); + + zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_ROWIDS_NAME, p->schemaName, + p->tableName); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + rc = SQLITE_ERROR; + goto done; + } + sqlite3_finalize(stmt); + + for (int i = 0; i < p->numVectorColumns; i++) { + zSql = sqlite3_mprintf("DROP TABLE \"%w\".\"%w\"", p->schemaName, + p->shadowVectorChunksNames[i]); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + rc = SQLITE_ERROR; + goto done; + } + sqlite3_finalize(stmt); + } + + if(p->numAuxiliaryColumns > 0) { + zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_AUXILIARY_NAME, p->schemaName, p->tableName); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + rc = SQLITE_ERROR; + goto done; + } + sqlite3_finalize(stmt); + } + + + for (int i = 0; i < p->numMetadataColumns; i++) { + zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_METADATA_N_NAME, p->schemaName,p->tableName, i); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + rc = SQLITE_ERROR; + goto done; + } + sqlite3_finalize(stmt); + + if(p->metadata_columns[i].kind == VEC0_METADATA_COLUMN_KIND_TEXT) { + zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_METADATA_TEXT_DATA_NAME, p->schemaName,p->tableName, i); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + rc = SQLITE_ERROR; + goto done; + } + sqlite3_finalize(stmt); + } + } + + stmt = NULL; + rc = SQLITE_OK; + +done: + sqlite3_finalize(stmt); + vec0_free(p); + // If there was an error + if (rc == SQLITE_OK) { + sqlite3_free(p); + } + return rc; +} + +static int vec0Open(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor) { + UNUSED_PARAMETER(p); + vec0_cursor *pCur; + pCur = sqlite3_malloc(sizeof(*pCur)); + if (pCur == 0) + return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +static int vec0Close(sqlite3_vtab_cursor *cur) { + vec0_cursor *pCur = (vec0_cursor *)cur; + vec0_cursor_clear(pCur); + sqlite3_free(pCur); + return SQLITE_OK; +} + +// All the different type of "values" provided to argv/argc in vec0Filter. +// These enums denote the use and purpose of all of them. +typedef enum { + // If any values are updated, please update the ARCHITECTURE.md docs accordingly! + + VEC0_IDXSTR_KIND_KNN_MATCH = '{', + VEC0_IDXSTR_KIND_KNN_K = '}', + VEC0_IDXSTR_KIND_KNN_ROWID_IN = '[', + VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT = ']', + VEC0_IDXSTR_KIND_POINT_ID = '!', + VEC0_IDXSTR_KIND_METADATA_CONSTRAINT = '&', +} vec0_idxstr_kind; + +// The different SQLITE_INDEX_CONSTRAINT values that vec0 partition key columns +// support, but as characters that fit nicely in idxstr. +typedef enum { + // If any values are updated, please update the ARCHITECTURE.md docs accordingly! + + VEC0_PARTITION_OPERATOR_EQ = 'a', + VEC0_PARTITION_OPERATOR_GT = 'b', + VEC0_PARTITION_OPERATOR_LE = 'c', + VEC0_PARTITION_OPERATOR_LT = 'd', + VEC0_PARTITION_OPERATOR_GE = 'e', + VEC0_PARTITION_OPERATOR_NE = 'f', +} vec0_partition_operator; +typedef enum { + VEC0_METADATA_OPERATOR_EQ = 'a', + VEC0_METADATA_OPERATOR_GT = 'b', + VEC0_METADATA_OPERATOR_LE = 'c', + VEC0_METADATA_OPERATOR_LT = 'd', + VEC0_METADATA_OPERATOR_GE = 'e', + VEC0_METADATA_OPERATOR_NE = 'f', + VEC0_METADATA_OPERATOR_IN = 'g', +} vec0_metadata_operator; + +static int vec0BestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pIdxInfo) { + vec0_vtab *p = (vec0_vtab *)pVTab; + /** + * Possible query plans are: + * 1. KNN when: + * a) An `MATCH` op on vector column + * b) ORDER BY on distance column + * c) LIMIT + * d) rowid in (...) OPTIONAL + * 2. Point when: + * a) An `EQ` op on rowid column + * 3. else: fullscan + * + */ + int iMatchTerm = -1; + int iMatchVectorTerm = -1; + int iLimitTerm = -1; + int iRowidTerm = -1; + int iKTerm = -1; + int iRowidInTerm = -1; + int hasAuxConstraint = 0; + +#ifdef SQLITE_VEC_DEBUG + printf("pIdxInfo->nOrderBy=%d, pIdxInfo->nConstraint=%d\n", pIdxInfo->nOrderBy, pIdxInfo->nConstraint); +#endif + + for (int i = 0; i < pIdxInfo->nConstraint; i++) { + u8 vtabIn = 0; + +#if COMPILER_SUPPORTS_VTAB_IN + if (sqlite3_libversion_number() >= 3038000) { + vtabIn = sqlite3_vtab_in(pIdxInfo, i, -1); + } +#endif + +#ifdef SQLITE_VEC_DEBUG + printf("xBestIndex [%d] usable=%d iColumn=%d op=%d vtabin=%d\n", i, + pIdxInfo->aConstraint[i].usable, pIdxInfo->aConstraint[i].iColumn, + pIdxInfo->aConstraint[i].op, vtabIn); +#endif + if (!pIdxInfo->aConstraint[i].usable) + continue; + + int iColumn = pIdxInfo->aConstraint[i].iColumn; + int op = pIdxInfo->aConstraint[i].op; + + if (op == SQLITE_INDEX_CONSTRAINT_LIMIT) { + iLimitTerm = i; + } + if (op == SQLITE_INDEX_CONSTRAINT_MATCH && + vec0_column_idx_is_vector(p, iColumn)) { + if (iMatchTerm > -1) { + vtab_set_error( + pVTab, "only 1 MATCH operator is allowed in a single vec0 query"); + return SQLITE_ERROR; + } + iMatchTerm = i; + iMatchVectorTerm = vec0_column_idx_to_vector_idx(p, iColumn); + } + if (op == SQLITE_INDEX_CONSTRAINT_EQ && iColumn == VEC0_COLUMN_ID) { + if (vtabIn) { + if (iRowidInTerm != -1) { + vtab_set_error(pVTab, "only 1 'rowid in (..)' operator is allowed in " + "a single vec0 query"); + return SQLITE_ERROR; + } + iRowidInTerm = i; + + } else { + iRowidTerm = i; + } + } + if (op == SQLITE_INDEX_CONSTRAINT_EQ && iColumn == vec0_column_k_idx(p)) { + iKTerm = i; + } + if( + (op != SQLITE_INDEX_CONSTRAINT_LIMIT && op != SQLITE_INDEX_CONSTRAINT_OFFSET) + && vec0_column_idx_is_auxiliary(p, iColumn)) { + hasAuxConstraint = 1; + } + } + + sqlite3_str *idxStr = sqlite3_str_new(NULL); + int rc; + + if (iMatchTerm >= 0) { + if (iLimitTerm < 0 && iKTerm < 0) { + vtab_set_error( + pVTab, + "A LIMIT or 'k = ?' constraint is required on vec0 knn queries."); + rc = SQLITE_ERROR; + goto done; + } + if (iLimitTerm >= 0 && iKTerm >= 0) { + vtab_set_error(pVTab, "Only LIMIT or 'k =?' can be provided, not both"); + rc = SQLITE_ERROR; + goto done; + } + + if (pIdxInfo->nOrderBy) { + if (pIdxInfo->nOrderBy > 1) { + vtab_set_error(pVTab, "Only a single 'ORDER BY distance' clause is " + "allowed on vec0 KNN queries"); + rc = SQLITE_ERROR; + goto done; + } + if (pIdxInfo->aOrderBy[0].iColumn != vec0_column_distance_idx(p)) { + vtab_set_error(pVTab, + "Only a single 'ORDER BY distance' clause is allowed on " + "vec0 KNN queries, not on other columns"); + rc = SQLITE_ERROR; + goto done; + } + if (pIdxInfo->aOrderBy[0].desc) { + vtab_set_error( + pVTab, "Only ascending in ORDER BY distance clause is supported, " + "DESC is not supported yet."); + rc = SQLITE_ERROR; + goto done; + } + } + + if(hasAuxConstraint) { + // IMP: V25623_09693 + vtab_set_error(pVTab, "An illegal WHERE constraint was provided on a vec0 auxiliary column in a KNN query."); + rc = SQLITE_ERROR; + goto done; + } + + sqlite3_str_appendchar(idxStr, 1, VEC0_QUERY_PLAN_KNN); + + int argvIndex = 1; + pIdxInfo->aConstraintUsage[iMatchTerm].argvIndex = argvIndex++; + pIdxInfo->aConstraintUsage[iMatchTerm].omit = 1; + sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_KNN_MATCH); + sqlite3_str_appendchar(idxStr, 3, '_'); + + if (iLimitTerm >= 0) { + pIdxInfo->aConstraintUsage[iLimitTerm].argvIndex = argvIndex++; + pIdxInfo->aConstraintUsage[iLimitTerm].omit = 1; + } else { + pIdxInfo->aConstraintUsage[iKTerm].argvIndex = argvIndex++; + pIdxInfo->aConstraintUsage[iKTerm].omit = 1; + } + sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_KNN_K); + sqlite3_str_appendchar(idxStr, 3, '_'); + +#if COMPILER_SUPPORTS_VTAB_IN + if (iRowidInTerm >= 0) { + // already validated as >= SQLite 3.38 bc iRowidInTerm is only >= 0 when + // vtabIn == 1 + sqlite3_vtab_in(pIdxInfo, iRowidInTerm, 1); + pIdxInfo->aConstraintUsage[iRowidInTerm].argvIndex = argvIndex++; + pIdxInfo->aConstraintUsage[iRowidInTerm].omit = 1; + sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_KNN_ROWID_IN); + sqlite3_str_appendchar(idxStr, 3, '_'); + } +#endif + + for (int i = 0; i < pIdxInfo->nConstraint; i++) { + if (!pIdxInfo->aConstraint[i].usable) + continue; + + int iColumn = pIdxInfo->aConstraint[i].iColumn; + int op = pIdxInfo->aConstraint[i].op; + if(op == SQLITE_INDEX_CONSTRAINT_LIMIT || op == SQLITE_INDEX_CONSTRAINT_OFFSET) { + continue; + } + if(!vec0_column_idx_is_partition(p, iColumn)) { + continue; + } + + int partition_idx = vec0_column_idx_to_partition_idx(p, iColumn); + char value = 0; + + switch(op) { + case SQLITE_INDEX_CONSTRAINT_EQ: { + value = VEC0_PARTITION_OPERATOR_EQ; + break; + } + case SQLITE_INDEX_CONSTRAINT_GT: { + value = VEC0_PARTITION_OPERATOR_GT; + break; + } + case SQLITE_INDEX_CONSTRAINT_LE: { + value = VEC0_PARTITION_OPERATOR_LE; + break; + } + case SQLITE_INDEX_CONSTRAINT_LT: { + value = VEC0_PARTITION_OPERATOR_LT; + break; + } + case SQLITE_INDEX_CONSTRAINT_GE: { + value = VEC0_PARTITION_OPERATOR_GE; + break; + } + case SQLITE_INDEX_CONSTRAINT_NE: { + value = VEC0_PARTITION_OPERATOR_NE; + break; + } + } + + if(value) { + pIdxInfo->aConstraintUsage[i].argvIndex = argvIndex++; + pIdxInfo->aConstraintUsage[i].omit = 1; + sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT); + sqlite3_str_appendchar(idxStr, 1, 'A' + partition_idx); + sqlite3_str_appendchar(idxStr, 1, value); + sqlite3_str_appendchar(idxStr, 1, '_'); + } + + } + + for (int i = 0; i < pIdxInfo->nConstraint; i++) { + if (!pIdxInfo->aConstraint[i].usable) + continue; + + int iColumn = pIdxInfo->aConstraint[i].iColumn; + int op = pIdxInfo->aConstraint[i].op; + if(op == SQLITE_INDEX_CONSTRAINT_LIMIT || op == SQLITE_INDEX_CONSTRAINT_OFFSET) { + continue; + } + if(!vec0_column_idx_is_metadata(p, iColumn)) { + continue; + } + + int metadata_idx = vec0_column_idx_to_metadata_idx(p, iColumn); + char value = 0; + + switch(op) { + case SQLITE_INDEX_CONSTRAINT_EQ: { + int vtabIn = 0; + #if COMPILER_SUPPORTS_VTAB_IN + if (sqlite3_libversion_number() >= 3038000) { + vtabIn = sqlite3_vtab_in(pIdxInfo, i, -1); + } + if(vtabIn) { + switch(p->metadata_columns[metadata_idx].kind) { + case VEC0_METADATA_COLUMN_KIND_FLOAT: + case VEC0_METADATA_COLUMN_KIND_BOOLEAN: { + // IMP: V15248_32086 + rc = SQLITE_ERROR; + vtab_set_error(pVTab, "'xxx in (...)' is only available on INTEGER or TEXT metadata columns."); + goto done; + break; + } + case VEC0_METADATA_COLUMN_KIND_INTEGER: + case VEC0_METADATA_COLUMN_KIND_TEXT: { + break; + } + } + value = VEC0_METADATA_OPERATOR_IN; + sqlite3_vtab_in(pIdxInfo, i, 1); + }else + #endif + { + value = VEC0_PARTITION_OPERATOR_EQ; + } + break; + } + case SQLITE_INDEX_CONSTRAINT_GT: { + value = VEC0_METADATA_OPERATOR_GT; + break; + } + case SQLITE_INDEX_CONSTRAINT_LE: { + value = VEC0_METADATA_OPERATOR_LE; + break; + } + case SQLITE_INDEX_CONSTRAINT_LT: { + value = VEC0_METADATA_OPERATOR_LT; + break; + } + case SQLITE_INDEX_CONSTRAINT_GE: { + value = VEC0_METADATA_OPERATOR_GE; + break; + } + case SQLITE_INDEX_CONSTRAINT_NE: { + value = VEC0_METADATA_OPERATOR_NE; + break; + } + default: { + // IMP: V16511_00582 + rc = SQLITE_ERROR; + vtab_set_error(pVTab, + "An illegal WHERE constraint was provided on a vec0 metadata column in a KNN query. " + "Only one of EQUALS, GREATER_THAN, LESS_THAN_OR_EQUAL, LESS_THAN, GREATER_THAN_OR_EQUAL, NOT_EQUALS is allowed." + ); + goto done; + } + } + + if(p->metadata_columns[metadata_idx].kind == VEC0_METADATA_COLUMN_KIND_BOOLEAN) { + if(!(value == VEC0_METADATA_OPERATOR_EQ || value == VEC0_METADATA_OPERATOR_NE)) { + // IMP: V10145_26984 + rc = SQLITE_ERROR; + vtab_set_error(pVTab, "ONLY EQUALS (=) or NOT_EQUALS (!=) operators are allowed on boolean metadata columns."); + goto done; + } + } + + pIdxInfo->aConstraintUsage[i].argvIndex = argvIndex++; + pIdxInfo->aConstraintUsage[i].omit = 1; + sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_METADATA_CONSTRAINT); + sqlite3_str_appendchar(idxStr, 1, 'A' + metadata_idx); + sqlite3_str_appendchar(idxStr, 1, value); + sqlite3_str_appendchar(idxStr, 1, '_'); + + } + + + + pIdxInfo->idxNum = iMatchVectorTerm; + pIdxInfo->estimatedCost = 30.0; + pIdxInfo->estimatedRows = 10; + + } else if (iRowidTerm >= 0) { + sqlite3_str_appendchar(idxStr, 1, VEC0_QUERY_PLAN_POINT); + pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; + sqlite3_str_appendchar(idxStr, 1, VEC0_IDXSTR_KIND_POINT_ID); + sqlite3_str_appendchar(idxStr, 3, '_'); + pIdxInfo->idxNum = pIdxInfo->colUsed; + pIdxInfo->estimatedCost = 10.0; + pIdxInfo->estimatedRows = 1; + } else { + sqlite3_str_appendchar(idxStr, 1, VEC0_QUERY_PLAN_FULLSCAN); + pIdxInfo->estimatedCost = 3000000.0; + pIdxInfo->estimatedRows = 100000; + } + pIdxInfo->idxStr = sqlite3_str_finish(idxStr); + idxStr = NULL; + if (!pIdxInfo->idxStr) { + rc = SQLITE_OK; + goto done; + } + pIdxInfo->needToFreeIdxStr = 1; + + + rc = SQLITE_OK; + + done: + if(idxStr) { + sqlite3_str_finish(idxStr); + } + return rc; +} + +// forward delcaration bc vec0Filter uses it +static int vec0Next(sqlite3_vtab_cursor *cur); + +void merge_sorted_lists(f32 *a, i64 *a_rowids, i64 a_length, f32 *b, + i64 *b_rowids, i32 *b_top_idxs, i64 b_length, f32 *out, + i64 *out_rowids, i64 out_length, i64 *out_used) { + // assert((a_length >= out_length) || (b_length >= out_length)); + i64 ptrA = 0; + i64 ptrB = 0; + for (int i = 0; i < out_length; i++) { + if ((ptrA >= a_length) && (ptrB >= b_length)) { + *out_used = i; + return; + } + if (ptrA >= a_length) { + out[i] = b[b_top_idxs[ptrB]]; + out_rowids[i] = b_rowids[b_top_idxs[ptrB]]; + ptrB++; + } else if (ptrB >= b_length) { + out[i] = a[ptrA]; + out_rowids[i] = a_rowids[ptrA]; + ptrA++; + } else { + if (a[ptrA] <= b[b_top_idxs[ptrB]]) { + out[i] = a[ptrA]; + out_rowids[i] = a_rowids[ptrA]; + ptrA++; + } else { + out[i] = b[b_top_idxs[ptrB]]; + out_rowids[i] = b_rowids[b_top_idxs[ptrB]]; + ptrB++; + } + } + } + + *out_used = out_length; +} + +u8 *bitmap_new(i32 n) { + assert(n % 8 == 0); + u8 *p = sqlite3_malloc(n * sizeof(u8) / CHAR_BIT); + if (p) { + memset(p, 0, n * sizeof(u8) / CHAR_BIT); + } + return p; +} +u8 *bitmap_new_from(i32 n, u8 *from) { + assert(n % 8 == 0); + u8 *p = sqlite3_malloc(n * sizeof(u8) / CHAR_BIT); + if (p) { + memcpy(p, from, n / CHAR_BIT); + } + return p; +} + +void bitmap_copy(u8 *base, u8 *from, i32 n) { + assert(n % 8 == 0); + memcpy(base, from, n / CHAR_BIT); +} + +void bitmap_and_inplace(u8 *base, u8 *other, i32 n) { + assert((n % 8) == 0); + for (int i = 0; i < n / CHAR_BIT; i++) { + base[i] = base[i] & other[i]; + } +} + +void bitmap_set(u8 *bitmap, i32 position, int value) { + if (value) { + bitmap[position / CHAR_BIT] |= 1 << (position % CHAR_BIT); + } else { + bitmap[position / CHAR_BIT] &= ~(1 << (position % CHAR_BIT)); + } +} + +int bitmap_get(u8 *bitmap, i32 position) { + return (((bitmap[position / CHAR_BIT]) >> (position % CHAR_BIT)) & 1); +} + +void bitmap_clear(u8 *bitmap, i32 n) { + assert((n % 8) == 0); + memset(bitmap, 0, n / CHAR_BIT); +} + +void bitmap_fill(u8 *bitmap, i32 n) { + assert((n % 8) == 0); + memset(bitmap, 0xFF, n / CHAR_BIT); +} + +/** + * @brief Finds the minimum k items in distances, and writes the indicies to + * out. + * + * @param distances input f32 array of size n, the items to consider. + * @param n: size of distances array. + * @param out: Output array of size k, will contain at most k element indicies + * @param k: Size of output array + * @return int + */ +int min_idx(const f32 *distances, i32 n, u8 *candidates, i32 *out, i32 k, + u8 *bTaken, i32 *k_used) { + assert(k > 0); + assert(k <= n); + + bitmap_clear(bTaken, n); + + for (int ik = 0; ik < k; ik++) { + int min_idx = 0; + while (min_idx < n && + (bitmap_get(bTaken, min_idx) || !bitmap_get(candidates, min_idx))) { + min_idx++; + } + if (min_idx >= n) { + *k_used = ik; + return SQLITE_OK; + } + + for (int i = 0; i < n; i++) { + if (distances[i] <= distances[min_idx] && !bitmap_get(bTaken, i) && + (bitmap_get(candidates, i))) { + min_idx = i; + } + } + + out[ik] = min_idx; + bitmap_set(bTaken, min_idx, 1); + } + *k_used = k; + return SQLITE_OK; +} + +int vec0_get_metadata_text_long_value( + vec0_vtab * p, + sqlite3_stmt ** stmt, + int metadata_idx, + i64 rowid, + int *n, + char ** s) { + int rc; + if(!(*stmt)) { + const char * zSql = sqlite3_mprintf("select data from " VEC0_SHADOW_METADATA_TEXT_DATA_NAME " where rowid = ?", p->schemaName, p->tableName, metadata_idx); + if(!zSql) { + rc = SQLITE_NOMEM; + goto done; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, stmt, NULL); + sqlite3_free( (void *) zSql); + if(rc != SQLITE_OK) { + goto done; + } + } + + sqlite3_reset(*stmt); + sqlite3_bind_int64(*stmt, 1, rowid); + rc = sqlite3_step(*stmt); + if(rc != SQLITE_ROW) { + rc = SQLITE_ERROR; + goto done; + } + *s = (char *) sqlite3_column_text(*stmt, 0); + *n = sqlite3_column_bytes(*stmt, 0); + rc = SQLITE_OK; + done: + return rc; +} + +/** + * @brief Crete at "iterator" (sqlite3_stmt) of chunks with the given constraints + * + * Any VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT values in idxStr/argv will be applied + * as WHERE constraints in the underlying stmt SQL, and any consumer of the stmt + * can freely step through the stmt with all constraints satisfied. + * + * @param p - vec0_vtab + * @param idxStr - the xBestIndex/xFilter idxstr containing VEC0_IDXSTR values + * @param argc - number of argv values from xFilter + * @param argv - array of sqlite3_value from xFilter + * @param outStmt - output sqlite3_stmt of chunks with all filters applied + * @return int SQLITE_OK on success, error code otherwise + */ +int vec0_chunks_iter(vec0_vtab * p, const char * idxStr, int argc, sqlite3_value ** argv, sqlite3_stmt** outStmt) { + // always null terminated, enforced by SQLite + int idxStrLength = strlen(idxStr); + // "1" refers to the initial vec0_query_plan char, 4 is the number of chars per "element" + int numValueEntries = (idxStrLength-1) / 4; + assert(argc == numValueEntries); + + int rc; + sqlite3_str * s = sqlite3_str_new(NULL); + sqlite3_str_appendf(s, "select chunk_id, validity, rowids " + " from " VEC0_SHADOW_CHUNKS_NAME, + p->schemaName, p->tableName); + + int appendedWhere = 0; + for(int i = 0; i < numValueEntries; i++) { + int idx = 1 + (i * 4); + char kind = idxStr[idx + 0]; + if(kind != VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT) { + continue; + } + + int partition_idx = idxStr[idx + 1] - 'A'; + int operator = idxStr[idx + 2]; + // idxStr[idx + 3] is just null, a '_' placeholder + + if(!appendedWhere) { + sqlite3_str_appendall(s, " WHERE "); + appendedWhere = 1; + }else { + sqlite3_str_appendall(s, " AND "); + } + switch(operator) { + case VEC0_PARTITION_OPERATOR_EQ: + sqlite3_str_appendf(s, " partition%02d = ? ", partition_idx); + break; + case VEC0_PARTITION_OPERATOR_GT: + sqlite3_str_appendf(s, " partition%02d > ? ", partition_idx); + break; + case VEC0_PARTITION_OPERATOR_LE: + sqlite3_str_appendf(s, " partition%02d <= ? ", partition_idx); + break; + case VEC0_PARTITION_OPERATOR_LT: + sqlite3_str_appendf(s, " partition%02d < ? ", partition_idx); + break; + case VEC0_PARTITION_OPERATOR_GE: + sqlite3_str_appendf(s, " partition%02d >= ? ", partition_idx); + break; + case VEC0_PARTITION_OPERATOR_NE: + sqlite3_str_appendf(s, " partition%02d != ? ", partition_idx); + break; + default: { + char * zSql = sqlite3_str_finish(s); + sqlite3_free(zSql); + return SQLITE_ERROR; + } + + } + + } + + char *zSql = sqlite3_str_finish(s); + if (!zSql) { + return SQLITE_NOMEM; + } + + rc = sqlite3_prepare_v2(p->db, zSql, -1, outStmt, NULL); + sqlite3_free(zSql); + if(rc != SQLITE_OK) { + return rc; + } + + int n = 1; + for(int i = 0; i < numValueEntries; i++) { + int idx = 1 + (i * 4); + char kind = idxStr[idx + 0]; + if(kind != VEC0_IDXSTR_KIND_KNN_PARTITON_CONSTRAINT) { + continue; + } + sqlite3_bind_value(*outStmt, n++, argv[i]); + } + + return rc; +} + +// a single `xxx in (...)` constraint on a metadata column. TEXT or INTEGER only for now. +struct Vec0MetadataIn{ + // index of argv[i]` the constraint is on + int argv_idx; + // metadata column index of the constraint, derived from idxStr + argv_idx + int metadata_idx; + // array of the copied `(...)` values from sqlite3_vtab_in_first()/sqlite3_vtab_in_next() + struct Array array; +}; + +// Array elements for `xxx in (...)` values for a text column. basically just a string +struct Vec0MetadataInTextEntry { + int n; + char * zString; +}; + + +int vec0_metadata_filter_text(vec0_vtab * p, sqlite3_value * value, const void * buffer, int size, vec0_metadata_operator op, u8* b, int metadata_idx, int chunk_rowid, struct Array * aMetadataIn, int argv_idx) { + int rc; + sqlite3_stmt * stmt = NULL; + i64 * rowids = NULL; + sqlite3_blob * rowidsBlob; + const char * sTarget = (const char *) sqlite3_value_text(value); + int nTarget = sqlite3_value_bytes(value); + + + // TODO(perf): only text metadata news the rowids BLOB. Make it so that + // rowids BLOB is re-used when multiple fitlers on text columns, + // ex "name BETWEEN 'a' and 'b'"" + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, "rowids", chunk_rowid, 0, &rowidsBlob); + if(rc != SQLITE_OK) { + return rc; + } + assert(sqlite3_blob_bytes(rowidsBlob) % sizeof(i64) == 0); + assert((sqlite3_blob_bytes(rowidsBlob) / sizeof(i64)) == size); + + rowids = sqlite3_malloc(sqlite3_blob_bytes(rowidsBlob)); + if(!rowids) { + sqlite3_blob_close(rowidsBlob); + return SQLITE_NOMEM; + } + + rc = sqlite3_blob_read(rowidsBlob, rowids, sqlite3_blob_bytes(rowidsBlob), 0); + if(rc != SQLITE_OK) { + sqlite3_blob_close(rowidsBlob); + return rc; + } + sqlite3_blob_close(rowidsBlob); + + switch(op) { + int nPrefix; + char * sPrefix; + char *sFull; + int nFull; + u8 * view; + case VEC0_METADATA_OPERATOR_EQ: { + for(int i = 0; i < size; i++) { + view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + nPrefix = ((int*) view)[0]; + sPrefix = (char *) &view[4]; + + // for EQ the text lengths must match + if(nPrefix != nTarget) { + bitmap_set(b, i, 0); + continue; + } + int cmpPrefix = strncmp(sPrefix, sTarget, min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH)); + + // for short strings, use the prefix comparison direclty + if(nPrefix <= VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + bitmap_set(b, i, cmpPrefix == 0); + continue; + } + // for EQ on longs strings, the prefix must match + if(cmpPrefix) { + bitmap_set(b, i, 0); + continue; + } + // consult the full string + rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull); + if(rc != SQLITE_OK) { + goto done; + } + if(nPrefix != nFull) { + rc = SQLITE_ERROR; + goto done; + } + bitmap_set(b, i, strncmp(sFull, sTarget, nFull) == 0); + } + break; + } + case VEC0_METADATA_OPERATOR_NE: { + for(int i = 0; i < size; i++) { + view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + nPrefix = ((int*) view)[0]; + sPrefix = (char *) &view[4]; + + // for NE if text lengths dont match, it never will + if(nPrefix != nTarget) { + bitmap_set(b, i, 1); + continue; + } + + int cmpPrefix = strncmp(sPrefix, sTarget, min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH)); + + // for short strings, use the prefix comparison direclty + if(nPrefix <= VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + bitmap_set(b, i, cmpPrefix != 0); + continue; + } + // for NE on longs strings, if prefixes dont match, then long string wont + if(cmpPrefix) { + bitmap_set(b, i, 1); + continue; + } + // consult the full string + rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull); + if(rc != SQLITE_OK) { + goto done; + } + if(nPrefix != nFull) { + rc = SQLITE_ERROR; + goto done; + } + bitmap_set(b, i, strncmp(sFull, sTarget, nFull) != 0); + } + break; + } + case VEC0_METADATA_OPERATOR_GT: { + for(int i = 0; i < size; i++) { + view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + nPrefix = ((int*) view)[0]; + sPrefix = (char *) &view[4]; + int cmpPrefix = strncmp(sPrefix, sTarget, min(min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH), nTarget)); + + if(nPrefix < VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + // if prefix match, check which is longer + if(cmpPrefix == 0) { + bitmap_set(b, i, nPrefix > nTarget); + } + else { + bitmap_set(b, i, cmpPrefix > 0); + } + continue; + } + // TODO(perf): may not need to compare full text in some cases + + rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull); + if(rc != SQLITE_OK) { + goto done; + } + if(nPrefix != nFull) { + rc = SQLITE_ERROR; + goto done; + } + bitmap_set(b, i, strncmp(sFull, sTarget, nFull) > 0); + } + break; + } + case VEC0_METADATA_OPERATOR_GE: { + for(int i = 0; i < size; i++) { + view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + nPrefix = ((int*) view)[0]; + sPrefix = (char *) &view[4]; + int cmpPrefix = strncmp(sPrefix, sTarget, min(min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH), nTarget)); + + if(nPrefix < VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + // if prefix match, check which is longer + if(cmpPrefix == 0) { + bitmap_set(b, i, nPrefix >= nTarget); + } + else { + bitmap_set(b, i, cmpPrefix >= 0); + } + continue; + } + // TODO(perf): may not need to compare full text in some cases + + rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull); + if(rc != SQLITE_OK) { + goto done; + } + if(nPrefix != nFull) { + rc = SQLITE_ERROR; + goto done; + } + bitmap_set(b, i, strncmp(sFull, sTarget, nFull) >= 0); + } + break; + } + case VEC0_METADATA_OPERATOR_LE: { + for(int i = 0; i < size; i++) { + view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + nPrefix = ((int*) view)[0]; + sPrefix = (char *) &view[4]; + int cmpPrefix = strncmp(sPrefix, sTarget, min(min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH), nTarget)); + + if(nPrefix < VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + // if prefix match, check which is longer + if(cmpPrefix == 0) { + bitmap_set(b, i, nPrefix <= nTarget); + } + else { + bitmap_set(b, i, cmpPrefix <= 0); + } + continue; + } + // TODO(perf): may not need to compare full text in some cases + + rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull); + if(rc != SQLITE_OK) { + goto done; + } + if(nPrefix != nFull) { + rc = SQLITE_ERROR; + goto done; + } + bitmap_set(b, i, strncmp(sFull, sTarget, nFull) <= 0); + } + break; + } + case VEC0_METADATA_OPERATOR_LT: { + for(int i = 0; i < size; i++) { + view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + nPrefix = ((int*) view)[0]; + sPrefix = (char *) &view[4]; + int cmpPrefix = strncmp(sPrefix, sTarget, min(min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH), nTarget)); + + if(nPrefix < VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + // if prefix match, check which is longer + if(cmpPrefix == 0) { + bitmap_set(b, i, nPrefix < nTarget); + } + else { + bitmap_set(b, i, cmpPrefix < 0); + } + continue; + } + // TODO(perf): may not need to compare full text in some cases + + rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull); + if(rc != SQLITE_OK) { + goto done; + } + if(nPrefix != nFull) { + rc = SQLITE_ERROR; + goto done; + } + bitmap_set(b, i, strncmp(sFull, sTarget, nFull) < 0); + } + break; + } + + case VEC0_METADATA_OPERATOR_IN: { + size_t metadataInIdx = -1; + for(size_t i = 0; i < aMetadataIn->length; i++) { + struct Vec0MetadataIn * metadataIn = &(((struct Vec0MetadataIn *) aMetadataIn->z)[i]); + if(metadataIn->argv_idx == argv_idx) { + metadataInIdx = i; + break; + } + } + if(metadataInIdx < 0) { + rc = SQLITE_ERROR; + goto done; + } + + struct Vec0MetadataIn * metadataIn = &((struct Vec0MetadataIn *) aMetadataIn->z)[metadataInIdx]; + struct Array * aTarget = &(metadataIn->array); + + + int nPrefix; + char * sPrefix; + char *sFull; + int nFull; + u8 * view; + for(int i = 0; i < size; i++) { + view = &((u8*) buffer)[i * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + nPrefix = ((int*) view)[0]; + sPrefix = (char *) &view[4]; + for(size_t target_idx = 0; target_idx < aTarget->length; target_idx++) { + struct Vec0MetadataInTextEntry * entry = &(((struct Vec0MetadataInTextEntry*)aTarget->z)[target_idx]); + if(entry->n != nPrefix) { + continue; + } + int cmpPrefix = strncmp(sPrefix, entry->zString, min(nPrefix, VEC0_METADATA_TEXT_VIEW_DATA_LENGTH)); + if(nPrefix <= VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + if(cmpPrefix == 0) { + bitmap_set(b, i, 1); + break; + } + continue; + } + if(cmpPrefix) { + continue; + } + + rc = vec0_get_metadata_text_long_value(p, &stmt, metadata_idx, rowids[i], &nFull, &sFull); + if(rc != SQLITE_OK) { + goto done; + } + if(nPrefix != nFull) { + rc = SQLITE_ERROR; + goto done; + } + if(strncmp(sFull, entry->zString, nFull) == 0) { + bitmap_set(b, i, 1); + break; + } + } + } + break; + } + + } + rc = SQLITE_OK; + + done: + sqlite3_finalize(stmt); + sqlite3_free(rowids); + return rc; + +} + +/** + * @brief Fill in bitmap of chunk values, whether or not the values match a metadata constraint + * + * @param p vec0_vtab + * @param metadata_idx index of the metatadata column to perfrom constraints on + * @param value sqlite3_value of the constraints value + * @param blob sqlite3_blob that is already opened on the metdata column's shadow chunk table + * @param chunk_rowid rowid of the chunk to calculate on + * @param b pre-allocated and zero'd out bitmap to write results to + * @param size size of the chunk + * @return int SQLITE_OK on success, error code otherwise + */ +int vec0_set_metadata_filter_bitmap( + vec0_vtab *p, + int metadata_idx, + vec0_metadata_operator op, + sqlite3_value * value, + sqlite3_blob * blob, + i64 chunk_rowid, + u8* b, + int size, + struct Array * aMetadataIn, int argv_idx) { + // TODO: shouldn't this skip in-valid entries from the chunk's validity bitmap? + + int rc; + rc = sqlite3_blob_reopen(blob, chunk_rowid); + if(rc != SQLITE_OK) { + return rc; + } + + vec0_metadata_column_kind kind = p->metadata_columns[metadata_idx].kind; + int szMatch = 0; + int blobSize = sqlite3_blob_bytes(blob); + switch(kind) { + case VEC0_METADATA_COLUMN_KIND_BOOLEAN: { + szMatch = blobSize == size / CHAR_BIT; + break; + } + case VEC0_METADATA_COLUMN_KIND_INTEGER: { + szMatch = blobSize == size * sizeof(i64); + break; + } + case VEC0_METADATA_COLUMN_KIND_FLOAT: { + szMatch = blobSize == size * sizeof(double); + break; + } + case VEC0_METADATA_COLUMN_KIND_TEXT: { + szMatch = blobSize == size * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH; + break; + } + } + if(!szMatch) { + return SQLITE_ERROR; + } + void * buffer = sqlite3_malloc(blobSize); + if(!buffer) { + return SQLITE_NOMEM; + } + rc = sqlite3_blob_read(blob, buffer, blobSize, 0); + if(rc != SQLITE_OK) { + goto done; + } + switch(kind) { + case VEC0_METADATA_COLUMN_KIND_BOOLEAN: { + int target = sqlite3_value_int(value); + if( (target && op == VEC0_METADATA_OPERATOR_EQ) || (!target && op == VEC0_METADATA_OPERATOR_NE)) { + for(int i = 0; i < size; i++) { bitmap_set(b, i, bitmap_get((u8*) buffer, i)); } + } + else { + for(int i = 0; i < size; i++) { bitmap_set(b, i, !bitmap_get((u8*) buffer, i)); } + } + break; + } + case VEC0_METADATA_COLUMN_KIND_INTEGER: { + i64 * array = (i64*) buffer; + i64 target = sqlite3_value_int64(value); + switch(op) { + case VEC0_METADATA_OPERATOR_EQ: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] == target); } + break; + } + case VEC0_METADATA_OPERATOR_GT: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] > target); } + break; + } + case VEC0_METADATA_OPERATOR_LE: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] <= target); } + break; + } + case VEC0_METADATA_OPERATOR_LT: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] < target); } + break; + } + case VEC0_METADATA_OPERATOR_GE: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] >= target); } + break; + } + case VEC0_METADATA_OPERATOR_NE: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] != target); } + break; + } + case VEC0_METADATA_OPERATOR_IN: { + int metadataInIdx = -1; + for(size_t i = 0; i < aMetadataIn->length; i++) { + struct Vec0MetadataIn * metadataIn = &((struct Vec0MetadataIn *) aMetadataIn->z)[i]; + if(metadataIn->argv_idx == argv_idx) { + metadataInIdx = i; + break; + } + } + if(metadataInIdx < 0) { + rc = SQLITE_ERROR; + goto done; + } + struct Vec0MetadataIn * metadataIn = &((struct Vec0MetadataIn *) aMetadataIn->z)[metadataInIdx]; + struct Array * aTarget = &(metadataIn->array); + + for(int i = 0; i < size; i++) { + for(size_t target_idx = 0; target_idx < aTarget->length; target_idx++) { + if( ((i64*)aTarget->z)[target_idx] == array[i]) { + bitmap_set(b, i, 1); + break; + } + } + } + break; + } + } + break; + } + case VEC0_METADATA_COLUMN_KIND_FLOAT: { + double * array = (double*) buffer; + double target = sqlite3_value_double(value); + switch(op) { + case VEC0_METADATA_OPERATOR_EQ: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] == target); } + break; + } + case VEC0_METADATA_OPERATOR_GT: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] > target); } + break; + } + case VEC0_METADATA_OPERATOR_LE: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] <= target); } + break; + } + case VEC0_METADATA_OPERATOR_LT: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] < target); } + break; + } + case VEC0_METADATA_OPERATOR_GE: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] >= target); } + break; + } + case VEC0_METADATA_OPERATOR_NE: { + for(int i = 0; i < size; i++) { bitmap_set(b, i, array[i] != target); } + break; + } + case VEC0_METADATA_OPERATOR_IN: { + // should never be reached + break; + } + } + break; + } + case VEC0_METADATA_COLUMN_KIND_TEXT: { + rc = vec0_metadata_filter_text(p, value, buffer, size, op, b, metadata_idx, chunk_rowid, aMetadataIn, argv_idx); + if(rc != SQLITE_OK) { + goto done; + } + break; + } + } + done: + sqlite3_free(buffer); + return rc; +} + +int vec0Filter_knn_chunks_iter(vec0_vtab *p, sqlite3_stmt *stmtChunks, + struct VectorColumnDefinition *vector_column, + int vectorColumnIdx, struct Array *arrayRowidsIn, + struct Array * aMetadataIn, + const char * idxStr, int argc, sqlite3_value ** argv, + void *queryVector, i64 k, i64 **out_topk_rowids, + f32 **out_topk_distances, i64 *out_used) { + // for each chunk, get top min(k, chunk_size) rowid + distances to query vec. + // then reconcile all topk_chunks for a true top k. + // output only rowids + distances for now + + int rc = SQLITE_OK; + sqlite3_blob *blobVectors = NULL; + + void *baseVectors = NULL; // memory: chunk_size * dimensions * element_size + + // OWNED BY CALLER ON SUCCESS + i64 *topk_rowids = NULL; // memory: k * 4 + // OWNED BY CALLER ON SUCCESS + f32 *topk_distances = NULL; // memory: k * 4 + + i64 *tmp_topk_rowids = NULL; // memory: k * 4 + f32 *tmp_topk_distances = NULL; // memory: k * 4 + f32 *chunk_distances = NULL; // memory: chunk_size * 4 + u8 *b = NULL; // memory: chunk_size / 8 + u8 *bTaken = NULL; // memory: chunk_size / 8 + i32 *chunk_topk_idxs = NULL; // memory: k * 4 + u8 *bmRowids = NULL; // memory: chunk_size / 8 + u8 *bmMetadata = NULL; // memory: chunk_size / 8 + // // total: a lot??? + + // 6 * (k * 4) + (k * 2) + (chunk_size / 8) + (chunk_size * dimensions * 4) + + topk_rowids = sqlite3_malloc(k * sizeof(i64)); + if (!topk_rowids) { + rc = SQLITE_NOMEM; + goto cleanup; + } + memset(topk_rowids, 0, k * sizeof(i64)); + + topk_distances = sqlite3_malloc(k * sizeof(f32)); + if (!topk_distances) { + rc = SQLITE_NOMEM; + goto cleanup; + } + memset(topk_distances, 0, k * sizeof(f32)); + + tmp_topk_rowids = sqlite3_malloc(k * sizeof(i64)); + if (!tmp_topk_rowids) { + rc = SQLITE_NOMEM; + goto cleanup; + } + memset(tmp_topk_rowids, 0, k * sizeof(i64)); + + tmp_topk_distances = sqlite3_malloc(k * sizeof(f32)); + if (!tmp_topk_distances) { + rc = SQLITE_NOMEM; + goto cleanup; + } + memset(tmp_topk_distances, 0, k * sizeof(f32)); + + i64 k_used = 0; + i64 baseVectorsSize = p->chunk_size * vector_column_byte_size(*vector_column); + baseVectors = sqlite3_malloc(baseVectorsSize); + if (!baseVectors) { + rc = SQLITE_NOMEM; + goto cleanup; + } + + chunk_distances = sqlite3_malloc(p->chunk_size * sizeof(f32)); + if (!chunk_distances) { + rc = SQLITE_NOMEM; + goto cleanup; + } + + b = bitmap_new(p->chunk_size); + if (!b) { + rc = SQLITE_NOMEM; + goto cleanup; + } + + bTaken = bitmap_new(p->chunk_size); + if (!bTaken) { + rc = SQLITE_NOMEM; + goto cleanup; + } + + chunk_topk_idxs = sqlite3_malloc(k * sizeof(i32)); + if (!chunk_topk_idxs) { + rc = SQLITE_NOMEM; + goto cleanup; + } + + bmRowids = arrayRowidsIn ? bitmap_new(p->chunk_size) : NULL; + if (arrayRowidsIn && !bmRowids) { + rc = SQLITE_NOMEM; + goto cleanup; + } + + sqlite3_blob * metadataBlobs[VEC0_MAX_METADATA_COLUMNS]; + memset(metadataBlobs, 0, sizeof(sqlite3_blob*) * VEC0_MAX_METADATA_COLUMNS); + + bmMetadata = bitmap_new(p->chunk_size); + if(!bmMetadata) { + rc = SQLITE_NOMEM; + goto cleanup; + } + + int idxStrLength = strlen(idxStr); + int numValueEntries = (idxStrLength-1) / 4; + assert(numValueEntries == argc); + int hasMetadataFilters = 0; + for(int i = 0; i < argc; i++) { + int idx = 1 + (i * 4); + char kind = idxStr[idx + 0]; + if(kind == VEC0_IDXSTR_KIND_METADATA_CONSTRAINT) { + hasMetadataFilters = 1; + break; + } + } + + while (true) { + rc = sqlite3_step(stmtChunks); + if (rc == SQLITE_DONE) { + break; + } + if (rc != SQLITE_ROW) { + vtab_set_error(&p->base, "chunks iter error"); + rc = SQLITE_ERROR; + goto cleanup; + } + memset(chunk_distances, 0, p->chunk_size * sizeof(f32)); + memset(chunk_topk_idxs, 0, k * sizeof(i32)); + bitmap_clear(b, p->chunk_size); + + i64 chunk_id = sqlite3_column_int64(stmtChunks, 0); + unsigned char *chunkValidity = + (unsigned char *)sqlite3_column_blob(stmtChunks, 1); + i64 validitySize = sqlite3_column_bytes(stmtChunks, 1); + if (validitySize != p->chunk_size / CHAR_BIT) { + // IMP: V05271_22109 + vtab_set_error( + &p->base, + "chunk validity size doesn't match - expected %lld, found %lld", + p->chunk_size / CHAR_BIT, validitySize); + rc = SQLITE_ERROR; + goto cleanup; + } + + i64 *chunkRowids = (i64 *)sqlite3_column_blob(stmtChunks, 2); + i64 rowidsSize = sqlite3_column_bytes(stmtChunks, 2); + if (rowidsSize != p->chunk_size * sizeof(i64)) { + // IMP: V02796_19635 + vtab_set_error(&p->base, "rowids size doesn't match"); + vtab_set_error( + &p->base, + "chunk rowids size doesn't match - expected %lld, found %lld", + p->chunk_size * sizeof(i64), rowidsSize); + rc = SQLITE_ERROR; + goto cleanup; + } + + // open the vector chunk blob for the current chunk + rc = sqlite3_blob_open(p->db, p->schemaName, + p->shadowVectorChunksNames[vectorColumnIdx], + "vectors", chunk_id, 0, &blobVectors); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, "could not open vectors blob for chunk %lld", + chunk_id); + rc = SQLITE_ERROR; + goto cleanup; + } + + i64 currentBaseVectorsSize = sqlite3_blob_bytes(blobVectors); + i64 expectedBaseVectorsSize = + p->chunk_size * vector_column_byte_size(*vector_column); + if (currentBaseVectorsSize != expectedBaseVectorsSize) { + // IMP: V16465_00535 + vtab_set_error( + &p->base, + "vectors blob size doesn't match - expected %lld, found %lld", + expectedBaseVectorsSize, currentBaseVectorsSize); + rc = SQLITE_ERROR; + goto cleanup; + } + rc = sqlite3_blob_read(blobVectors, baseVectors, currentBaseVectorsSize, 0); + + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, "vectors blob read error for %lld", chunk_id); + rc = SQLITE_ERROR; + goto cleanup; + } + + bitmap_copy(b, chunkValidity, p->chunk_size); + if (arrayRowidsIn) { + bitmap_clear(bmRowids, p->chunk_size); + + for (int i = 0; i < p->chunk_size; i++) { + if (!bitmap_get(chunkValidity, i)) { + continue; + } + i64 rowid = chunkRowids[i]; + void *in = bsearch(&rowid, arrayRowidsIn->z, arrayRowidsIn->length, + sizeof(i64), _cmp); + bitmap_set(bmRowids, i, in ? 1 : 0); + } + bitmap_and_inplace(b, bmRowids, p->chunk_size); + } + + if(hasMetadataFilters) { + for(int i = 0; i < argc; i++) { + int idx = 1 + (i * 4); + char kind = idxStr[idx + 0]; + if(kind != VEC0_IDXSTR_KIND_METADATA_CONSTRAINT) { + continue; + } + int metadata_idx = idxStr[idx + 1] - 'A'; + int operator = idxStr[idx + 2]; + + if(!metadataBlobs[metadata_idx]) { + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowMetadataChunksNames[metadata_idx], "data", chunk_id, 0, &metadataBlobs[metadata_idx]); + vtab_set_error(&p->base, "Could not open metadata blob"); + if(rc != SQLITE_OK) { + goto cleanup; + } + } + + bitmap_clear(bmMetadata, p->chunk_size); + rc = vec0_set_metadata_filter_bitmap(p, metadata_idx, operator, argv[i], metadataBlobs[metadata_idx], chunk_id, bmMetadata, p->chunk_size, aMetadataIn, i); + if(rc != SQLITE_OK) { + vtab_set_error(&p->base, "Could not filter metadata fields"); + if(rc != SQLITE_OK) { + goto cleanup; + } + } + bitmap_and_inplace(b, bmMetadata, p->chunk_size); + } + } + + + for (int i = 0; i < p->chunk_size; i++) { + if (!bitmap_get(b, i)) { + continue; + }; + + f32 result; + switch (vector_column->element_type) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: { + const f32 *base_i = + ((f32 *)baseVectors) + (i * vector_column->dimensions); + switch (vector_column->distance_metric) { + case VEC0_DISTANCE_METRIC_L2: { + result = distance_l2_sqr_float(base_i, (f32 *)queryVector, + &vector_column->dimensions); + break; + } + case VEC0_DISTANCE_METRIC_L1: { + result = distance_l1_f32(base_i, (f32 *)queryVector, + &vector_column->dimensions); + break; + } + case VEC0_DISTANCE_METRIC_COSINE: { + result = distance_cosine_float(base_i, (f32 *)queryVector, + &vector_column->dimensions); + break; + } + } + break; + } + case SQLITE_VEC_ELEMENT_TYPE_INT8: { + const i8 *base_i = + ((i8 *)baseVectors) + (i * vector_column->dimensions); + switch (vector_column->distance_metric) { + case VEC0_DISTANCE_METRIC_L2: { + result = distance_l2_sqr_int8(base_i, (i8 *)queryVector, + &vector_column->dimensions); + break; + } + case VEC0_DISTANCE_METRIC_L1: { + result = distance_l1_int8(base_i, (i8 *)queryVector, + &vector_column->dimensions); + break; + } + case VEC0_DISTANCE_METRIC_COSINE: { + result = distance_cosine_int8(base_i, (i8 *)queryVector, + &vector_column->dimensions); + break; + } + } + + break; + } + case SQLITE_VEC_ELEMENT_TYPE_BIT: { + const u8 *base_i = + ((u8 *)baseVectors) + (i * (vector_column->dimensions / CHAR_BIT)); + result = distance_hamming(base_i, (u8 *)queryVector, + &vector_column->dimensions); + break; + } + } + + chunk_distances[i] = result; + } + + int used1; + min_idx(chunk_distances, p->chunk_size, b, chunk_topk_idxs, + min(k, p->chunk_size), bTaken, &used1); + + i64 used; + merge_sorted_lists(topk_distances, topk_rowids, k_used, chunk_distances, + chunkRowids, chunk_topk_idxs, + min(min(k, p->chunk_size), used1), tmp_topk_distances, + tmp_topk_rowids, k, &used); + + for (int i = 0; i < used; i++) { + topk_rowids[i] = tmp_topk_rowids[i]; + topk_distances[i] = tmp_topk_distances[i]; + } + k_used = used; + // blobVectors is always opened with read-only permissions, so this never + // fails. + sqlite3_blob_close(blobVectors); + blobVectors = NULL; + } + + *out_topk_rowids = topk_rowids; + *out_topk_distances = topk_distances; + *out_used = k_used; + rc = SQLITE_OK; + +cleanup: + if (rc != SQLITE_OK) { + sqlite3_free(topk_rowids); + sqlite3_free(topk_distances); + } + sqlite3_free(chunk_topk_idxs); + sqlite3_free(tmp_topk_rowids); + sqlite3_free(tmp_topk_distances); + sqlite3_free(b); + sqlite3_free(bTaken); + sqlite3_free(bmRowids); + sqlite3_free(baseVectors); + sqlite3_free(chunk_distances); + sqlite3_free(bmMetadata); + for(int i = 0; i < VEC0_MAX_METADATA_COLUMNS; i++) { + sqlite3_blob_close(metadataBlobs[i]); + } + // blobVectors is always opened with read-only permissions, so this never + // fails. + sqlite3_blob_close(blobVectors); + return rc; +} + +int vec0Filter_knn(vec0_cursor *pCur, vec0_vtab *p, int idxNum, + const char *idxStr, int argc, sqlite3_value **argv) { + assert(argc == (strlen(idxStr)-1) / 4); + int rc; + struct vec0_query_knn_data *knn_data; + + int vectorColumnIdx = idxNum; + struct VectorColumnDefinition *vector_column = + &p->vector_columns[vectorColumnIdx]; + + struct Array *arrayRowidsIn = NULL; + sqlite3_stmt *stmtChunks = NULL; + void *queryVector; + size_t dimensions; + enum VectorElementType elementType; + vector_cleanup queryVectorCleanup = vector_cleanup_noop; + char *pzError; + knn_data = sqlite3_malloc(sizeof(*knn_data)); + if (!knn_data) { + return SQLITE_NOMEM; + } + memset(knn_data, 0, sizeof(*knn_data)); + // array of `struct Vec0MetadataIn`, IF there are any `xxx in (...)` metadata constraints + struct Array * aMetadataIn = NULL; + + int query_idx =-1; + int k_idx = -1; + int rowid_in_idx = -1; + for(int i = 0; i < argc; i++) { + if(idxStr[1 + (i*4)] == VEC0_IDXSTR_KIND_KNN_MATCH) { + query_idx = i; + } + if(idxStr[1 + (i*4)] == VEC0_IDXSTR_KIND_KNN_K) { + k_idx = i; + } + if(idxStr[1 + (i*4)] == VEC0_IDXSTR_KIND_KNN_ROWID_IN) { + rowid_in_idx = i; + } + } + assert(query_idx >= 0); + assert(k_idx >= 0); + + // make sure the query vector matches the vector column (type dimensions etc.) + rc = vector_from_value(argv[query_idx], &queryVector, &dimensions, &elementType, + &queryVectorCleanup, &pzError); + + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, + "Query vector on the \"%.*s\" column is invalid: %z", + vector_column->name_length, vector_column->name, pzError); + rc = SQLITE_ERROR; + goto cleanup; + } + if (elementType != vector_column->element_type) { + vtab_set_error( + &p->base, + "Query vector for the \"%.*s\" column is expected to be of type " + "%s, but a %s vector was provided.", + vector_column->name_length, vector_column->name, + vector_subtype_name(vector_column->element_type), + vector_subtype_name(elementType)); + rc = SQLITE_ERROR; + goto cleanup; + } + if (dimensions != vector_column->dimensions) { + vtab_set_error( + &p->base, + "Dimension mismatch for query vector for the \"%.*s\" column. " + "Expected %d dimensions but received %d.", + vector_column->name_length, vector_column->name, + vector_column->dimensions, dimensions); + rc = SQLITE_ERROR; + goto cleanup; + } + + i64 k = sqlite3_value_int64(argv[k_idx]); + if (k < 0) { + vtab_set_error( + &p->base, "k value in knn queries must be greater than or equal to 0."); + rc = SQLITE_ERROR; + goto cleanup; + } +#define SQLITE_VEC_VEC0_K_MAX 4096 + if (k > SQLITE_VEC_VEC0_K_MAX) { + vtab_set_error( + &p->base, + "k value in knn query too large, provided %lld and the limit is %lld", + k, SQLITE_VEC_VEC0_K_MAX); + rc = SQLITE_ERROR; + goto cleanup; + } + + if (k == 0) { + knn_data->k = 0; + pCur->knn_data = knn_data; + pCur->query_plan = VEC0_QUERY_PLAN_KNN; + rc = SQLITE_OK; + goto cleanup; + } + +// handle when a `rowid in (...)` operation was provided +// Array of all the rowids that appear in any `rowid in (...)` constraint. +// NULL if none were provided, which means a "full" scan. +#if COMPILER_SUPPORTS_VTAB_IN + if (rowid_in_idx >= 0) { + sqlite3_value *item; + int rc; + arrayRowidsIn = sqlite3_malloc(sizeof(*arrayRowidsIn)); + if (!arrayRowidsIn) { + rc = SQLITE_NOMEM; + goto cleanup; + } + memset(arrayRowidsIn, 0, sizeof(*arrayRowidsIn)); + + rc = array_init(arrayRowidsIn, sizeof(i64), 32); + if (rc != SQLITE_OK) { + goto cleanup; + } + for (rc = sqlite3_vtab_in_first(argv[rowid_in_idx], &item); rc == SQLITE_OK && item; + rc = sqlite3_vtab_in_next(argv[rowid_in_idx], &item)) { + i64 rowid; + if (p->pkIsText) { + rc = vec0_rowid_from_id(p, item, &rowid); + if (rc != SQLITE_OK) { + goto cleanup; + } + } else { + rowid = sqlite3_value_int64(item); + } + rc = array_append(arrayRowidsIn, &rowid); + if (rc != SQLITE_OK) { + goto cleanup; + } + } + if (rc != SQLITE_DONE) { + vtab_set_error(&p->base, "error processing rowid in (...) array"); + goto cleanup; + } + qsort(arrayRowidsIn->z, arrayRowidsIn->length, arrayRowidsIn->element_size, + _cmp); + } +#endif + + #if COMPILER_SUPPORTS_VTAB_IN + for(int i = 0; i < argc; i++) { + if(!(idxStr[1 + (i*4)] == VEC0_IDXSTR_KIND_METADATA_CONSTRAINT && idxStr[1 + (i*4) + 2] == VEC0_METADATA_OPERATOR_IN)) { + continue; + } + int metadata_idx = idxStr[1 + (i*4) + 1] - 'A'; + if(!aMetadataIn) { + aMetadataIn = sqlite3_malloc(sizeof(*aMetadataIn)); + if(!aMetadataIn) { + rc = SQLITE_NOMEM; + goto cleanup; + } + memset(aMetadataIn, 0, sizeof(*aMetadataIn)); + rc = array_init(aMetadataIn, sizeof(struct Vec0MetadataIn), 8); + if(rc != SQLITE_OK) { + goto cleanup; + } + } + + struct Vec0MetadataIn item; + memset(&item, 0, sizeof(item)); + item.metadata_idx=metadata_idx; + item.argv_idx = i; + + switch(p->metadata_columns[metadata_idx].kind) { + case VEC0_METADATA_COLUMN_KIND_INTEGER: { + rc = array_init(&item.array, sizeof(i64), 16); + if(rc != SQLITE_OK) { + goto cleanup; + } + sqlite3_value *entry; + for (rc = sqlite3_vtab_in_first(argv[i], &entry); rc == SQLITE_OK && entry; rc = sqlite3_vtab_in_next(argv[i], &entry)) { + i64 v = sqlite3_value_int64(entry); + rc = array_append(&item.array, &v); + if (rc != SQLITE_OK) { + goto cleanup; + } + } + + if (rc != SQLITE_DONE) { + vtab_set_error(&p->base, "Error fetching next value in `x in (...)` integer expression"); + goto cleanup; + } + + break; + } + case VEC0_METADATA_COLUMN_KIND_TEXT: { + rc = array_init(&item.array, sizeof(struct Vec0MetadataInTextEntry), 16); + if(rc != SQLITE_OK) { + goto cleanup; + } + sqlite3_value *entry; + for (rc = sqlite3_vtab_in_first(argv[i], &entry); rc == SQLITE_OK && entry; rc = sqlite3_vtab_in_next(argv[i], &entry)) { + const char * s = (const char *) sqlite3_value_text(entry); + int n = sqlite3_value_bytes(entry); + + struct Vec0MetadataInTextEntry entry; + entry.zString = sqlite3_mprintf("%.*s", n, s); + if(!entry.zString) { + rc = SQLITE_NOMEM; + goto cleanup; + } + entry.n = n; + rc = array_append(&item.array, &entry); + if (rc != SQLITE_OK) { + goto cleanup; + } + } + + if (rc != SQLITE_DONE) { + vtab_set_error(&p->base, "Error fetching next value in `x in (...)` text expression"); + goto cleanup; + } + + break; + } + default: { + vtab_set_error(&p->base, "Internal sqlite-vec error"); + goto cleanup; + } + } + + rc = array_append(aMetadataIn, &item); + if(rc != SQLITE_OK) { + goto cleanup; + } + } + #endif + + rc = vec0_chunks_iter(p, idxStr, argc, argv, &stmtChunks); + if (rc != SQLITE_OK) { + // IMP: V06942_23781 + vtab_set_error(&p->base, "Error preparing stmtChunk: %s", + sqlite3_errmsg(p->db)); + goto cleanup; + } + + i64 *topk_rowids = NULL; + f32 *topk_distances = NULL; + i64 k_used = 0; + rc = vec0Filter_knn_chunks_iter(p, stmtChunks, vector_column, vectorColumnIdx, + arrayRowidsIn, aMetadataIn, idxStr, argc, argv, queryVector, k, &topk_rowids, + &topk_distances, &k_used); + if (rc != SQLITE_OK) { + goto cleanup; + } + + knn_data->current_idx = 0; + knn_data->k = k; + knn_data->rowids = topk_rowids; + knn_data->distances = topk_distances; + knn_data->k_used = k_used; + + pCur->knn_data = knn_data; + pCur->query_plan = VEC0_QUERY_PLAN_KNN; + rc = SQLITE_OK; + +cleanup: + sqlite3_finalize(stmtChunks); + array_cleanup(arrayRowidsIn); + sqlite3_free(arrayRowidsIn); + queryVectorCleanup(queryVector); + if(aMetadataIn) { + for(size_t i = 0; i < aMetadataIn->length; i++) { + struct Vec0MetadataIn* item = &((struct Vec0MetadataIn *) aMetadataIn->z)[i]; + for(size_t j = 0; j < item->array.length; j++) { + if(p->metadata_columns[item->metadata_idx].kind == VEC0_METADATA_COLUMN_KIND_TEXT) { + struct Vec0MetadataInTextEntry entry = ((struct Vec0MetadataInTextEntry*)item->array.z)[j]; + sqlite3_free(entry.zString); + } + } + array_cleanup(&item->array); + } + array_cleanup(aMetadataIn); + } + + sqlite3_free(aMetadataIn); + + return rc; +} + +int vec0Filter_fullscan(vec0_vtab *p, vec0_cursor *pCur) { + int rc; + char *zSql; + struct vec0_query_fullscan_data *fullscan_data; + + fullscan_data = sqlite3_malloc(sizeof(*fullscan_data)); + if (!fullscan_data) { + return SQLITE_NOMEM; + } + memset(fullscan_data, 0, sizeof(*fullscan_data)); + + zSql = sqlite3_mprintf(" SELECT rowid " + " FROM " VEC0_SHADOW_ROWIDS_NAME + " ORDER by chunk_id, chunk_offset ", + p->schemaName, p->tableName); + if (!zSql) { + rc = SQLITE_NOMEM; + goto error; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &fullscan_data->rowids_stmt, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + // IMP: V09901_26739 + vtab_set_error(&p->base, "Error preparing rowid scan: %s", + sqlite3_errmsg(p->db)); + goto error; + } + + rc = sqlite3_step(fullscan_data->rowids_stmt); + + // DONE when there's no rowids, ROW when there are, both "success" + if (!(rc == SQLITE_ROW || rc == SQLITE_DONE)) { + goto error; + } + + fullscan_data->done = rc == SQLITE_DONE; + pCur->query_plan = VEC0_QUERY_PLAN_FULLSCAN; + pCur->fullscan_data = fullscan_data; + return SQLITE_OK; + +error: + vec0_query_fullscan_data_clear(fullscan_data); + sqlite3_free(fullscan_data); + return rc; +} + +int vec0Filter_point(vec0_cursor *pCur, vec0_vtab *p, int argc, + sqlite3_value **argv) { + int rc; + assert(argc == 1); + i64 rowid; + struct vec0_query_point_data *point_data = NULL; + + point_data = sqlite3_malloc(sizeof(*point_data)); + if (!point_data) { + rc = SQLITE_NOMEM; + goto error; + } + memset(point_data, 0, sizeof(*point_data)); + + if (p->pkIsText) { + rc = vec0_rowid_from_id(p, argv[0], &rowid); + if (rc == SQLITE_EMPTY) { + goto eof; + } + if (rc != SQLITE_OK) { + goto error; + } + } else { + rowid = sqlite3_value_int64(argv[0]); + } + + for (int i = 0; i < p->numVectorColumns; i++) { + rc = vec0_get_vector_data(p, rowid, i, &point_data->vectors[i], NULL); + if (rc == SQLITE_EMPTY) { + goto eof; + } + if (rc != SQLITE_OK) { + goto error; + } + } + + point_data->rowid = rowid; + point_data->done = 0; + pCur->point_data = point_data; + pCur->query_plan = VEC0_QUERY_PLAN_POINT; + return SQLITE_OK; + +eof: + point_data->rowid = rowid; + point_data->done = 1; + pCur->point_data = point_data; + pCur->query_plan = VEC0_QUERY_PLAN_POINT; + return SQLITE_OK; + +error: + vec0_query_point_data_clear(point_data); + sqlite3_free(point_data); + return rc; +} + +static int vec0Filter(sqlite3_vtab_cursor *pVtabCursor, int idxNum, + const char *idxStr, int argc, sqlite3_value **argv) { + vec0_vtab *p = (vec0_vtab *)pVtabCursor->pVtab; + vec0_cursor *pCur = (vec0_cursor *)pVtabCursor; + vec0_cursor_clear(pCur); + + int idxStrLength = strlen(idxStr); + if(idxStrLength <= 0) { + return SQLITE_ERROR; + } + if((idxStrLength-1) % 4 != 0) { + return SQLITE_ERROR; + } + int numValueEntries = (idxStrLength-1) / 4; + if(numValueEntries != argc) { + return SQLITE_ERROR; + } + + char query_plan = idxStr[0]; + switch(query_plan) { + case VEC0_QUERY_PLAN_FULLSCAN: + return vec0Filter_fullscan(p, pCur); + case VEC0_QUERY_PLAN_KNN: + return vec0Filter_knn(pCur, p, idxNum, idxStr, argc, argv); + case VEC0_QUERY_PLAN_POINT: + return vec0Filter_point(pCur, p, argc, argv); + default: + vtab_set_error(pVtabCursor->pVtab, "unknown idxStr '%s'", idxStr); + return SQLITE_ERROR; + } +} + +static int vec0Rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) { + vec0_cursor *pCur = (vec0_cursor *)cur; + switch (pCur->query_plan) { + case VEC0_QUERY_PLAN_FULLSCAN: { + *pRowid = sqlite3_column_int64(pCur->fullscan_data->rowids_stmt, 0); + return SQLITE_OK; + } + case VEC0_QUERY_PLAN_POINT: { + *pRowid = pCur->point_data->rowid; + return SQLITE_OK; + } + case VEC0_QUERY_PLAN_KNN: { + vtab_set_error(cur->pVtab, + "Internal sqlite-vec error: expected point query plan in " + "vec0Rowid, found %d", + pCur->query_plan); + return SQLITE_ERROR; + } + } + return SQLITE_ERROR; +} + +static int vec0Next(sqlite3_vtab_cursor *cur) { + vec0_cursor *pCur = (vec0_cursor *)cur; + switch (pCur->query_plan) { + case VEC0_QUERY_PLAN_FULLSCAN: { + if (!pCur->fullscan_data) { + return SQLITE_ERROR; + } + int rc = sqlite3_step(pCur->fullscan_data->rowids_stmt); + if (rc == SQLITE_DONE) { + pCur->fullscan_data->done = 1; + return SQLITE_OK; + } + if (rc == SQLITE_ROW) { + return SQLITE_OK; + } + return SQLITE_ERROR; + } + case VEC0_QUERY_PLAN_KNN: { + if (!pCur->knn_data) { + return SQLITE_ERROR; + } + + pCur->knn_data->current_idx++; + return SQLITE_OK; + } + case VEC0_QUERY_PLAN_POINT: { + if (!pCur->point_data) { + return SQLITE_ERROR; + } + pCur->point_data->done = 1; + return SQLITE_OK; + } + } + return SQLITE_ERROR; +} + +static int vec0Eof(sqlite3_vtab_cursor *cur) { + vec0_cursor *pCur = (vec0_cursor *)cur; + switch (pCur->query_plan) { + case VEC0_QUERY_PLAN_FULLSCAN: { + if (!pCur->fullscan_data) { + return 1; + } + return pCur->fullscan_data->done; + } + case VEC0_QUERY_PLAN_KNN: { + if (!pCur->knn_data) { + return 1; + } + // return (pCur->knn_data->current_idx >= pCur->knn_data->k) || + // (pCur->knn_data->distances[pCur->knn_data->current_idx] == FLT_MAX); + return (pCur->knn_data->current_idx >= pCur->knn_data->k_used); + } + case VEC0_QUERY_PLAN_POINT: { + if (!pCur->point_data) { + return 1; + } + return pCur->point_data->done; + } + } + return 1; +} + +static int vec0Column_fullscan(vec0_vtab *pVtab, vec0_cursor *pCur, + sqlite3_context *context, int i) { + if (!pCur->fullscan_data) { + sqlite3_result_error( + context, "Internal sqlite-vec error: fullscan_data is NULL.", -1); + return SQLITE_ERROR; + } + i64 rowid = sqlite3_column_int64(pCur->fullscan_data->rowids_stmt, 0); + if (i == VEC0_COLUMN_ID) { + return vec0_result_id(pVtab, context, rowid); + } + else if (vec0_column_idx_is_vector(pVtab, i)) { + void *v; + int sz; + int vector_idx = vec0_column_idx_to_vector_idx(pVtab, i); + int rc = vec0_get_vector_data(pVtab, rowid, vector_idx, &v, &sz); + if (rc != SQLITE_OK) { + return rc; + } + sqlite3_result_blob(context, v, sz, sqlite3_free); + sqlite3_result_subtype(context, + pVtab->vector_columns[vector_idx].element_type); + + } + else if (i == vec0_column_distance_idx(pVtab)) { + sqlite3_result_null(context); + } + else if(vec0_column_idx_is_partition(pVtab, i)) { + int partition_idx = vec0_column_idx_to_partition_idx(pVtab, i); + sqlite3_value * v; + int rc = vec0_get_partition_value_for_rowid(pVtab, rowid, partition_idx, &v); + if(rc == SQLITE_OK) { + sqlite3_result_value(context, v); + sqlite3_value_free(v); + }else { + sqlite3_result_error_code(context, rc); + } + } + else if(vec0_column_idx_is_auxiliary(pVtab, i)) { + int auxiliary_idx = vec0_column_idx_to_auxiliary_idx(pVtab, i); + sqlite3_value * v; + int rc = vec0_get_auxiliary_value_for_rowid(pVtab, rowid, auxiliary_idx, &v); + if(rc == SQLITE_OK) { + sqlite3_result_value(context, v); + sqlite3_value_free(v); + }else { + sqlite3_result_error_code(context, rc); + } + } + + else if(vec0_column_idx_is_metadata(pVtab, i)) { + if(sqlite3_vtab_nochange(context)) { + return SQLITE_OK; + } + int metadata_idx = vec0_column_idx_to_metadata_idx(pVtab, i); + int rc = vec0_result_metadata_value_for_rowid(pVtab, rowid, metadata_idx, context); + if(rc != SQLITE_OK) { + // IMP: V15466_32305 + const char * zErr = sqlite3_mprintf( + "Could not extract metadata value for column %.*s at rowid %lld", + pVtab->metadata_columns[metadata_idx].name_length, + pVtab->metadata_columns[metadata_idx].name, rowid + ); + if(zErr) { + sqlite3_result_error(context, zErr, -1); + sqlite3_free((void *) zErr); + }else { + sqlite3_result_error_nomem(context); + } + } + } + + return SQLITE_OK; +} + +static int vec0Column_point(vec0_vtab *pVtab, vec0_cursor *pCur, + sqlite3_context *context, int i) { + if (!pCur->point_data) { + sqlite3_result_error(context, + "Internal sqlite-vec error: point_data is NULL.", -1); + return SQLITE_ERROR; + } + if (i == VEC0_COLUMN_ID) { + return vec0_result_id(pVtab, context, pCur->point_data->rowid); + } + else if (i == vec0_column_distance_idx(pVtab)) { + sqlite3_result_null(context); + return SQLITE_OK; + } + else if (vec0_column_idx_is_vector(pVtab, i)) { + if (sqlite3_vtab_nochange(context)) { + sqlite3_result_null(context); + return SQLITE_OK; + } + int vector_idx = vec0_column_idx_to_vector_idx(pVtab, i); + sqlite3_result_blob( + context, pCur->point_data->vectors[vector_idx], + vector_column_byte_size(pVtab->vector_columns[vector_idx]), + SQLITE_TRANSIENT); + sqlite3_result_subtype(context, + pVtab->vector_columns[vector_idx].element_type); + return SQLITE_OK; + } + else if(vec0_column_idx_is_partition(pVtab, i)) { + if(sqlite3_vtab_nochange(context)) { + return SQLITE_OK; + } + int partition_idx = vec0_column_idx_to_partition_idx(pVtab, i); + i64 rowid = pCur->point_data->rowid; + sqlite3_value * v; + int rc = vec0_get_partition_value_for_rowid(pVtab, rowid, partition_idx, &v); + if(rc == SQLITE_OK) { + sqlite3_result_value(context, v); + sqlite3_value_free(v); + }else { + sqlite3_result_error_code(context, rc); + } + } + else if(vec0_column_idx_is_auxiliary(pVtab, i)) { + if(sqlite3_vtab_nochange(context)) { + return SQLITE_OK; + } + i64 rowid = pCur->point_data->rowid; + int auxiliary_idx = vec0_column_idx_to_auxiliary_idx(pVtab, i); + sqlite3_value * v; + int rc = vec0_get_auxiliary_value_for_rowid(pVtab, rowid, auxiliary_idx, &v); + if(rc == SQLITE_OK) { + sqlite3_result_value(context, v); + sqlite3_value_free(v); + }else { + sqlite3_result_error_code(context, rc); + } + } + + else if(vec0_column_idx_is_metadata(pVtab, i)) { + if(sqlite3_vtab_nochange(context)) { + return SQLITE_OK; + } + i64 rowid = pCur->point_data->rowid; + int metadata_idx = vec0_column_idx_to_metadata_idx(pVtab, i); + int rc = vec0_result_metadata_value_for_rowid(pVtab, rowid, metadata_idx, context); + if(rc != SQLITE_OK) { + const char * zErr = sqlite3_mprintf( + "Could not extract metadata value for column %.*s at rowid %lld", + pVtab->metadata_columns[metadata_idx].name_length, + pVtab->metadata_columns[metadata_idx].name, rowid + ); + if(zErr) { + sqlite3_result_error(context, zErr, -1); + sqlite3_free((void *) zErr); + }else { + sqlite3_result_error_nomem(context); + } + } + } + + return SQLITE_OK; +} + +static int vec0Column_knn(vec0_vtab *pVtab, vec0_cursor *pCur, + sqlite3_context *context, int i) { + if (!pCur->knn_data) { + sqlite3_result_error(context, + "Internal sqlite-vec error: knn_data is NULL.", -1); + return SQLITE_ERROR; + } + if (i == VEC0_COLUMN_ID) { + i64 rowid = pCur->knn_data->rowids[pCur->knn_data->current_idx]; + return vec0_result_id(pVtab, context, rowid); + } + else if (i == vec0_column_distance_idx(pVtab)) { + sqlite3_result_double( + context, pCur->knn_data->distances[pCur->knn_data->current_idx]); + return SQLITE_OK; + } + else if (vec0_column_idx_is_vector(pVtab, i)) { + void *out; + int sz; + int vector_idx = vec0_column_idx_to_vector_idx(pVtab, i); + int rc = vec0_get_vector_data( + pVtab, pCur->knn_data->rowids[pCur->knn_data->current_idx], vector_idx, + &out, &sz); + if (rc != SQLITE_OK) { + return rc; + } + sqlite3_result_blob(context, out, sz, sqlite3_free); + sqlite3_result_subtype(context, + pVtab->vector_columns[vector_idx].element_type); + return SQLITE_OK; + } + else if(vec0_column_idx_is_partition(pVtab, i)) { + int partition_idx = vec0_column_idx_to_partition_idx(pVtab, i); + i64 rowid = pCur->knn_data->rowids[pCur->knn_data->current_idx]; + sqlite3_value * v; + int rc = vec0_get_partition_value_for_rowid(pVtab, rowid, partition_idx, &v); + if(rc == SQLITE_OK) { + sqlite3_result_value(context, v); + sqlite3_value_free(v); + }else { + sqlite3_result_error_code(context, rc); + } + } + else if(vec0_column_idx_is_auxiliary(pVtab, i)) { + int auxiliary_idx = vec0_column_idx_to_auxiliary_idx(pVtab, i); + i64 rowid = pCur->knn_data->rowids[pCur->knn_data->current_idx]; + sqlite3_value * v; + int rc = vec0_get_auxiliary_value_for_rowid(pVtab, rowid, auxiliary_idx, &v); + if(rc == SQLITE_OK) { + sqlite3_result_value(context, v); + sqlite3_value_free(v); + }else { + sqlite3_result_error_code(context, rc); + } + } + + else if(vec0_column_idx_is_metadata(pVtab, i)) { + int metadata_idx = vec0_column_idx_to_metadata_idx(pVtab, i); + i64 rowid = pCur->knn_data->rowids[pCur->knn_data->current_idx]; + int rc = vec0_result_metadata_value_for_rowid(pVtab, rowid, metadata_idx, context); + if(rc != SQLITE_OK) { + const char * zErr = sqlite3_mprintf( + "Could not extract metadata value for column %.*s at rowid %lld", + pVtab->metadata_columns[metadata_idx].name_length, + pVtab->metadata_columns[metadata_idx].name, rowid + ); + if(zErr) { + sqlite3_result_error(context, zErr, -1); + sqlite3_free((void *) zErr); + }else { + sqlite3_result_error_nomem(context); + } + } + } + + return SQLITE_OK; +} + +static int vec0Column(sqlite3_vtab_cursor *cur, sqlite3_context *context, + int i) { + vec0_cursor *pCur = (vec0_cursor *)cur; + vec0_vtab *pVtab = (vec0_vtab *)cur->pVtab; + switch (pCur->query_plan) { + case VEC0_QUERY_PLAN_FULLSCAN: { + return vec0Column_fullscan(pVtab, pCur, context, i); + } + case VEC0_QUERY_PLAN_KNN: { + return vec0Column_knn(pVtab, pCur, context, i); + } + case VEC0_QUERY_PLAN_POINT: { + return vec0Column_point(pVtab, pCur, context, i); + } + } + return SQLITE_OK; +} + +/** + * @brief Handles the "insert rowid" step of a row insert operation of a vec0 + * table. + * + * This function will insert a new row into the _rowids vec0 shadow table. + * + * @param p: virtual table + * @param idValue: Value containing the inserted rowid/id value. + * @param rowid: Output rowid, will point to the "real" i64 rowid + * value that was inserted + * @return int SQLITE_OK on success, error code on failure + */ +int vec0Update_InsertRowidStep(vec0_vtab *p, sqlite3_value *idValue, + i64 *rowid) { + + /** + * An insert into a vec0 table can happen a few different ways: + * 1) With default INTEGER primary key: With a supplied i64 rowid + * 2) With default INTEGER primary key: WITHOUT a supplied rowid + * 3) With TEXT primary key: supplied text rowid + */ + + int rc; + + // Option 3: vtab has a user-defined TEXT primary key, so ensure a text value + // is provided. + if (p->pkIsText) { + if (sqlite3_value_type(idValue) != SQLITE_TEXT) { + // IMP: V04200_21039 + vtab_set_error(&p->base, + "The %s virtual table was declared with a TEXT primary " + "key, but a non-TEXT value was provided in an INSERT.", + p->tableName); + return SQLITE_ERROR; + } + + return vec0_rowids_insert_id(p, idValue, rowid); + } + + // Option 1: User supplied a i64 rowid + if (sqlite3_value_type(idValue) == SQLITE_INTEGER) { + i64 suppliedRowid = sqlite3_value_int64(idValue); + rc = vec0_rowids_insert_rowid(p, suppliedRowid); + if (rc == SQLITE_OK) { + *rowid = suppliedRowid; + } + return rc; + } + + // Option 2: User did not suppled a rowid + + if (sqlite3_value_type(idValue) != SQLITE_NULL) { + // IMP: V30855_14925 + vtab_set_error(&p->base, + "Only integers are allows for primary key values on %s", + p->tableName); + return SQLITE_ERROR; + } + // NULL to get next auto-incremented value + return vec0_rowids_insert_id(p, NULL, rowid); +} + +/** + * @brief Determines the "next available" chunk position for a newly inserted + * vec0 row. + * + * This operation may insert a new "blank" chunk the _chunks table, if there is + * no more space in previous chunks. + * + * @param p: virtual table + * @param partitionKeyValues: array of partition key column values, to constrain + * against any partition key columns. + * @param chunk_rowid: Output rowid of the chunk in the _chunks virtual table + * that has the avialabiity. + * @param chunk_offset: Output the index of the available space insert the + * chunk, based on the index of the first available validity bit. + * @param pBlobValidity: Output blob of the validity column of the available + * chunk. Will be opened with read/write permissions. + * @param pValidity: Output buffer of the original chunk's validity column. + * Needs to be cleaned up with sqlite3_free(). + * @return int SQLITE_OK on success, error code on failure + */ +int vec0Update_InsertNextAvailableStep( + vec0_vtab *p, + sqlite3_value ** partitionKeyValues, + i64 *chunk_rowid, i64 *chunk_offset, + sqlite3_blob **blobChunksValidity, + const unsigned char **bufferChunksValidity) { + + int rc; + i64 validitySize; + *chunk_offset = -1; + + rc = vec0_get_latest_chunk_rowid(p, chunk_rowid, partitionKeyValues); + if(rc == SQLITE_EMPTY) { + goto done; + } + if (rc != SQLITE_OK) { + goto cleanup; + } + + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, "validity", + *chunk_rowid, 1, blobChunksValidity); + if (rc != SQLITE_OK) { + // IMP: V22053_06123 + vtab_set_error(&p->base, + VEC_INTERAL_ERROR + "could not open validity blob on %s.%s.%lld", + p->schemaName, p->shadowChunksName, *chunk_rowid); + goto cleanup; + } + + validitySize = sqlite3_blob_bytes(*blobChunksValidity); + if (validitySize != p->chunk_size / CHAR_BIT) { + // IMP: V29362_13432 + vtab_set_error(&p->base, + VEC_INTERAL_ERROR + "validity blob size mismatch on " + "%s.%s.%lld, expected %lld but received %lld.", + p->schemaName, p->shadowChunksName, *chunk_rowid, + (i64)(p->chunk_size / CHAR_BIT), validitySize); + rc = SQLITE_ERROR; + goto cleanup; + } + + *bufferChunksValidity = sqlite3_malloc(validitySize); + if (!(*bufferChunksValidity)) { + vtab_set_error(&p->base, VEC_INTERAL_ERROR + "Could not allocate memory for validity bitmap"); + rc = SQLITE_NOMEM; + goto cleanup; + } + + rc = sqlite3_blob_read(*blobChunksValidity, (void *)*bufferChunksValidity, + validitySize, 0); + + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, + VEC_INTERAL_ERROR + "Could not read validity bitmap for %s.%s.%lld", + p->schemaName, p->shadowChunksName, *chunk_rowid); + goto cleanup; + } + + // find the next available offset, ie first `0` in the bitmap. + for (int i = 0; i < validitySize; i++) { + if ((*bufferChunksValidity)[i] == 0b11111111) + continue; + for (int j = 0; j < CHAR_BIT; j++) { + if (((((*bufferChunksValidity)[i] >> j) & 1) == 0)) { + *chunk_offset = (i * CHAR_BIT) + j; + goto done; + } + } + } + +done: + // latest chunk was full, so need to create a new one + if (*chunk_offset == -1) { + rc = vec0_new_chunk(p, partitionKeyValues, chunk_rowid); + if (rc != SQLITE_OK) { + // IMP: V08441_25279 + vtab_set_error(&p->base, + VEC_INTERAL_ERROR "Could not insert a new vector chunk"); + rc = SQLITE_ERROR; // otherwise raises a DatabaseError and not operational + // error? + goto cleanup; + } + *chunk_offset = 0; + + // blobChunksValidity and pValidity are stale, pointing to the previous + // (full) chunk. to re-assign them + rc = sqlite3_blob_close(*blobChunksValidity); + sqlite3_free((void *)*bufferChunksValidity); + *blobChunksValidity = NULL; + *bufferChunksValidity = NULL; + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, VEC_INTERAL_ERROR + "unknown error, blobChunksValidity could not be closed, " + "please file an issue."); + rc = SQLITE_ERROR; + goto cleanup; + } + + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, + "validity", *chunk_rowid, 1, blobChunksValidity); + if (rc != SQLITE_OK) { + vtab_set_error( + &p->base, + VEC_INTERAL_ERROR + "Could not open validity blob for newly created chunk %s.%s.%lld", + p->schemaName, p->shadowChunksName, *chunk_rowid); + goto cleanup; + } + validitySize = sqlite3_blob_bytes(*blobChunksValidity); + if (validitySize != p->chunk_size / CHAR_BIT) { + vtab_set_error(&p->base, + VEC_INTERAL_ERROR + "validity blob size mismatch for newly created chunk " + "%s.%s.%lld. Exepcted %lld, got %lld", + p->schemaName, p->shadowChunksName, *chunk_rowid, + p->chunk_size / CHAR_BIT, validitySize); + goto cleanup; + } + *bufferChunksValidity = sqlite3_malloc(validitySize); + rc = sqlite3_blob_read(*blobChunksValidity, (void *)*bufferChunksValidity, + validitySize, 0); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, + VEC_INTERAL_ERROR + "could not read validity blob newly created chunk " + "%s.%s.%lld", + p->schemaName, p->shadowChunksName, *chunk_rowid); + goto cleanup; + } + } + + rc = SQLITE_OK; + +cleanup: + return rc; +} + +/** + * @brief Write the vector data into the provided vector blob at the given + * offset + * + * @param blobVectors SQLite BLOB to write to + * @param chunk_offset the "offset" (ie validity bitmap position) to write the + * vector to + * @param bVector pointer to the vector containing data + * @param dimensions how many dimensions the vector has + * @param element_type the vector type + * @return result of sqlite3_blob_write, SQLITE_OK on success, otherwise failure + */ +static int +vec0_write_vector_to_vector_blob(sqlite3_blob *blobVectors, i64 chunk_offset, + const void *bVector, size_t dimensions, + enum VectorElementType element_type) { + int n; + int offset; + + switch (element_type) { + case SQLITE_VEC_ELEMENT_TYPE_FLOAT32: + n = dimensions * sizeof(f32); + offset = chunk_offset * dimensions * sizeof(f32); + break; + case SQLITE_VEC_ELEMENT_TYPE_INT8: + n = dimensions * sizeof(i8); + offset = chunk_offset * dimensions * sizeof(i8); + break; + case SQLITE_VEC_ELEMENT_TYPE_BIT: + n = dimensions / CHAR_BIT; + offset = chunk_offset * dimensions / CHAR_BIT; + break; + } + + return sqlite3_blob_write(blobVectors, bVector, n, offset); +} + +/** + * @brief + * + * @param p vec0 virtual table + * @param chunk_rowid: which chunk to write to + * @param chunk_offset: the offset inside the chunk to write the vector to. + * @param rowid: the rowid of the inserting row + * @param vectorDatas: array of the vector data to insert + * @param blobValidity: writeable validity blob of the row's assigned chunk. + * @param validity: snapshot buffer of the valdity column from the row's + * assigned chunk. + * @return int SQLITE_OK on success, error code on failure + */ +int vec0Update_InsertWriteFinalStep(vec0_vtab *p, i64 chunk_rowid, + i64 chunk_offset, i64 rowid, + void *vectorDatas[], + sqlite3_blob *blobChunksValidity, + const unsigned char *bufferChunksValidity) { + int rc, brc; + sqlite3_blob *blobChunksRowids = NULL; + + // mark the validity bit for this row in the chunk's validity bitmap + // Get the byte offset of the bitmap + char unsigned bx = bufferChunksValidity[chunk_offset / CHAR_BIT]; + // set the bit at the chunk_offset position inside that byte + bx = bx | (1 << (chunk_offset % CHAR_BIT)); + // write that 1 byte + rc = sqlite3_blob_write(blobChunksValidity, &bx, 1, chunk_offset / CHAR_BIT); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, VEC_INTERAL_ERROR "could not mark validity bit "); + return rc; + } + + // Go insert the vector data into the vector chunk shadow tables + for (int i = 0; i < p->numVectorColumns; i++) { + sqlite3_blob *blobVectors; + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowVectorChunksNames[i], + "vectors", chunk_rowid, 1, &blobVectors); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, "Error opening vector blob at %s.%s.%lld", + p->schemaName, p->shadowVectorChunksNames[i], chunk_rowid); + goto cleanup; + } + + i64 expected = + p->chunk_size * vector_column_byte_size(p->vector_columns[i]); + i64 actual = sqlite3_blob_bytes(blobVectors); + + if (actual != expected) { + // IMP: V16386_00456 + vtab_set_error( + &p->base, + VEC_INTERAL_ERROR + "vector blob size mismatch on %s.%s.%lld. Expected %lld, actual %lld", + p->schemaName, p->shadowVectorChunksNames[i], chunk_rowid, expected, + actual); + rc = SQLITE_ERROR; + // already error, can ignore result code + sqlite3_blob_close(blobVectors); + goto cleanup; + }; + + rc = vec0_write_vector_to_vector_blob( + blobVectors, chunk_offset, vectorDatas[i], + p->vector_columns[i].dimensions, p->vector_columns[i].element_type); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, + VEC_INTERAL_ERROR + "could not write vector blob on %s.%s.%lld", + p->schemaName, p->shadowVectorChunksNames[i], chunk_rowid); + rc = SQLITE_ERROR; + // already error, can ignore result code + sqlite3_blob_close(blobVectors); + goto cleanup; + } + rc = sqlite3_blob_close(blobVectors); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, + VEC_INTERAL_ERROR + "could not close vector blob on %s.%s.%lld", + p->schemaName, p->shadowVectorChunksNames[i], chunk_rowid); + rc = SQLITE_ERROR; + goto cleanup; + } + } + + // write the new rowid to the rowids column of the _chunks table + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, "rowids", + chunk_rowid, 1, &blobChunksRowids); + if (rc != SQLITE_OK) { + // IMP: V09221_26060 + vtab_set_error(&p->base, + VEC_INTERAL_ERROR "could not open rowids blob on %s.%s.%lld", + p->schemaName, p->shadowChunksName, chunk_rowid); + goto cleanup; + } + i64 expected = p->chunk_size * sizeof(i64); + i64 actual = sqlite3_blob_bytes(blobChunksRowids); + if (expected != actual) { + // IMP: V12779_29618 + vtab_set_error( + &p->base, + VEC_INTERAL_ERROR + "rowids blob size mismatch on %s.%s.%lld. Expected %lld, actual %lld", + p->schemaName, p->shadowChunksName, chunk_rowid, expected, actual); + rc = SQLITE_ERROR; + goto cleanup; + } + rc = sqlite3_blob_write(blobChunksRowids, &rowid, sizeof(i64), + chunk_offset * sizeof(i64)); + if (rc != SQLITE_OK) { + vtab_set_error( + &p->base, VEC_INTERAL_ERROR "could not write rowids blob on %s.%s.%lld", + p->schemaName, p->shadowChunksName, chunk_rowid); + rc = SQLITE_ERROR; + goto cleanup; + } + + // Now with all the vectors inserted, go back and update the _rowids table + // with the new chunk_rowid/chunk_offset values + rc = vec0_rowids_update_position(p, rowid, chunk_rowid, chunk_offset); + +cleanup: + brc = sqlite3_blob_close(blobChunksRowids); + if ((rc == SQLITE_OK) && (brc != SQLITE_OK)) { + vtab_set_error( + &p->base, VEC_INTERAL_ERROR "could not close rowids blob on %s.%s.%lld", + p->schemaName, p->shadowChunksName, chunk_rowid); + return brc; + } + return rc; +} + +int vec0_write_metadata_value(vec0_vtab *p, int metadata_column_idx, i64 rowid, i64 chunk_id, i64 chunk_offset, sqlite3_value * v, int isupdate) { + int rc; + struct Vec0MetadataColumnDefinition * metadata_column = &p->metadata_columns[metadata_column_idx]; + vec0_metadata_column_kind kind = metadata_column->kind; + + // verify input value matches column type + switch(kind) { + case VEC0_METADATA_COLUMN_KIND_BOOLEAN: { + if(sqlite3_value_type(v) != SQLITE_INTEGER || ((sqlite3_value_int(v) != 0) && (sqlite3_value_int(v) != 1))) { + rc = SQLITE_ERROR; + vtab_set_error(&p->base, "Expected 0 or 1 for BOOLEAN metadata column %.*s", metadata_column->name_length, metadata_column->name); + goto done; + } + break; + } + case VEC0_METADATA_COLUMN_KIND_INTEGER: { + if(sqlite3_value_type(v) != SQLITE_INTEGER) { + rc = SQLITE_ERROR; + vtab_set_error(&p->base, "Expected integer for INTEGER metadata column %.*s, received %s", metadata_column->name_length, metadata_column->name, type_name(sqlite3_value_type(v))); + goto done; + } + break; + } + case VEC0_METADATA_COLUMN_KIND_FLOAT: { + if(sqlite3_value_type(v) != SQLITE_FLOAT) { + rc = SQLITE_ERROR; + vtab_set_error(&p->base, "Expected float for FLOAT metadata column %.*s, received %s", metadata_column->name_length, metadata_column->name, type_name(sqlite3_value_type(v))); + goto done; + } + break; + } + case VEC0_METADATA_COLUMN_KIND_TEXT: { + if(sqlite3_value_type(v) != SQLITE_TEXT) { + rc = SQLITE_ERROR; + vtab_set_error(&p->base, "Expected text for TEXT metadata column %.*s, received %s", metadata_column->name_length, metadata_column->name, type_name(sqlite3_value_type(v))); + goto done; + } + break; + } + } + + sqlite3_blob * blobValue = NULL; + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowMetadataChunksNames[metadata_column_idx], "data", chunk_id, 1, &blobValue); + if(rc != SQLITE_OK) { + goto done; + } + + switch(kind) { + case VEC0_METADATA_COLUMN_KIND_BOOLEAN: { + u8 block; + int value = sqlite3_value_int(v); + rc = sqlite3_blob_read(blobValue, &block, sizeof(u8), (int) (chunk_offset / CHAR_BIT)); + if(rc != SQLITE_OK) { + goto done; + } + + if (value) { + block |= 1 << (chunk_offset % CHAR_BIT); + } else { + block &= ~(1 << (chunk_offset % CHAR_BIT)); + } + + rc = sqlite3_blob_write(blobValue, &block, sizeof(u8), chunk_offset / CHAR_BIT); + break; + } + case VEC0_METADATA_COLUMN_KIND_INTEGER: { + i64 value = sqlite3_value_int64(v); + rc = sqlite3_blob_write(blobValue, &value, sizeof(value), chunk_offset * sizeof(i64)); + break; + } + case VEC0_METADATA_COLUMN_KIND_FLOAT: { + double value = sqlite3_value_double(v); + rc = sqlite3_blob_write(blobValue, &value, sizeof(value), chunk_offset * sizeof(double)); + break; + } + case VEC0_METADATA_COLUMN_KIND_TEXT: { + int prev_n; + rc = sqlite3_blob_read(blobValue, &prev_n, sizeof(int), chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH); + if(rc != SQLITE_OK) { + goto done; + } + + const char * s = (const char *) sqlite3_value_text(v); + int n = sqlite3_value_bytes(v); + u8 view[VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + memset(view, 0, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH); + memcpy(view, &n, sizeof(int)); + memcpy(view+4, s, min(n, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH-4)); + + rc = sqlite3_blob_write(blobValue, &view, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH, chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH); + if(n > VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + const char * zSql; + + if(isupdate && (prev_n > VEC0_METADATA_TEXT_VIEW_DATA_LENGTH)) { + zSql = sqlite3_mprintf("UPDATE " VEC0_SHADOW_METADATA_TEXT_DATA_NAME " SET data = ?2 WHERE rowid = ?1", p->schemaName, p->tableName, metadata_column_idx); + }else { + zSql = sqlite3_mprintf("INSERT INTO " VEC0_SHADOW_METADATA_TEXT_DATA_NAME " (rowid, data) VALUES (?1, ?2)", p->schemaName, p->tableName, metadata_column_idx); + } + if(!zSql) { + rc = SQLITE_NOMEM; + goto done; + } + sqlite3_stmt * stmt; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + if(rc != SQLITE_OK) { + goto done; + } + sqlite3_bind_int64(stmt, 1, rowid); + sqlite3_bind_text(stmt, 2, s, n, SQLITE_STATIC); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + + if(rc != SQLITE_DONE) { + rc = SQLITE_ERROR; + goto done; + } + } + else if(prev_n > VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + const char * zSql = sqlite3_mprintf("DELETE FROM " VEC0_SHADOW_METADATA_TEXT_DATA_NAME " WHERE rowid = ?", p->schemaName, p->tableName, metadata_column_idx); + if(!zSql) { + rc = SQLITE_NOMEM; + goto done; + } + sqlite3_stmt * stmt; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + if(rc != SQLITE_OK) { + goto done; + } + sqlite3_bind_int64(stmt, 1, rowid); + rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + + if(rc != SQLITE_DONE) { + rc = SQLITE_ERROR; + goto done; + } + } + break; + } + } + + if(rc != SQLITE_OK) { + + } + rc = sqlite3_blob_close(blobValue); + if(rc != SQLITE_OK) { + goto done; + } + + done: + return rc; +} + + +/** + * @brief Handles INSERT INTO operations on a vec0 table. + * + * @return int SQLITE_OK on success, otherwise error code on failure + */ +int vec0Update_Insert(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv, + sqlite_int64 *pRowid) { + UNUSED_PARAMETER(argc); + vec0_vtab *p = (vec0_vtab *)pVTab; + int rc; + // Rowid for the inserted row, deterimined by the inserted ID + _rowids shadow + // table + i64 rowid; + + // Array to hold the vector data of the inserted row. Individual elements will + // have a lifetime bound to the argv[..] values. + void *vectorDatas[VEC0_MAX_VECTOR_COLUMNS]; + // Array to hold cleanup functions for vectorDatas[] + vector_cleanup cleanups[VEC0_MAX_VECTOR_COLUMNS]; + + sqlite3_value * partitionKeyValues[VEC0_MAX_PARTITION_COLUMNS]; + + // Rowid of the chunk in the _chunks shadow table that the row will be a part + // of. + i64 chunk_rowid; + // offset within the chunk where the rowid belongs + i64 chunk_offset; + + // a write-able blob of the validity column for the given chunk. Used to mark + // validity bit + sqlite3_blob *blobChunksValidity = NULL; + // buffer for the valididty column for the given chunk. Maybe not needed here? + const unsigned char *bufferChunksValidity = NULL; + int numReadVectors = 0; + + // Read all provided partition key values into partitionKeyValues + for (int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_PARTITION) { + continue; + } + int partition_key_idx = p->user_column_idxs[i]; + partitionKeyValues[partition_key_idx] = argv[2+VEC0_COLUMN_USERN_START + i]; + + int new_value_type = sqlite3_value_type(partitionKeyValues[partition_key_idx]); + if((new_value_type != SQLITE_NULL) && (new_value_type != p->paritition_columns[partition_key_idx].type)) { + // IMP: V11454_28292 + vtab_set_error( + pVTab, + "Parition key type mismatch: The partition key column %.*s has type %s, but %s was provided.", + p->paritition_columns[partition_key_idx].name_length, + p->paritition_columns[partition_key_idx].name, + type_name(p->paritition_columns[partition_key_idx].type), + type_name(new_value_type) + ); + rc = SQLITE_ERROR; + goto cleanup; + } + } + + // read all the inserted vectors into vectorDatas, validate their lengths. + for (int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_VECTOR) { + continue; + } + int vector_column_idx = p->user_column_idxs[i]; + sqlite3_value *valueVector = argv[2 + VEC0_COLUMN_USERN_START + i]; + size_t dimensions; + + char *pzError; + enum VectorElementType elementType; + rc = vector_from_value(valueVector, &vectorDatas[vector_column_idx], &dimensions, + &elementType, &cleanups[vector_column_idx], &pzError); + if (rc != SQLITE_OK) { + // IMP: V06519_23358 + vtab_set_error( + pVTab, "Inserted vector for the \"%.*s\" column is invalid: %z", + p->vector_columns[vector_column_idx].name_length, p->vector_columns[vector_column_idx].name, pzError); + rc = SQLITE_ERROR; + goto cleanup; + } + + numReadVectors++; + if (elementType != p->vector_columns[vector_column_idx].element_type) { + // IMP: V08221_25059 + vtab_set_error( + pVTab, + "Inserted vector for the \"%.*s\" column is expected to be of type " + "%s, but a %s vector was provided.", + p->vector_columns[i].name_length, p->vector_columns[i].name, + vector_subtype_name(p->vector_columns[i].element_type), + vector_subtype_name(elementType)); + rc = SQLITE_ERROR; + goto cleanup; + } + + if (dimensions != p->vector_columns[vector_column_idx].dimensions) { + // IMP: V01145_17984 + vtab_set_error( + pVTab, + "Dimension mismatch for inserted vector for the \"%.*s\" column. " + "Expected %d dimensions but received %d.", + p->vector_columns[vector_column_idx].name_length, p->vector_columns[vector_column_idx].name, + p->vector_columns[vector_column_idx].dimensions, dimensions); + rc = SQLITE_ERROR; + goto cleanup; + } + } + + // Cannot insert a value in the hidden "distance" column + if (sqlite3_value_type(argv[2 + vec0_column_distance_idx(p)]) != + SQLITE_NULL) { + // IMP: V24228_08298 + vtab_set_error(pVTab, + "A value was provided for the hidden \"distance\" column."); + rc = SQLITE_ERROR; + goto cleanup; + } + // Cannot insert a value in the hidden "k" column + if (sqlite3_value_type(argv[2 + vec0_column_k_idx(p)]) != SQLITE_NULL) { + // IMP: V11875_28713 + vtab_set_error(pVTab, "A value was provided for the hidden \"k\" column."); + rc = SQLITE_ERROR; + goto cleanup; + } + + // Step #1: Insert/get a rowid for this row, from the _rowids table. + rc = vec0Update_InsertRowidStep(p, argv[2 + VEC0_COLUMN_ID], &rowid); + if (rc != SQLITE_OK) { + goto cleanup; + } + + // Step #2: Find the next "available" position in the _chunks table for this + // row. + rc = vec0Update_InsertNextAvailableStep(p, partitionKeyValues, + &chunk_rowid, &chunk_offset, + &blobChunksValidity, + &bufferChunksValidity); + if (rc != SQLITE_OK) { + goto cleanup; + } + + // Step #3: With the next available chunk position, write out all the vectors + // to their specified location. + rc = vec0Update_InsertWriteFinalStep(p, chunk_rowid, chunk_offset, rowid, + vectorDatas, blobChunksValidity, + bufferChunksValidity); + if (rc != SQLITE_OK) { + goto cleanup; + } + + if(p->numAuxiliaryColumns > 0) { + sqlite3_stmt *stmt; + sqlite3_str * s = sqlite3_str_new(NULL); + sqlite3_str_appendf(s, "INSERT INTO " VEC0_SHADOW_AUXILIARY_NAME "(rowid ", p->schemaName, p->tableName); + for(int i = 0; i < p->numAuxiliaryColumns; i++) { + sqlite3_str_appendf(s, ", value%02d", i); + } + sqlite3_str_appendall(s, ") VALUES (? "); + for(int i = 0; i < p->numAuxiliaryColumns; i++) { + sqlite3_str_appendall(s, ", ?"); + } + sqlite3_str_appendall(s, ")"); + char * zSql = sqlite3_str_finish(s); + // TODO double check error handling ehre + if(!zSql) { + rc = SQLITE_NOMEM; + goto cleanup; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + if(rc != SQLITE_OK) { + goto cleanup; + } + sqlite3_bind_int64(stmt, 1, rowid); + + for (int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY) { + continue; + } + int auxiliary_key_idx = p->user_column_idxs[i]; + sqlite3_value * v = argv[2+VEC0_COLUMN_USERN_START + i]; + int v_type = sqlite3_value_type(v); + if(v_type != SQLITE_NULL && (v_type != p->auxiliary_columns[auxiliary_key_idx].type)) { + sqlite3_finalize(stmt); + rc = SQLITE_CONSTRAINT; + vtab_set_error( + pVTab, + "Auxiliary column type mismatch: The auxiliary column %.*s has type %s, but %s was provided.", + p->auxiliary_columns[auxiliary_key_idx].name_length, + p->auxiliary_columns[auxiliary_key_idx].name, + type_name(p->auxiliary_columns[auxiliary_key_idx].type), + type_name(v_type) + ); + goto cleanup; + } + // first 1 is for 1-based indexing on sqlite3_bind_*, second 1 is to account for initial rowid parameter + sqlite3_bind_value(stmt, 1 + 1 + auxiliary_key_idx, v); + } + + rc = sqlite3_step(stmt); + if(rc != SQLITE_DONE) { + sqlite3_finalize(stmt); + rc = SQLITE_ERROR; + goto cleanup; + } + sqlite3_finalize(stmt); + } + + + for(int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_METADATA) { + continue; + } + int metadata_idx = p->user_column_idxs[i]; + sqlite3_value *v = argv[2 + VEC0_COLUMN_USERN_START + i]; + rc = vec0_write_metadata_value(p, metadata_idx, rowid, chunk_rowid, chunk_offset, v, 0); + if(rc != SQLITE_OK) { + goto cleanup; + } + } + + *pRowid = rowid; + rc = SQLITE_OK; + +cleanup: + for (int i = 0; i < numReadVectors; i++) { + cleanups[i](vectorDatas[i]); + } + sqlite3_free((void *)bufferChunksValidity); + int brc = sqlite3_blob_close(blobChunksValidity); + if ((rc == SQLITE_OK) && (brc != SQLITE_OK)) { + vtab_set_error(&p->base, + VEC_INTERAL_ERROR "unknown error, blobChunksValidity could " + "not be closed, please file an issue"); + return brc; + } + return rc; +} + +int vec0Update_Delete_ClearValidity(vec0_vtab *p, i64 chunk_id, + u64 chunk_offset) { + int rc, brc; + sqlite3_blob *blobChunksValidity = NULL; + char unsigned bx; + int validityOffset = chunk_offset / CHAR_BIT; + + // 2. ensure chunks.validity bit is 1, then set to 0 + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowChunksName, "validity", + chunk_id, 1, &blobChunksValidity); + if (rc != SQLITE_OK) { + // IMP: V26002_10073 + vtab_set_error(&p->base, "could not open validity blob for %s.%s.%lld", + p->schemaName, p->shadowChunksName, chunk_id); + return SQLITE_ERROR; + } + // will skip the sqlite3_blob_bytes(blobChunksValidity) check for now, + // the read below would catch it + + rc = sqlite3_blob_read(blobChunksValidity, &bx, sizeof(bx), validityOffset); + if (rc != SQLITE_OK) { + // IMP: V21193_05263 + vtab_set_error( + &p->base, "could not read validity blob for %s.%s.%lld at %d", + p->schemaName, p->shadowChunksName, chunk_id, validityOffset); + goto cleanup; + } + if (!(bx >> (chunk_offset % CHAR_BIT))) { + // IMP: V21193_05263 + rc = SQLITE_ERROR; + vtab_set_error( + &p->base, + "vec0 deletion error: validity bit is not set for %s.%s.%lld at %d", + p->schemaName, p->shadowChunksName, chunk_id, validityOffset); + goto cleanup; + } + char unsigned mask = ~(1 << (chunk_offset % CHAR_BIT)); + char result = bx & mask; + rc = sqlite3_blob_write(blobChunksValidity, &result, sizeof(bx), + validityOffset); + if (rc != SQLITE_OK) { + vtab_set_error( + &p->base, "could not write to validity blob for %s.%s.%lld at %d", + p->schemaName, p->shadowChunksName, chunk_id, validityOffset); + goto cleanup; + } + +cleanup: + + brc = sqlite3_blob_close(blobChunksValidity); + if (rc != SQLITE_OK) + return rc; + if (brc != SQLITE_OK) { + vtab_set_error(&p->base, + "vec0 deletion error: Error commiting validity blob " + "transaction on %s.%s.%lld at %d", + p->schemaName, p->shadowChunksName, chunk_id, + validityOffset); + return brc; + } + return SQLITE_OK; +} + +int vec0Update_Delete_DeleteRowids(vec0_vtab *p, i64 rowid) { + int rc; + sqlite3_stmt *stmt = NULL; + + char *zSql = + sqlite3_mprintf("DELETE FROM " VEC0_SHADOW_ROWIDS_NAME " WHERE rowid = ?", + p->schemaName, p->tableName); + if (!zSql) { + return SQLITE_NOMEM; + } + + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + goto cleanup; + } + sqlite3_bind_int64(stmt, 1, rowid); + rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + goto cleanup; + } + rc = SQLITE_OK; + +cleanup: + sqlite3_finalize(stmt); + return rc; +} + +int vec0Update_Delete_DeleteAux(vec0_vtab *p, i64 rowid) { + int rc; + sqlite3_stmt *stmt = NULL; + + char *zSql = + sqlite3_mprintf("DELETE FROM " VEC0_SHADOW_AUXILIARY_NAME " WHERE rowid = ?", + p->schemaName, p->tableName); + if (!zSql) { + return SQLITE_NOMEM; + } + + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + sqlite3_free(zSql); + if (rc != SQLITE_OK) { + goto cleanup; + } + sqlite3_bind_int64(stmt, 1, rowid); + rc = sqlite3_step(stmt); + if (rc != SQLITE_DONE) { + goto cleanup; + } + rc = SQLITE_OK; + +cleanup: + sqlite3_finalize(stmt); + return rc; +} + +int vec0Update_Delete_ClearMetadata(vec0_vtab *p, int metadata_idx, i64 rowid, i64 chunk_id, + u64 chunk_offset) { + int rc; + sqlite3_blob * blobValue; + vec0_metadata_column_kind kind = p->metadata_columns[metadata_idx].kind; + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowMetadataChunksNames[metadata_idx], "data", chunk_id, 1, &blobValue); + if(rc != SQLITE_OK) { + return rc; + } + + switch(kind) { + case VEC0_METADATA_COLUMN_KIND_BOOLEAN: { + u8 block; + rc = sqlite3_blob_read(blobValue, &block, sizeof(u8), (int) (chunk_offset / CHAR_BIT)); + if(rc != SQLITE_OK) { + goto done; + } + + block &= ~(1 << (chunk_offset % CHAR_BIT)); + rc = sqlite3_blob_write(blobValue, &block, sizeof(u8), chunk_offset / CHAR_BIT); + break; + } + case VEC0_METADATA_COLUMN_KIND_INTEGER: { + i64 v = 0; + rc = sqlite3_blob_write(blobValue, &v, sizeof(v), chunk_offset * sizeof(i64)); + break; + } + case VEC0_METADATA_COLUMN_KIND_FLOAT: { + double v = 0; + rc = sqlite3_blob_write(blobValue, &v, sizeof(v), chunk_offset * sizeof(double)); + break; + } + case VEC0_METADATA_COLUMN_KIND_TEXT: { + int n; + rc = sqlite3_blob_read(blobValue, &n, sizeof(int), chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH); + if(rc != SQLITE_OK) { + goto done; + } + + u8 view[VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH]; + memset(view, 0, VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH); + rc = sqlite3_blob_write(blobValue, &view, sizeof(view), chunk_offset * VEC0_METADATA_TEXT_VIEW_BUFFER_LENGTH); + if(rc != SQLITE_OK) { + goto done; + } + + if(n > VEC0_METADATA_TEXT_VIEW_DATA_LENGTH) { + const char * zSql = sqlite3_mprintf("DELETE FROM " VEC0_SHADOW_METADATA_TEXT_DATA_NAME " WHERE rowid = ?", p->schemaName, p->tableName, metadata_idx); + if(!zSql) { + rc = SQLITE_NOMEM; + goto done; + } + sqlite3_stmt * stmt; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + if(rc != SQLITE_OK) { + goto done; + } + sqlite3_bind_int64(stmt, 1, rowid); + rc = sqlite3_step(stmt); + if(rc != SQLITE_DONE) { + rc = SQLITE_ERROR; + goto done; + } + sqlite3_finalize(stmt); + } + break; + } + } + int rc2; + done: + rc2 = sqlite3_blob_close(blobValue); + if(rc == SQLITE_OK) { + return rc2; + } + return rc; +} + +int vec0Update_Delete(sqlite3_vtab *pVTab, sqlite3_value *idValue) { + vec0_vtab *p = (vec0_vtab *)pVTab; + int rc; + i64 rowid; + i64 chunk_id; + i64 chunk_offset; + + if (p->pkIsText) { + rc = vec0_rowid_from_id(p, idValue, &rowid); + if (rc != SQLITE_OK) { + return rc; + } + } else { + rowid = sqlite3_value_int64(idValue); + } + + // 1. Find chunk position for given rowid + // 2. Ensure that validity bit for position is 1, then set to 0 + // 3. Zero out rowid in chunks.rowid + // 4. Zero out vector data in all vector column chunks + // 5. Delete value in _rowids table + + // 1. get chunk_id and chunk_offset from _rowids + rc = vec0_get_chunk_position(p, rowid, NULL, &chunk_id, &chunk_offset); + if (rc != SQLITE_OK) { + return rc; + } + + rc = vec0Update_Delete_ClearValidity(p, chunk_id, chunk_offset); + if (rc != SQLITE_OK) { + return rc; + } + + // 3. zero out rowid in chunks.rowids + // https://github.com/asg017/sqlite-vec/issues/54 + + // 4. zero out any data in vector chunks tables + // https://github.com/asg017/sqlite-vec/issues/54 + + // 5. delete from _rowids table + rc = vec0Update_Delete_DeleteRowids(p, rowid); + if (rc != SQLITE_OK) { + return rc; + } + + // 6. delete any auxiliary rows + if(p->numAuxiliaryColumns > 0) { + rc = vec0Update_Delete_DeleteAux(p, rowid); + if (rc != SQLITE_OK) { + return rc; + } + } + + // 6. delete metadata + for(int i = 0; i < p->numMetadataColumns; i++) { + rc = vec0Update_Delete_ClearMetadata(p, i, rowid, chunk_id, chunk_offset); + } + + return SQLITE_OK; +} + +int vec0Update_UpdateAuxColumn(vec0_vtab *p, int auxiliary_column_idx, sqlite3_value * value, i64 rowid) { + int rc; + sqlite3_stmt *stmt; + const char * zSql = sqlite3_mprintf("UPDATE " VEC0_SHADOW_AUXILIARY_NAME " SET value%02d = ? WHERE rowid = ?", p->schemaName, p->tableName, auxiliary_column_idx); + if(!zSql) { + return SQLITE_NOMEM; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, NULL); + if(rc != SQLITE_OK) { + return rc; + } + sqlite3_bind_value(stmt, 1, value); + sqlite3_bind_int64(stmt, 2, rowid); + rc = sqlite3_step(stmt); + if(rc != SQLITE_DONE) { + sqlite3_finalize(stmt); + return SQLITE_ERROR; + } + sqlite3_finalize(stmt); + return SQLITE_OK; +} + +int vec0Update_UpdateVectorColumn(vec0_vtab *p, i64 chunk_id, i64 chunk_offset, + int i, sqlite3_value *valueVector) { + int rc; + + sqlite3_blob *blobVectors = NULL; + + char *pzError; + size_t dimensions; + enum VectorElementType elementType; + void *vector; + vector_cleanup cleanup = vector_cleanup_noop; + // https://github.com/asg017/sqlite-vec/issues/53 + rc = vector_from_value(valueVector, &vector, &dimensions, &elementType, + &cleanup, &pzError); + if (rc != SQLITE_OK) { + // IMP: V15203_32042 + vtab_set_error( + &p->base, "Updated vector for the \"%.*s\" column is invalid: %z", + p->vector_columns[i].name_length, p->vector_columns[i].name, pzError); + rc = SQLITE_ERROR; + goto cleanup; + } + if (elementType != p->vector_columns[i].element_type) { + // IMP: V03643_20481 + vtab_set_error( + &p->base, + "Updated vector for the \"%.*s\" column is expected to be of type " + "%s, but a %s vector was provided.", + p->vector_columns[i].name_length, p->vector_columns[i].name, + vector_subtype_name(p->vector_columns[i].element_type), + vector_subtype_name(elementType)); + rc = SQLITE_ERROR; + goto cleanup; + } + if (dimensions != p->vector_columns[i].dimensions) { + // IMP: V25739_09810 + vtab_set_error( + &p->base, + "Dimension mismatch for new updated vector for the \"%.*s\" column. " + "Expected %d dimensions but received %d.", + p->vector_columns[i].name_length, p->vector_columns[i].name, + p->vector_columns[i].dimensions, dimensions); + rc = SQLITE_ERROR; + goto cleanup; + } + + rc = sqlite3_blob_open(p->db, p->schemaName, p->shadowVectorChunksNames[i], + "vectors", chunk_id, 1, &blobVectors); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, "Could not open vectors blob for %s.%s.%lld", + p->schemaName, p->shadowVectorChunksNames[i], chunk_id); + goto cleanup; + } + rc = vec0_write_vector_to_vector_blob(blobVectors, chunk_offset, vector, + p->vector_columns[i].dimensions, + p->vector_columns[i].element_type); + if (rc != SQLITE_OK) { + vtab_set_error(&p->base, "Could not write to vectors blob for %s.%s.%lld", + p->schemaName, p->shadowVectorChunksNames[i], chunk_id); + goto cleanup; + } + +cleanup: + cleanup(vector); + int brc = sqlite3_blob_close(blobVectors); + if (rc != SQLITE_OK) { + return rc; + } + if (brc != SQLITE_OK) { + vtab_set_error( + &p->base, + "Could not commit blob transaction for vectors blob for %s.%s.%lld", + p->schemaName, p->shadowVectorChunksNames[i], chunk_id); + return brc; + } + return SQLITE_OK; +} + +int vec0Update_Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(argc); + vec0_vtab *p = (vec0_vtab *)pVTab; + int rc; + i64 chunk_id; + i64 chunk_offset; + + i64 rowid; + if (p->pkIsText) { + const char *a = (const char *)sqlite3_value_text(argv[0]); + const char *b = (const char *)sqlite3_value_text(argv[1]); + // IMP: V08886_25725 + if ((sqlite3_value_bytes(argv[0]) != sqlite3_value_bytes(argv[1])) || + strncmp(a, b, sqlite3_value_bytes(argv[0])) != 0) { + vtab_set_error(pVTab, + "UPDATEs on vec0 primary key values are not allowed."); + return SQLITE_ERROR; + } + rc = vec0_rowid_from_id(p, argv[0], &rowid); + if (rc != SQLITE_OK) { + return rc; + } + } else { + rowid = sqlite3_value_int64(argv[0]); + } + + // 1) get chunk_id and chunk_offset from _rowids + rc = vec0_get_chunk_position(p, rowid, NULL, &chunk_id, &chunk_offset); + if (rc != SQLITE_OK) { + return rc; + } + + // 2) update any partition key values + for (int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_PARTITION) { + continue; + } + sqlite3_value * value = argv[2+VEC0_COLUMN_USERN_START + i]; + if(sqlite3_value_nochange(value)) { + continue; + } + vtab_set_error(pVTab, "UPDATE on partition key columns are not supported yet. "); + return SQLITE_ERROR; + } + + // 3) handle auxiliary column updates + for (int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_AUXILIARY) { + continue; + } + int auxiliary_column_idx = p->user_column_idxs[i]; + sqlite3_value * value = argv[2+VEC0_COLUMN_USERN_START + i]; + if(sqlite3_value_nochange(value)) { + continue; + } + rc = vec0Update_UpdateAuxColumn(p, auxiliary_column_idx, value, rowid); + if(rc != SQLITE_OK) { + return SQLITE_ERROR; + } + } + + // 4) handle metadata column updates + for (int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_METADATA) { + continue; + } + int metadata_column_idx = p->user_column_idxs[i]; + sqlite3_value * value = argv[2+VEC0_COLUMN_USERN_START + i]; + if(sqlite3_value_nochange(value)) { + continue; + } + rc = vec0_write_metadata_value(p, metadata_column_idx, rowid, chunk_id, chunk_offset, value, 1); + if(rc != SQLITE_OK) { + return rc; + } + } + + // 5) iterate over all new vectors, update the vectors + for (int i = 0; i < vec0_num_defined_user_columns(p); i++) { + if(p->user_column_kinds[i] != SQLITE_VEC0_USER_COLUMN_KIND_VECTOR) { + continue; + } + int vector_idx = p->user_column_idxs[i]; + sqlite3_value *valueVector = argv[2 + VEC0_COLUMN_USERN_START + i]; + // in vec0Column, we check sqlite3_vtab_nochange() on vector columns. + // If the vector column isn't being changed, we return NULL; + // That's not great, that means vector columns can never be NULLABLE + // (bc we cant distinguish if an updated vector is truly NULL or nochange). + // Also it means that if someone tries to run `UPDATE v SET X = NULL`, + // we can't effectively detect and raise an error. + // A better solution would be to use a custom result_type for "empty", + // but subtypes don't appear to survive xColumn -> xUpdate, it's always 0. + // So for now, we'll just use NULL and warn people to not SET X = NULL + // in the docs. + if (sqlite3_value_type(valueVector) == SQLITE_NULL) { + continue; + } + + rc = vec0Update_UpdateVectorColumn(p, chunk_id, chunk_offset, vector_idx, + valueVector); + if (rc != SQLITE_OK) { + return SQLITE_ERROR; + } + } + + return SQLITE_OK; +} + +static int vec0Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv, + sqlite_int64 *pRowid) { + // DELETE operation + if (argc == 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL) { + return vec0Update_Delete(pVTab, argv[0]); + } + // INSERT operation + else if (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL) { + return vec0Update_Insert(pVTab, argc, argv, pRowid); + } + // UPDATE operation + else if (argc > 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL) { + return vec0Update_Update(pVTab, argc, argv); + } else { + vtab_set_error(pVTab, "Unrecognized xUpdate operation provided for vec0."); + return SQLITE_ERROR; + } +} + +static int vec0ShadowName(const char *zName) { + static const char *azName[] = { + "rowids", "chunks", "auxiliary", "info", + + // Up to VEC0_MAX_METADATA_COLUMNS + // TODO be smarter about this man + "metadatachunks00", + "metadatachunks01", + "metadatachunks02", + "metadatachunks03", + "metadatachunks04", + "metadatachunks05", + "metadatachunks06", + "metadatachunks07", + "metadatachunks08", + "metadatachunks09", + "metadatachunks10", + "metadatachunks11", + "metadatachunks12", + "metadatachunks13", + "metadatachunks14", + "metadatachunks15", + + // Up to + "metadatatext00", + "metadatatext01", + "metadatatext02", + "metadatatext03", + "metadatatext04", + "metadatatext05", + "metadatatext06", + "metadatatext07", + "metadatatext08", + "metadatatext09", + "metadatatext10", + "metadatatext11", + "metadatatext12", + "metadatatext13", + "metadatatext14", + "metadatatext15", + }; + + for (size_t i = 0; i < sizeof(azName) / sizeof(azName[0]); i++) { + if (sqlite3_stricmp(zName, azName[i]) == 0) + return 1; + } + //for(size_t i = 0; i < )"vector_chunks", "metadatachunks" + return 0; +} + +static int vec0Begin(sqlite3_vtab *pVTab) { + UNUSED_PARAMETER(pVTab); + return SQLITE_OK; +} +static int vec0Sync(sqlite3_vtab *pVTab) { + UNUSED_PARAMETER(pVTab); + vec0_vtab *p = (vec0_vtab *)pVTab; + if (p->stmtLatestChunk) { + sqlite3_finalize(p->stmtLatestChunk); + p->stmtLatestChunk = NULL; + } + if (p->stmtRowidsInsertRowid) { + sqlite3_finalize(p->stmtRowidsInsertRowid); + p->stmtRowidsInsertRowid = NULL; + } + if (p->stmtRowidsInsertId) { + sqlite3_finalize(p->stmtRowidsInsertId); + p->stmtRowidsInsertId = NULL; + } + if (p->stmtRowidsUpdatePosition) { + sqlite3_finalize(p->stmtRowidsUpdatePosition); + p->stmtRowidsUpdatePosition = NULL; + } + if (p->stmtRowidsGetChunkPosition) { + sqlite3_finalize(p->stmtRowidsGetChunkPosition); + p->stmtRowidsGetChunkPosition = NULL; + } + return SQLITE_OK; +} +static int vec0Commit(sqlite3_vtab *pVTab) { + UNUSED_PARAMETER(pVTab); + return SQLITE_OK; +} +static int vec0Rollback(sqlite3_vtab *pVTab) { + UNUSED_PARAMETER(pVTab); + return SQLITE_OK; +} + +static sqlite3_module vec0Module = { + /* iVersion */ 3, + /* xCreate */ vec0Create, + /* xConnect */ vec0Connect, + /* xBestIndex */ vec0BestIndex, + /* xDisconnect */ vec0Disconnect, + /* xDestroy */ vec0Destroy, + /* xOpen */ vec0Open, + /* xClose */ vec0Close, + /* xFilter */ vec0Filter, + /* xNext */ vec0Next, + /* xEof */ vec0Eof, + /* xColumn */ vec0Column, + /* xRowid */ vec0Rowid, + /* xUpdate */ vec0Update, + /* xBegin */ vec0Begin, + /* xSync */ vec0Sync, + /* xCommit */ vec0Commit, + /* xRollback */ vec0Rollback, + /* xFindFunction */ 0, + /* xRename */ 0, // https://github.com/asg017/sqlite-vec/issues/43 + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ vec0ShadowName, +#if SQLITE_VERSION_NUMBER >= 3044000 + /* xIntegrity */ 0, // https://github.com/asg017/sqlite-vec/issues/44 +#endif +}; +#pragma endregion + +static char *POINTER_NAME_STATIC_BLOB_DEF = "vec0-static_blob_def"; +struct static_blob_definition { + void *p; + size_t dimensions; + size_t nvectors; + enum VectorElementType element_type; +}; +static void vec_static_blob_from_raw(sqlite3_context *context, int argc, + sqlite3_value **argv) { + + assert(argc == 4); + struct static_blob_definition *p; + p = sqlite3_malloc(sizeof(*p)); + if (!p) { + sqlite3_result_error_nomem(context); + return; + } + memset(p, 0, sizeof(*p)); + p->p = (void *)sqlite3_value_int64(argv[0]); + p->element_type = SQLITE_VEC_ELEMENT_TYPE_FLOAT32; + p->dimensions = sqlite3_value_int64(argv[2]); + p->nvectors = sqlite3_value_int64(argv[3]); + sqlite3_result_pointer(context, p, POINTER_NAME_STATIC_BLOB_DEF, + sqlite3_free); +} +#pragma region vec_static_blobs() table function + +#define MAX_STATIC_BLOBS 16 + +typedef struct static_blob static_blob; +struct static_blob { + char *name; + void *p; + size_t dimensions; + size_t nvectors; + enum VectorElementType element_type; +}; + +typedef struct vec_static_blob_data vec_static_blob_data; +struct vec_static_blob_data { + static_blob static_blobs[MAX_STATIC_BLOBS]; +}; + +typedef struct vec_static_blobs_vtab vec_static_blobs_vtab; +struct vec_static_blobs_vtab { + sqlite3_vtab base; + vec_static_blob_data *data; +}; + +typedef struct vec_static_blobs_cursor vec_static_blobs_cursor; +struct vec_static_blobs_cursor { + sqlite3_vtab_cursor base; + sqlite3_int64 iRowid; +}; + +static int vec_static_blobsConnect(sqlite3 *db, void *pAux, int argc, + const char *const *argv, + sqlite3_vtab **ppVtab, char **pzErr) { + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + UNUSED_PARAMETER(pzErr); + + vec_static_blobs_vtab *pNew; +#define VEC_STATIC_BLOBS_NAME 0 +#define VEC_STATIC_BLOBS_DATA 1 +#define VEC_STATIC_BLOBS_DIMENSIONS 2 +#define VEC_STATIC_BLOBS_COUNT 3 + int rc = sqlite3_declare_vtab( + db, "CREATE TABLE x(name, data, dimensions hidden, count hidden)"); + if (rc == SQLITE_OK) { + pNew = sqlite3_malloc(sizeof(*pNew)); + *ppVtab = (sqlite3_vtab *)pNew; + if (pNew == 0) + return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->data = pAux; + } + return rc; +} + +static int vec_static_blobsDisconnect(sqlite3_vtab *pVtab) { + vec_static_blobs_vtab *p = (vec_static_blobs_vtab *)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +static int vec_static_blobsUpdate(sqlite3_vtab *pVTab, int argc, + sqlite3_value **argv, sqlite_int64 *pRowid) { + UNUSED_PARAMETER(pRowid); + vec_static_blobs_vtab *p = (vec_static_blobs_vtab *)pVTab; + // DELETE operation + if (argc == 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL) { + return SQLITE_ERROR; + } + // INSERT operation + else if (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL) { + const char *key = + (const char *)sqlite3_value_text(argv[2 + VEC_STATIC_BLOBS_NAME]); + int idx = -1; + for (int i = 0; i < MAX_STATIC_BLOBS; i++) { + if (!p->data->static_blobs[i].name) { + p->data->static_blobs[i].name = sqlite3_mprintf("%s", key); + idx = i; + break; + } + } + if (idx < 0) + abort(); + struct static_blob_definition *def = sqlite3_value_pointer( + argv[2 + VEC_STATIC_BLOBS_DATA], POINTER_NAME_STATIC_BLOB_DEF); + p->data->static_blobs[idx].p = def->p; + p->data->static_blobs[idx].dimensions = def->dimensions; + p->data->static_blobs[idx].nvectors = def->nvectors; + p->data->static_blobs[idx].element_type = def->element_type; + + return SQLITE_OK; + } + // UPDATE operation + else if (argc > 1 && sqlite3_value_type(argv[0]) != SQLITE_NULL) { + return SQLITE_ERROR; + } + return SQLITE_ERROR; +} + +static int vec_static_blobsOpen(sqlite3_vtab *p, + sqlite3_vtab_cursor **ppCursor) { + UNUSED_PARAMETER(p); + vec_static_blobs_cursor *pCur; + pCur = sqlite3_malloc(sizeof(*pCur)); + if (pCur == 0) + return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +static int vec_static_blobsClose(sqlite3_vtab_cursor *cur) { + vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur; + sqlite3_free(pCur); + return SQLITE_OK; +} + +static int vec_static_blobsBestIndex(sqlite3_vtab *pVTab, + sqlite3_index_info *pIdxInfo) { + UNUSED_PARAMETER(pVTab); + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = (double)10; + pIdxInfo->estimatedRows = 10; + return SQLITE_OK; +} + +static int vec_static_blobsNext(sqlite3_vtab_cursor *cur); +static int vec_static_blobsFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum, + const char *idxStr, int argc, + sqlite3_value **argv) { + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)pVtabCursor; + pCur->iRowid = -1; + vec_static_blobsNext(pVtabCursor); + return SQLITE_OK; +} + +static int vec_static_blobsRowid(sqlite3_vtab_cursor *cur, + sqlite_int64 *pRowid) { + vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +static int vec_static_blobsNext(sqlite3_vtab_cursor *cur) { + vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur; + vec_static_blobs_vtab *p = (vec_static_blobs_vtab *)pCur->base.pVtab; + pCur->iRowid++; + while (pCur->iRowid < MAX_STATIC_BLOBS) { + if (p->data->static_blobs[pCur->iRowid].name) { + return SQLITE_OK; + } + pCur->iRowid++; + } + return SQLITE_OK; +} + +static int vec_static_blobsEof(sqlite3_vtab_cursor *cur) { + vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur; + return pCur->iRowid >= MAX_STATIC_BLOBS; +} + +static int vec_static_blobsColumn(sqlite3_vtab_cursor *cur, + sqlite3_context *context, int i) { + vec_static_blobs_cursor *pCur = (vec_static_blobs_cursor *)cur; + vec_static_blobs_vtab *p = (vec_static_blobs_vtab *)cur->pVtab; + switch (i) { + case VEC_STATIC_BLOBS_NAME: + sqlite3_result_text(context, p->data->static_blobs[pCur->iRowid].name, -1, + SQLITE_TRANSIENT); + break; + case VEC_STATIC_BLOBS_DATA: + sqlite3_result_null(context); + break; + case VEC_STATIC_BLOBS_DIMENSIONS: + sqlite3_result_int64(context, + p->data->static_blobs[pCur->iRowid].dimensions); + break; + case VEC_STATIC_BLOBS_COUNT: + sqlite3_result_int64(context, p->data->static_blobs[pCur->iRowid].nvectors); + break; + } + return SQLITE_OK; +} + +static sqlite3_module vec_static_blobsModule = { + /* iVersion */ 3, + /* xCreate */ 0, + /* xConnect */ vec_static_blobsConnect, + /* xBestIndex */ vec_static_blobsBestIndex, + /* xDisconnect */ vec_static_blobsDisconnect, + /* xDestroy */ 0, + /* xOpen */ vec_static_blobsOpen, + /* xClose */ vec_static_blobsClose, + /* xFilter */ vec_static_blobsFilter, + /* xNext */ vec_static_blobsNext, + /* xEof */ vec_static_blobsEof, + /* xColumn */ vec_static_blobsColumn, + /* xRowid */ vec_static_blobsRowid, + /* xUpdate */ vec_static_blobsUpdate, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, +#if SQLITE_VERSION_NUMBER >= 3044000 + /* xIntegrity */ 0 +#endif +}; +#pragma endregion + +#pragma region vec_static_blob_entries() table function + +typedef struct vec_static_blob_entries_vtab vec_static_blob_entries_vtab; +struct vec_static_blob_entries_vtab { + sqlite3_vtab base; + static_blob *blob; +}; +typedef enum { + VEC_SBE__QUERYPLAN_FULLSCAN = 1, + VEC_SBE__QUERYPLAN_KNN = 2 +} vec_sbe_query_plan; + +struct sbe_query_knn_data { + i64 k; + i64 k_used; + // Array of rowids of size k. Must be freed with sqlite3_free(). + i32 *rowids; + // Array of distances of size k. Must be freed with sqlite3_free(). + f32 *distances; + i64 current_idx; +}; +void sbe_query_knn_data_clear(struct sbe_query_knn_data *knn_data) { + if (!knn_data) + return; + + if (knn_data->rowids) { + sqlite3_free(knn_data->rowids); + knn_data->rowids = NULL; + } + if (knn_data->distances) { + sqlite3_free(knn_data->distances); + knn_data->distances = NULL; + } +} + +typedef struct vec_static_blob_entries_cursor vec_static_blob_entries_cursor; +struct vec_static_blob_entries_cursor { + sqlite3_vtab_cursor base; + sqlite3_int64 iRowid; + vec_sbe_query_plan query_plan; + struct sbe_query_knn_data *knn_data; +}; + +static int vec_static_blob_entriesConnect(sqlite3 *db, void *pAux, int argc, + const char *const *argv, + sqlite3_vtab **ppVtab, char **pzErr) { + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + UNUSED_PARAMETER(pzErr); + vec_static_blob_data *blob_data = pAux; + int idx = -1; + for (int i = 0; i < MAX_STATIC_BLOBS; i++) { + if (!blob_data->static_blobs[i].name) + continue; + if (strncmp(blob_data->static_blobs[i].name, argv[3], + strlen(blob_data->static_blobs[i].name)) == 0) { + idx = i; + break; + } + } + if (idx < 0) + abort(); + vec_static_blob_entries_vtab *pNew; +#define VEC_STATIC_BLOB_ENTRIES_VECTOR 0 +#define VEC_STATIC_BLOB_ENTRIES_DISTANCE 1 +#define VEC_STATIC_BLOB_ENTRIES_K 2 + int rc = sqlite3_declare_vtab( + db, "CREATE TABLE x(vector, distance hidden, k hidden)"); + if (rc == SQLITE_OK) { + pNew = sqlite3_malloc(sizeof(*pNew)); + *ppVtab = (sqlite3_vtab *)pNew; + if (pNew == 0) + return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->blob = &blob_data->static_blobs[idx]; + } + return rc; +} + +static int vec_static_blob_entriesCreate(sqlite3 *db, void *pAux, int argc, + const char *const *argv, + sqlite3_vtab **ppVtab, char **pzErr) { + return vec_static_blob_entriesConnect(db, pAux, argc, argv, ppVtab, pzErr); +} + +static int vec_static_blob_entriesDisconnect(sqlite3_vtab *pVtab) { + vec_static_blob_entries_vtab *p = (vec_static_blob_entries_vtab *)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +static int vec_static_blob_entriesOpen(sqlite3_vtab *p, + sqlite3_vtab_cursor **ppCursor) { + UNUSED_PARAMETER(p); + vec_static_blob_entries_cursor *pCur; + pCur = sqlite3_malloc(sizeof(*pCur)); + if (pCur == 0) + return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +static int vec_static_blob_entriesClose(sqlite3_vtab_cursor *cur) { + vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur; + sqlite3_free(pCur->knn_data); + sqlite3_free(pCur); + return SQLITE_OK; +} + +static int vec_static_blob_entriesBestIndex(sqlite3_vtab *pVTab, + sqlite3_index_info *pIdxInfo) { + vec_static_blob_entries_vtab *p = (vec_static_blob_entries_vtab *)pVTab; + int iMatchTerm = -1; + int iLimitTerm = -1; + // int iRowidTerm = -1; // https://github.com/asg017/sqlite-vec/issues/47 + int iKTerm = -1; + + for (int i = 0; i < pIdxInfo->nConstraint; i++) { + if (!pIdxInfo->aConstraint[i].usable) + continue; + + int iColumn = pIdxInfo->aConstraint[i].iColumn; + int op = pIdxInfo->aConstraint[i].op; + if (op == SQLITE_INDEX_CONSTRAINT_MATCH && + iColumn == VEC_STATIC_BLOB_ENTRIES_VECTOR) { + if (iMatchTerm > -1) { + // https://github.com/asg017/sqlite-vec/issues/51 + return SQLITE_ERROR; + } + iMatchTerm = i; + } + if (op == SQLITE_INDEX_CONSTRAINT_LIMIT) { + iLimitTerm = i; + } + if (op == SQLITE_INDEX_CONSTRAINT_EQ && + iColumn == VEC_STATIC_BLOB_ENTRIES_K) { + iKTerm = i; + } + } + if (iMatchTerm >= 0) { + if (iLimitTerm < 0 && iKTerm < 0) { + // https://github.com/asg017/sqlite-vec/issues/51 + return SQLITE_ERROR; + } + if (iLimitTerm >= 0 && iKTerm >= 0) { + return SQLITE_ERROR; // limit or k, not both + } + if (pIdxInfo->nOrderBy < 1) { + vtab_set_error(pVTab, "ORDER BY distance required"); + return SQLITE_CONSTRAINT; + } + if (pIdxInfo->nOrderBy > 1) { + // https://github.com/asg017/sqlite-vec/issues/51 + vtab_set_error(pVTab, "more than 1 ORDER BY clause provided"); + return SQLITE_CONSTRAINT; + } + if (pIdxInfo->aOrderBy[0].iColumn != VEC_STATIC_BLOB_ENTRIES_DISTANCE) { + vtab_set_error(pVTab, "ORDER BY must be on the distance column"); + return SQLITE_CONSTRAINT; + } + if (pIdxInfo->aOrderBy[0].desc) { + vtab_set_error(pVTab, + "Only ascending in ORDER BY distance clause is supported, " + "DESC is not supported yet."); + return SQLITE_CONSTRAINT; + } + + pIdxInfo->idxNum = VEC_SBE__QUERYPLAN_KNN; + pIdxInfo->estimatedCost = (double)10; + pIdxInfo->estimatedRows = 10; + + pIdxInfo->orderByConsumed = 1; + pIdxInfo->aConstraintUsage[iMatchTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iMatchTerm].omit = 1; + if (iLimitTerm >= 0) { + pIdxInfo->aConstraintUsage[iLimitTerm].argvIndex = 2; + pIdxInfo->aConstraintUsage[iLimitTerm].omit = 1; + } else { + pIdxInfo->aConstraintUsage[iKTerm].argvIndex = 2; + pIdxInfo->aConstraintUsage[iKTerm].omit = 1; + } + + } else { + pIdxInfo->idxNum = VEC_SBE__QUERYPLAN_FULLSCAN; + pIdxInfo->estimatedCost = (double)p->blob->nvectors; + pIdxInfo->estimatedRows = p->blob->nvectors; + } + return SQLITE_OK; +} + +static int vec_static_blob_entriesFilter(sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv) { + UNUSED_PARAMETER(idxStr); + assert(argc >= 0 && argc <= 3); + vec_static_blob_entries_cursor *pCur = + (vec_static_blob_entries_cursor *)pVtabCursor; + vec_static_blob_entries_vtab *p = + (vec_static_blob_entries_vtab *)pCur->base.pVtab; + + if (idxNum == VEC_SBE__QUERYPLAN_KNN) { + assert(argc == 2); + pCur->query_plan = VEC_SBE__QUERYPLAN_KNN; + struct sbe_query_knn_data *knn_data; + knn_data = sqlite3_malloc(sizeof(*knn_data)); + if (!knn_data) { + return SQLITE_NOMEM; + } + memset(knn_data, 0, sizeof(*knn_data)); + + void *queryVector; + size_t dimensions; + enum VectorElementType elementType; + vector_cleanup cleanup; + char *err; + int rc = vector_from_value(argv[0], &queryVector, &dimensions, &elementType, + &cleanup, &err); + if (rc != SQLITE_OK) { + return SQLITE_ERROR; + } + if (elementType != p->blob->element_type) { + return SQLITE_ERROR; + } + if (dimensions != p->blob->dimensions) { + return SQLITE_ERROR; + } + + i64 k = min(sqlite3_value_int64(argv[1]), (i64)p->blob->nvectors); + if (k < 0) { + // HANDLE https://github.com/asg017/sqlite-vec/issues/55 + return SQLITE_ERROR; + } + if (k == 0) { + knn_data->k = 0; + pCur->knn_data = knn_data; + return SQLITE_OK; + } + + size_t bsize = (p->blob->nvectors + 7) & ~7; + + i32 *topk_rowids = sqlite3_malloc(k * sizeof(i32)); + if (!topk_rowids) { + // HANDLE https://github.com/asg017/sqlite-vec/issues/55 + return SQLITE_ERROR; + } + f32 *distances = sqlite3_malloc(bsize * sizeof(f32)); + if (!distances) { + // HANDLE https://github.com/asg017/sqlite-vec/issues/55 + return SQLITE_ERROR; + } + + for (size_t i = 0; i < p->blob->nvectors; i++) { + // https://github.com/asg017/sqlite-vec/issues/52 + float *v = ((float *)p->blob->p) + (i * p->blob->dimensions); + distances[i] = + distance_l2_sqr_float(v, (float *)queryVector, &p->blob->dimensions); + } + u8 *candidates = bitmap_new(bsize); + assert(candidates); + + u8 *taken = bitmap_new(bsize); + assert(taken); + + bitmap_fill(candidates, bsize); + for (size_t i = bsize; i >= p->blob->nvectors; i--) { + bitmap_set(candidates, i, 0); + } + i32 k_used = 0; + min_idx(distances, bsize, candidates, topk_rowids, k, taken, &k_used); + knn_data->current_idx = 0; + knn_data->distances = distances; + knn_data->k = k; + knn_data->rowids = topk_rowids; + + pCur->knn_data = knn_data; + } else { + pCur->query_plan = VEC_SBE__QUERYPLAN_FULLSCAN; + pCur->iRowid = 0; + } + + return SQLITE_OK; +} + +static int vec_static_blob_entriesRowid(sqlite3_vtab_cursor *cur, + sqlite_int64 *pRowid) { + vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur; + switch (pCur->query_plan) { + case VEC_SBE__QUERYPLAN_FULLSCAN: { + *pRowid = pCur->iRowid; + return SQLITE_OK; + } + case VEC_SBE__QUERYPLAN_KNN: { + i32 rowid = ((i32 *)pCur->knn_data->rowids)[pCur->knn_data->current_idx]; + *pRowid = (sqlite3_int64)rowid; + return SQLITE_OK; + } + } + return SQLITE_ERROR; +} + +static int vec_static_blob_entriesNext(sqlite3_vtab_cursor *cur) { + vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur; + switch (pCur->query_plan) { + case VEC_SBE__QUERYPLAN_FULLSCAN: { + pCur->iRowid++; + return SQLITE_OK; + } + case VEC_SBE__QUERYPLAN_KNN: { + pCur->knn_data->current_idx++; + return SQLITE_OK; + } + } + return SQLITE_ERROR; +} + +static int vec_static_blob_entriesEof(sqlite3_vtab_cursor *cur) { + vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur; + vec_static_blob_entries_vtab *p = + (vec_static_blob_entries_vtab *)pCur->base.pVtab; + switch (pCur->query_plan) { + case VEC_SBE__QUERYPLAN_FULLSCAN: { + return (size_t)pCur->iRowid >= p->blob->nvectors; + } + case VEC_SBE__QUERYPLAN_KNN: { + return pCur->knn_data->current_idx >= pCur->knn_data->k; + } + } + return SQLITE_ERROR; +} + +static int vec_static_blob_entriesColumn(sqlite3_vtab_cursor *cur, + sqlite3_context *context, int i) { + vec_static_blob_entries_cursor *pCur = (vec_static_blob_entries_cursor *)cur; + vec_static_blob_entries_vtab *p = (vec_static_blob_entries_vtab *)cur->pVtab; + + switch (pCur->query_plan) { + case VEC_SBE__QUERYPLAN_FULLSCAN: { + switch (i) { + case VEC_STATIC_BLOB_ENTRIES_VECTOR: + + sqlite3_result_blob( + context, + ((unsigned char *)p->blob->p) + + (pCur->iRowid * p->blob->dimensions * sizeof(float)), + p->blob->dimensions * sizeof(float), SQLITE_TRANSIENT); + sqlite3_result_subtype(context, p->blob->element_type); + break; + } + return SQLITE_OK; + } + case VEC_SBE__QUERYPLAN_KNN: { + switch (i) { + case VEC_STATIC_BLOB_ENTRIES_VECTOR: { + i32 rowid = ((i32 *)pCur->knn_data->rowids)[pCur->knn_data->current_idx]; + sqlite3_result_blob(context, + ((unsigned char *)p->blob->p) + + (rowid * p->blob->dimensions * sizeof(float)), + p->blob->dimensions * sizeof(float), + SQLITE_TRANSIENT); + sqlite3_result_subtype(context, p->blob->element_type); + break; + } + } + return SQLITE_OK; + } + } + return SQLITE_ERROR; +} + +static sqlite3_module vec_static_blob_entriesModule = { + /* iVersion */ 3, + /* xCreate */ + vec_static_blob_entriesCreate, // handle rm? + // https://github.com/asg017/sqlite-vec/issues/55 + /* xConnect */ vec_static_blob_entriesConnect, + /* xBestIndex */ vec_static_blob_entriesBestIndex, + /* xDisconnect */ vec_static_blob_entriesDisconnect, + /* xDestroy */ vec_static_blob_entriesDisconnect, + /* xOpen */ vec_static_blob_entriesOpen, + /* xClose */ vec_static_blob_entriesClose, + /* xFilter */ vec_static_blob_entriesFilter, + /* xNext */ vec_static_blob_entriesNext, + /* xEof */ vec_static_blob_entriesEof, + /* xColumn */ vec_static_blob_entriesColumn, + /* xRowid */ vec_static_blob_entriesRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, +#if SQLITE_VERSION_NUMBER >= 3044000 + /* xIntegrity */ 0 +#endif +}; +#pragma endregion + +#ifdef SQLITE_VEC_ENABLE_AVX +#define SQLITE_VEC_DEBUG_BUILD_AVX "avx" +#else +#define SQLITE_VEC_DEBUG_BUILD_AVX "" +#endif +#ifdef SQLITE_VEC_ENABLE_NEON +#define SQLITE_VEC_DEBUG_BUILD_NEON "neon" +#else +#define SQLITE_VEC_DEBUG_BUILD_NEON "" +#endif + +#define SQLITE_VEC_DEBUG_BUILD \ + SQLITE_VEC_DEBUG_BUILD_AVX " " SQLITE_VEC_DEBUG_BUILD_NEON + +#define SQLITE_VEC_DEBUG_STRING \ + "Version: " SQLITE_VEC_VERSION "\n" \ + "Date: " SQLITE_VEC_DATE "\n" \ + "Commit: " SQLITE_VEC_SOURCE "\n" \ + "Build flags: " SQLITE_VEC_DEBUG_BUILD + +SQLITE_VEC_API int sqlite3_vec_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi) { +#ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT2(pApi); +#endif + int rc = SQLITE_OK; + +#define DEFAULT_FLAGS (SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC) + + rc = sqlite3_create_function_v2(db, "vec_version", 0, DEFAULT_FLAGS, + SQLITE_VEC_VERSION, _static_text_func, NULL, + NULL, NULL); + if (rc != SQLITE_OK) { + return rc; + } + rc = sqlite3_create_function_v2(db, "vec_debug", 0, DEFAULT_FLAGS, + SQLITE_VEC_DEBUG_STRING, _static_text_func, + NULL, NULL, NULL); + if (rc != SQLITE_OK) { + return rc; + } + static struct { + const char *zFName; + void (*xFunc)(sqlite3_context *, int, sqlite3_value **); + int nArg; + int flags; + } aFunc[] = { + // clang-format off + //{"vec_version", _static_text_func, 0, DEFAULT_FLAGS, (void *) SQLITE_VEC_VERSION }, + //{"vec_debug", _static_text_func, 0, DEFAULT_FLAGS, (void *) SQLITE_VEC_DEBUG_STRING }, + {"vec_distance_l2", vec_distance_l2, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE, }, + {"vec_distance_l1", vec_distance_l1, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE, }, + {"vec_distance_hamming",vec_distance_hamming, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE, }, + {"vec_distance_cosine", vec_distance_cosine, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE, }, + {"vec_length", vec_length, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE, }, + {"vec_type", vec_type, 1, DEFAULT_FLAGS, }, + {"vec_to_json", vec_to_json, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + {"vec_add", vec_add, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + {"vec_sub", vec_sub, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + {"vec_slice", vec_slice, 3, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + {"vec_normalize", vec_normalize, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + {"vec_f32", vec_f32, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + {"vec_bit", vec_bit, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + {"vec_int8", vec_int8, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + {"vec_quantize_int8", vec_quantize_int8, 2, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + {"vec_quantize_binary", vec_quantize_binary, 1, DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, }, + // clang-format on + }; + + static struct { + char *name; + const sqlite3_module *module; + void *p; + void (*xDestroy)(void *); + } aMod[] = { + // clang-format off + {"vec0", &vec0Module, NULL, NULL}, + {"vec_each", &vec_eachModule, NULL, NULL}, + // clang-format on + }; + + for (unsigned long i = 0; i < countof(aFunc) && rc == SQLITE_OK; i++) { + rc = sqlite3_create_function_v2(db, aFunc[i].zFName, aFunc[i].nArg, + aFunc[i].flags, NULL, aFunc[i].xFunc, NULL, + NULL, NULL); + if (rc != SQLITE_OK) { + *pzErrMsg = sqlite3_mprintf("Error creating function %s: %s", + aFunc[i].zFName, sqlite3_errmsg(db)); + return rc; + } + } + + for (unsigned long i = 0; i < countof(aMod) && rc == SQLITE_OK; i++) { + rc = sqlite3_create_module_v2(db, aMod[i].name, aMod[i].module, NULL, NULL); + if (rc != SQLITE_OK) { + *pzErrMsg = sqlite3_mprintf("Error creating module %s: %s", aMod[i].name, + sqlite3_errmsg(db)); + return rc; + } + } + + return SQLITE_OK; +} + +#ifndef SQLITE_VEC_OMIT_FS +SQLITE_VEC_API int sqlite3_vec_numpy_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi) { + UNUSED_PARAMETER(pzErrMsg); +#ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT2(pApi); +#endif + int rc = SQLITE_OK; + rc = sqlite3_create_function_v2(db, "vec_npy_file", 1, SQLITE_RESULT_SUBTYPE, + NULL, vec_npy_file, NULL, NULL, NULL); + if(rc != SQLITE_OK) { + return rc; + } + rc = sqlite3_create_module_v2(db, "vec_npy_each", &vec_npy_eachModule, NULL, NULL); + return rc; +} +#endif + +SQLITE_VEC_API int +sqlite3_vec_static_blobs_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi) { + UNUSED_PARAMETER(pzErrMsg); +#ifndef SQLITE_CORE + SQLITE_EXTENSION_INIT2(pApi); +#endif + + int rc = SQLITE_OK; + vec_static_blob_data *static_blob_data; + static_blob_data = sqlite3_malloc(sizeof(*static_blob_data)); + if (!static_blob_data) { + return SQLITE_NOMEM; + } + memset(static_blob_data, 0, sizeof(*static_blob_data)); + + rc = sqlite3_create_function_v2( + db, "vec_static_blob_from_raw", 4, + DEFAULT_FLAGS | SQLITE_SUBTYPE | SQLITE_RESULT_SUBTYPE, NULL, + vec_static_blob_from_raw, NULL, NULL, NULL); + if (rc != SQLITE_OK) + return rc; + + rc = sqlite3_create_module_v2(db, "vec_static_blobs", &vec_static_blobsModule, + static_blob_data, sqlite3_free); + if (rc != SQLITE_OK) + return rc; + rc = sqlite3_create_module_v2(db, "vec_static_blob_entries", + &vec_static_blob_entriesModule, + static_blob_data, NULL); + if (rc != SQLITE_OK) + return rc; + return rc; +} diff --git a/backend/storage/dbext/sqlite-vec/sqlite-vec.h b/backend/storage/dbext/sqlite-vec/sqlite-vec.h new file mode 100644 index 000000000..084035db6 --- /dev/null +++ b/backend/storage/dbext/sqlite-vec/sqlite-vec.h @@ -0,0 +1,41 @@ +#ifndef SQLITE_VEC_H +#define SQLITE_VEC_H + +#ifndef SQLITE_CORE +#include "sqlite3ext.h" +#else +#include "sqlite3.h" +#endif + +#ifdef SQLITE_VEC_STATIC + #define SQLITE_VEC_API +#else + #ifdef _WIN32 + #define SQLITE_VEC_API __declspec(dllexport) + #else + #define SQLITE_VEC_API + #endif +#endif + +#define SQLITE_VEC_VERSION "v0.1.7-alpha.2" +// TODO rm +#define SQLITE_VEC_DATE "2025-01-10T23:18:50Z+0000" +#define SQLITE_VEC_SOURCE "bdc336d1cf2a2222b6227784bd30c6631603279b" + + +#define SQLITE_VEC_VERSION_MAJOR 0 +#define SQLITE_VEC_VERSION_MINOR 1 +#define SQLITE_VEC_VERSION_PATCH 7 + +#ifdef __cplusplus +extern "C" { +#endif + +SQLITE_VEC_API int sqlite3_vec_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi); + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef SQLITE_VEC_H */ diff --git a/backend/storage/schema.gen.go b/backend/storage/schema.gen.go index 2a4928d1e..495303552 100644 --- a/backend/storage/schema.gen.go +++ b/backend/storage/schema.gen.go @@ -138,6 +138,104 @@ const ( C_DocumentGenerationsResource = "document_generations.resource" ) +// Table embeddings. +const ( + Embeddings sqlitegen.Table = "embeddings" + EmbeddingsDistance sqlitegen.Column = "embeddings.distance" + EmbeddingsFtsID sqlitegen.Column = "embeddings.fts_id" + EmbeddingsK sqlitegen.Column = "embeddings.k" + EmbeddingsMultilingualMinilmL12V2 sqlitegen.Column = "embeddings.multilingual_minilm_l12_v2" + EmbeddingsRowid sqlitegen.Column = "embeddings.rowid" +) + +// Table embeddings. Plain strings. +const ( + T_Embeddings = "embeddings" + C_EmbeddingsDistance = "embeddings.distance" + C_EmbeddingsFtsID = "embeddings.fts_id" + C_EmbeddingsK = "embeddings.k" + C_EmbeddingsMultilingualMinilmL12V2 = "embeddings.multilingual_minilm_l12_v2" + C_EmbeddingsRowid = "embeddings.rowid" +) + +// Table embeddings_chunks. +const ( + EmbeddingsChunks sqlitegen.Table = "embeddings_chunks" + EmbeddingsChunksChunkID sqlitegen.Column = "embeddings_chunks.chunk_id" + EmbeddingsChunksRowids sqlitegen.Column = "embeddings_chunks.rowids" + EmbeddingsChunksSize sqlitegen.Column = "embeddings_chunks.size" + EmbeddingsChunksValidity sqlitegen.Column = "embeddings_chunks.validity" +) + +// Table embeddings_chunks. Plain strings. +const ( + T_EmbeddingsChunks = "embeddings_chunks" + C_EmbeddingsChunksChunkID = "embeddings_chunks.chunk_id" + C_EmbeddingsChunksRowids = "embeddings_chunks.rowids" + C_EmbeddingsChunksSize = "embeddings_chunks.size" + C_EmbeddingsChunksValidity = "embeddings_chunks.validity" +) + +// Table embeddings_info. +const ( + EmbeddingsInfo sqlitegen.Table = "embeddings_info" + EmbeddingsInfoKey sqlitegen.Column = "embeddings_info.key" + EmbeddingsInfoValue sqlitegen.Column = "embeddings_info.value" +) + +// Table embeddings_info. Plain strings. +const ( + T_EmbeddingsInfo = "embeddings_info" + C_EmbeddingsInfoKey = "embeddings_info.key" + C_EmbeddingsInfoValue = "embeddings_info.value" +) + +// Table embeddings_metadatachunks00. +const ( + EmbeddingsMetadatachunks00 sqlitegen.Table = "embeddings_metadatachunks00" + EmbeddingsMetadatachunks00Data sqlitegen.Column = "embeddings_metadatachunks00.data" + EmbeddingsMetadatachunks00Rowid sqlitegen.Column = "embeddings_metadatachunks00.rowid" +) + +// Table embeddings_metadatachunks00. Plain strings. +const ( + T_EmbeddingsMetadatachunks00 = "embeddings_metadatachunks00" + C_EmbeddingsMetadatachunks00Data = "embeddings_metadatachunks00.data" + C_EmbeddingsMetadatachunks00Rowid = "embeddings_metadatachunks00.rowid" +) + +// Table embeddings_rowids. +const ( + EmbeddingsRowids sqlitegen.Table = "embeddings_rowids" + EmbeddingsRowidsChunkID sqlitegen.Column = "embeddings_rowids.chunk_id" + EmbeddingsRowidsChunkOffset sqlitegen.Column = "embeddings_rowids.chunk_offset" + EmbeddingsRowidsID sqlitegen.Column = "embeddings_rowids.id" + EmbeddingsRowidsRowid sqlitegen.Column = "embeddings_rowids.rowid" +) + +// Table embeddings_rowids. Plain strings. +const ( + T_EmbeddingsRowids = "embeddings_rowids" + C_EmbeddingsRowidsChunkID = "embeddings_rowids.chunk_id" + C_EmbeddingsRowidsChunkOffset = "embeddings_rowids.chunk_offset" + C_EmbeddingsRowidsID = "embeddings_rowids.id" + C_EmbeddingsRowidsRowid = "embeddings_rowids.rowid" +) + +// Table embeddings_vector_chunks00. +const ( + EmbeddingsVectorChunks00 sqlitegen.Table = "embeddings_vector_chunks00" + EmbeddingsVectorChunks00Rowid sqlitegen.Column = "embeddings_vector_chunks00.rowid" + EmbeddingsVectorChunks00Vectors sqlitegen.Column = "embeddings_vector_chunks00.vectors" +) + +// Table embeddings_vector_chunks00. Plain strings. +const ( + T_EmbeddingsVectorChunks00 = "embeddings_vector_chunks00" + C_EmbeddingsVectorChunks00Rowid = "embeddings_vector_chunks00.rowid" + C_EmbeddingsVectorChunks00Vectors = "embeddings_vector_chunks00.vectors" +) + // Table fts. const ( Fts sqlitegen.Table = "fts" @@ -539,6 +637,25 @@ var Schema = sqlitegen.Schema{ DocumentGenerationsLastTombstoneRefTime: {Table: DocumentGenerations, SQLType: "INTEGER"}, DocumentGenerationsMetadata: {Table: DocumentGenerations, SQLType: "JSON"}, DocumentGenerationsResource: {Table: DocumentGenerations, SQLType: "INTEGER"}, + EmbeddingsDistance: {Table: Embeddings, SQLType: ""}, + EmbeddingsFtsID: {Table: Embeddings, SQLType: ""}, + EmbeddingsK: {Table: Embeddings, SQLType: ""}, + EmbeddingsMultilingualMinilmL12V2: {Table: Embeddings, SQLType: ""}, + EmbeddingsRowid: {Table: Embeddings, SQLType: ""}, + EmbeddingsChunksChunkID: {Table: EmbeddingsChunks, SQLType: "INTEGER"}, + EmbeddingsChunksRowids: {Table: EmbeddingsChunks, SQLType: "BLOB"}, + EmbeddingsChunksSize: {Table: EmbeddingsChunks, SQLType: "INTEGER"}, + EmbeddingsChunksValidity: {Table: EmbeddingsChunks, SQLType: "BLOB"}, + EmbeddingsInfoKey: {Table: EmbeddingsInfo, SQLType: "TEXT"}, + EmbeddingsInfoValue: {Table: EmbeddingsInfo, SQLType: "ANY"}, + EmbeddingsMetadatachunks00Data: {Table: EmbeddingsMetadatachunks00, SQLType: "BLOB"}, + EmbeddingsMetadatachunks00Rowid: {Table: EmbeddingsMetadatachunks00, SQLType: ""}, + EmbeddingsRowidsChunkID: {Table: EmbeddingsRowids, SQLType: "INTEGER"}, + EmbeddingsRowidsChunkOffset: {Table: EmbeddingsRowids, SQLType: "INTEGER"}, + EmbeddingsRowidsID: {Table: EmbeddingsRowids, SQLType: ""}, + EmbeddingsRowidsRowid: {Table: EmbeddingsRowids, SQLType: "INTEGER"}, + EmbeddingsVectorChunks00Rowid: {Table: EmbeddingsVectorChunks00, SQLType: ""}, + EmbeddingsVectorChunks00Vectors: {Table: EmbeddingsVectorChunks00, SQLType: "BLOB"}, FtsBlobID: {Table: Fts, SQLType: ""}, FtsBlockID: {Table: Fts, SQLType: ""}, FtsFts: {Table: Fts, SQLType: ""}, diff --git a/backend/storage/schema.gensum b/backend/storage/schema.gensum index a3b0ba063..2c8ab731a 100644 --- a/backend/storage/schema.gensum +++ b/backend/storage/schema.gensum @@ -1,2 +1,2 @@ -srcs: ec92b9c6d2959a1645a10796f9de83b2 -outs: d100e6b11ea49189f4542c8b9a317929 +srcs: 971e71ebc97121b4e9ec99a9478aebfc +outs: d0b4f6797206ffb86f50c6ef59d932d1 diff --git a/backend/storage/schema.sql b/backend/storage/schema.sql index 0a859d2c0..99c0354e2 100644 --- a/backend/storage/schema.sql +++ b/backend/storage/schema.sql @@ -344,3 +344,12 @@ CREATE INDEX fts_index_by_block ON fts_index (block_id); CREATE INDEX fts_index_by_type ON fts_index (type); CREATE INDEX fts_index_by_ts ON fts_index (ts); CREATE INDEX fts_index_by_genesis_blob ON fts_index (genesis_blob); + +-- Stores text content to a full text search. +-- https://sqlite.org/fts5.html. + +-- Sqlite vector extension tables for different embedding models. +CREATE VIRTUAL TABLE embeddings USING vec0( + multilingual_minilm_l12_v2 int8[384] distance_metric=cosine, + fts_id int +); \ No newline at end of file diff --git a/backend/storage/sqlite_test.go b/backend/storage/sqlite_test.go index 2e7e871fb..a7ee99992 100644 --- a/backend/storage/sqlite_test.go +++ b/backend/storage/sqlite_test.go @@ -2,6 +2,7 @@ package storage import ( "context" + "math" "os" "seed/backend/util/sqlite" "seed/backend/util/sqlite/sqlitex" @@ -45,6 +46,249 @@ INSERT INTO data VALUES (rb64_create(1,2,3,4,5,6,200,100,300,400)); } +func TestSqliteVec(t *testing.T) { + pool, err := OpenSQLite("file::memory:?mode=memory&cache=shared", 0, 1) + require.NoError(t, err) + + defer pool.Close() + + conn, release, err := pool.Conn(t.Context()) + require.NoError(t, err) + defer release() + var sqliteVersion, vecVersion string + require.NoError(t, sqlitex.Exec(conn, `select sqlite_version(), vec_version();`, func(stmt *sqlite.Stmt) error { + sqliteVersion = stmt.ColumnText(0) + vecVersion = stmt.ColumnText(1) + return nil + })) + require.NotEmpty(t, sqliteVersion) + require.NotEmpty(t, vecVersion) + + inputVectorAllMiniLm := []float32{ // I like soccer + //-0.03257234, -0.020275954, -0.02915192, -0.01172466, 0.031904545, 0.011192954, 0.10219336, 0.02276321, 0.12111865, 0.08637059, -0.022683695, -0.06439873, -0.029596794, 0.05034988, 0.061908953, -0.026607659, -0.0151267005, -0.05004665, -0.027256573, -0.01857386, -0.10482127, 0.04007108, 0.03138711, 0.011234581, -0.024798265, 0.044401262, 0.02563706, 0.025156049, -0.042645738, -0.065788515, -0.07628015, 0.04963409, 0.027579589, 0.02952144, -0.022162069, 0.057708915, 0.07827545, -0.035521325, -0.010836728, 0.0792433, -0.012516824, -0.0033106236, 0.050476864, 0.004977566, -0.021043738, 0.056125063, 0.058390588, -0.024164367, 0.03224752, 0.055666763, 0.087544195, 0.060462955, -0.028417531, 0.029594362, 0.038643602, 0.04388487, -0.015719697, 0.05811514, -0.014786908, -0.06234272, 0.034604803, 0.0270749, 0.011145769, -0.020372638, -0.023128554, -0.04426869, -0.03686764, 0.0185646, -0.0025217198, -0.025713688, 0.029340398, -0.008640267, 0.044104565, 0.030574776, 0.04785601, 0.023422701, -0.03134766, -0.10379501, -0.023353228, -0.035810463, 0.047315888, -0.07183014, -0.107640855, -0.03328787, 0.007991169, -0.12103763, -0.04951489, 0.03058528, -0.027524313, 0.0070702145, -0.021583248, 0.029068232, 0.0014781371, 0.059825376, -0.07779791, 0.10151957, 0.06316044, -0.044828914, -0.030898642, 0.11246318, 0.02134709, 0.004497722, -0.04648277, 0.12790102, 0.009714707, 0.029447963, -0.00858738, 0.079104856, 0.012753146, 0.016612291, -0.018559923, 0.026222723, -0.013069614, -0.02324394, -0.030813279, 0.088258624, 0.031152612, 0.020986639, 0.010379855, 0.00030765648, 0.013378957, 0.008272375, -0.02759703, 0.03644591, 0.00869511, -0.027509125, 0.022814099, -4.7136302e-33, -0.10784748, -0.09083707, -0.004995101, 0.023950664, -0.07033372, 0.12298115, 0.107010305, -0.012009156, -0.045789, -0.04215043, -0.014397025, 0.039956767, -0.0676023, 0.08447218, 0.05570281, -0.026699368, -0.0035680863, 0.016073534, -0.050091106, 0.0031498498, 0.027758501, 0.029227579, 0.03017978, 0.086469404, 0.025539331, -0.01844842, 0.042864162, -0.12124215, -0.00038697594, 0.021988917, 0.05295174, 0.04153748, -0.04568292, -0.03338462, 0.014437593, -0.049470726, 0.0255458, -0.0062810406, -0.0073792655, 0.034901503, 0.073809996, -0.018376313, -0.036652543, -0.015464334, 0.08736831, 0.047898635, 0.0045617805, 0.005081279, -0.058757063, -0.0823459, -0.004856788, -0.06899088, 0.11166826, 0.030801175, 0.062608615, -0.042762976, 0.0024605435, -0.0011548119, -0.109767966, -0.09466228, 0.007890052, 0.09033448, 0.034990758, -0.037624054, -0.040212877, 0.07100699, 0.039126758, 0.049417846, 0.019883387, -0.09141434, -0.03130304, 0.06572834, -0.04347879, 0.026804034, -0.05681226, -0.033314727, 0.09549145, 0.01470645, -0.0056920326, -0.032014996, -0.012232391, -0.0055070976, -0.0012501355, -0.096169315, 0.06698281, 0.05922922, -0.034795634, -0.09309826, -0.017462883, -0.027060775, -0.06493784, -0.025806291, 0.024348747, 0.060736388, -0.016188974, 3.8337727e-33, -0.04267163, -0.101351365, 0.03048647, -0.008369032, -0.038549926, -0.043085612, -0.019983524, 0.011508683, -0.0003445781, 0.09753054, 0.005428901, -0.07541155, -0.0073050256, 0.021510795, -0.032526188, -0.03943356, -0.022794114, 0.040944584, -0.049734995, 0.0710797, -0.016609821, 0.032434884, -0.020383624, -0.04652756, -0.03889133, -0.026369289, -0.040475905, 0.026547609, -0.055322777, 0.077175416, 0.06787637, 0.016755862, -0.0067203045, -0.089394465, 0.0093914345, -0.00009991144, -0.033687223, -0.055027463, -0.026996223, -0.03755737, 0.028166125, -0.019265572, 0.002567804, 0.06134494, 0.026206829, -0.0113179665, 0.01962878, 0.019154781, -0.025315149, -0.04377693, 0.018600048, 0.01095702, -0.046246365, 0.0023348662, 0.103011794, -0.009636349, -0.018710285, -0.028760685, -0.0775619, -0.09241314, -0.061219703, 0.10269988, -0.089270405, 0.10081492, 0.05596098, 0.036888573, -0.064108334, -0.05342945, -0.04043235, -0.054844964, -0.0386991, -0.0005178135, -0.07662602, 0.06311619, 0.0011052727, -0.019020827, -0.004344792, 0.11929148, 0.0377563, 0.05692379, -0.0020071885, -0.048899014, -0.07558866, 0.024566732, 0.003332356, 0.021104975, 0.012700458, 0.032035876, -0.0030066527, 0.025236236, 0.11667197, 0.025472248, 0.027071044, -0.039523058, 0.0062581245, -1.2720366e-8, -0.010500026, 0.036132365, 0.05614091, -0.00939868, -0.03078763, 0.049232826, -0.08952176, -0.05725441, -0.013394438, -0.06725342, 0.056669466, 0.039468262, 0.019421516, 0.030734736, 0.04997463, -0.03896074, -0.013775474, -0.012380326, 0.03720688, 0.15915723, -0.04173079, -0.038121887, -0.05343048, 0.03682137, -0.024873184, -0.1000044, -0.04427363, -0.058869336, 0.07630673, -0.051046718, 0.029713554, -0.012995457, 0.02715823, 0.062924445, -0.02696283, -0.06153242, 0.06730022, -0.11495782, -0.025453374, -0.10572444, 0.026596364, 0.055077195, 0.00997414, -0.08405912, -0.03478074, -0.036789693, 0.027432412, -0.1436866, -0.035689753, 0.011686381, 0.00790386, 0.043073013, 0.04827841, 0.03335682, 0.069851816, 0.039644755, -0.053829536, 0.04433935, -0.015757568, -0.0009559446, 0.10678372, 0.030213417, 0.04567417, 0.015694924} + -0.031400226, -0.019592829, -0.026423635, -0.012345122, 0.031504694, 0.009964937, 0.10350055, 0.023546549, 0.12261914, 0.08654113, -0.021991745, -0.06501513, -0.030366648, 0.051408015, 0.063949, -0.025503756, -0.015627224, -0.051127605, -0.028143074, -0.020148342, -0.10618655, 0.03885293, 0.030691035, 0.01090022, -0.026965369, 0.042391825, 0.026441406, 0.024786962, -0.043460436, -0.06346836, -0.07781889, 0.052630756, 0.029166643, 0.02885241, -0.022810847, 0.059833646, 0.07846331, -0.036273044, -0.011340247, 0.07606225, -0.012469678, -0.0048275557, 0.049335353, 0.00554439, -0.021876069, 0.05604594, 0.057364825, -0.02513788, 0.03283383, 0.05481786, 0.08830751, 0.059901778, -0.025457887, 0.030574486, 0.037700906, 0.045329735, -0.016444204, 0.058020473, -0.014109312, -0.06423927, 0.037324365, 0.024996785, 0.008719374, -0.020799508, -0.022336, -0.04299596, -0.03524984, 0.016941503, -0.0032612435, -0.026629463, 0.03125763, -0.007827222, 0.04343776, 0.029869938, 0.046129704, 0.022690779, -0.032546915, -0.10285696, -0.025260933, -0.03582219, 0.049142078, -0.072484255, -0.10611507, -0.033661317, 0.010063381, -0.12279051, -0.04747059, 0.031435065, -0.030804316, 0.0077477987, -0.021978032, 0.030837966, 0.0025910374, 0.05991078, -0.07778117, 0.099446625, 0.06396328, -0.04332428, -0.031903055, 0.1126283, 0.022478074, 0.005274577, -0.046339173, 0.1270092, 0.009785454, 0.030183107, -0.01154031, 0.07997376, 0.011904627, 0.016018389, -0.018534977, 0.025487816, -0.014884275, -0.021397585, -0.03106893, 0.088628754, 0.03141358, 0.02190127, 0.009643565, -0.0004124832, 0.015956575, 0.008689782, -0.026356714, 0.03777219, 0.008036844, -0.027742615, 0.020934958, -4.7199478e-33, -0.10728847, -0.09341516, -0.0068809, 0.02529912, -0.07059931, 0.120726794, 0.10871382, -0.012361775, -0.046972286, -0.041852977, -0.0135194, 0.041073825, -0.06747464, 0.08462977, 0.056306615, -0.024730997, -0.0051699886, 0.013864007, -0.04893004, 0.0022343085, 0.026662175, 0.029736234, 0.028337548, 0.086573444, 0.02376542, -0.017143544, 0.043316692, -0.12254426, -0.0013854865, 0.022728251, 0.053712726, 0.040667124, -0.04655418, -0.033638604, 0.016398167, -0.04751641, 0.025322232, -0.0049468735, -0.007489803, 0.035500407, 0.074282125, -0.01730695, -0.03771438, -0.015784753, 0.08655405, 0.04910417, 0.005815138, 0.00286227, -0.058181632, -0.08294427, -0.0039870474, -0.06942425, 0.11143673, 0.033723995, 0.062841594, -0.03982585, 0.0022220344, 0.000513615, -0.10922448, -0.095094524, 0.009057723, 0.092711054, 0.034367163, -0.0367694, -0.04128117, 0.06899694, 0.0409905, 0.050280295, 0.018229086, -0.09035983, -0.030531574, 0.06563717, -0.043308813, 0.026535481, -0.057725295, -0.03767157, 0.09412213, 0.014072554, -0.0071522505, -0.032211903, -0.012245658, -0.006829617, -0.002062376, -0.09584323, 0.06823529, 0.057480402, -0.032019816, -0.09135943, -0.017504739, -0.024874192, -0.06409617, -0.023994585, 0.025270857, 0.058949653, -0.018030757, 3.8233913e-33, -0.04304542, -0.101597816, 0.030463707, -0.0089173885, -0.04258402, -0.04347481, -0.021771882, 0.013439859, -0.001302613, 0.09849311, 0.0043489593, -0.07343665, -0.008524562, 0.02194241, -0.032384656, -0.039486524, -0.02176333, 0.04140441, -0.048904777, 0.06945691, -0.017456615, 0.032617696, -0.01991533, -0.045926824, -0.040625647, -0.025383236, -0.039400645, 0.026314352, -0.057309154, 0.073796004, 0.0678358, 0.0155726, -0.007704668, -0.089223735, 0.011070448, -0.002384746, -0.035642266, -0.053164456, -0.02531605, -0.039448522, 0.0258088, -0.021162758, 0.000024034282, 0.060785256, 0.02589979, -0.013311769, 0.021885158, 0.01799113, -0.027079, -0.043177523, 0.018666783, 0.011428517, -0.047755435, 0.001535328, 0.10148385, -0.009902444, -0.018315518, -0.02918652, -0.07848919, -0.09339473, -0.061614774, 0.10425477, -0.08561818, 0.10293462, 0.058346443, 0.03657018, -0.063472755, -0.055958036, -0.041695658, -0.053166673, -0.037571035, -0.00031496305, -0.07772246, 0.06607958, 0.0006988308, -0.018190863, -0.0033614072, 0.12068534, 0.038543586, 0.055527147, -0.0032133076, -0.049357757, -0.074208274, 0.027119722, 0.0015180071, 0.019669818, 0.0137763675, 0.030222319, -0.0036028163, 0.02538881, 0.11675209, 0.025628144, 0.027840238, -0.037062593, 0.0069681, -1.2740055e-8, -0.012079577, 0.032545913, 0.055001266, -0.007606999, -0.03278127, 0.05010415, -0.09225471, -0.057210073, -0.014155238, -0.06807407, 0.05738117, 0.038556244, 0.019358037, 0.03188907, 0.050605584, -0.038415164, -0.011293157, -0.011765533, 0.035469927, 0.16003932, -0.043188736, -0.03693733, -0.05157381, 0.038188055, -0.024569238, -0.09622164, -0.042481463, -0.058155093, 0.074671805, -0.05058784, 0.028090877, -0.01076562, 0.02536369, 0.064006634, -0.026738178, -0.06218561, 0.066478275, -0.11502656, -0.022953942, -0.10485117, 0.026024569, 0.054819703, 0.009411742, -0.08349918, -0.034318514, -0.03722574, 0.025190426, -0.14181867, -0.036501262, 0.014563417, 0.008070703, 0.04368794, 0.04972652, 0.035275213, 0.067561366, 0.038905967, -0.050684504, 0.044970967, -0.01562001, -0.00043346905, 0.10684897, 0.031699352, 0.04633555, 0.014935026} + _ /*testVectorIdenticalAllMiniLm :*/ = []float32{ // I like soccer. + 0.01131454, 0.010620677, 0.0015508715, -0.010856348, 0.05597948, 0.0049805087, 0.067461275, 0.03328733, 0.13447464, 0.08586853, -0.040513035, -0.02009317, -0.038548514, 0.08698898, 0.07263258, -0.03728402, -0.059083074, -0.06667204, -0.052769758, -0.031058554, -0.10082352, 0.03688198, 0.026651856, 0.007680728, -0.04502012, 0.015208153, 0.022945326, 0.020155728, -0.06520617, -0.062964804, -0.054135237, 0.06084455, 0.025465416, 0.0499017, -0.028564401, 0.084335275, 0.08214002, -0.039602946, -0.0069430717, 0.10292513, 0.01535616, 0.011049228, 0.04324932, 0.022051258, -0.023089372, 0.07477042, 0.05344905, -0.0049753296, 0.04334521, 0.038609523, 0.07665334, 0.09241342, -0.018674862, 0.007181402, 0.06569979, 0.04053445, 0.032584935, 0.00030079833, -0.044334758, -0.040868845, 0.049704153, 0.078073926, -0.036172584, 0.034329403, -0.04763751, -0.049621355, -0.038334493, 0.0024775774, -0.015790021, -0.022870313, 0.009895804, 0.014663572, 0.015788544, 0.040897634, 0.01586229, 0.06007576, -0.043608364, -0.09733833, -0.004243416, -0.038167093, 0.06366465, -0.10200126, -0.06222235, -0.034948494, 0.032497965, -0.1220878, -0.019688893, 0.04300853, -0.0145288855, 0.005731905, -0.03247285, 0.021495162, -0.02089261, 0.07675593, -0.083423644, 0.03240602, 0.050584674, -0.021981465, -0.046977244, 0.06844076, 0.005320283, -0.00238157, -0.018965999, 0.110728405, -0.03276246, 0.04178051, -0.037217066, 0.11065087, 0.04146306, 0.016847352, -0.03435455, 0.04039201, -0.017638713, 0.009564772, -0.03983149, 0.08853525, 0.015211783, 0.028543757, 0.009481288, 0.010211514, 0.036960095, 0.034983817, -0.0010532216, 0.02753564, 0.033574384, -0.051937006, 0.032995727, -7.451915e-33, -0.11409583, -0.08105908, 0.0007347877, 0.048730176, -0.0828607, 0.07039157, 0.089923985, -0.01617394, -0.06374847, -0.010371178, -0.025564581, 0.047218904, -0.032684807, 0.045501344, 0.018751347, -0.016856253, -0.038395617, -0.024334978, -0.02723327, 0.0436584, 0.056480836, -0.009069128, 0.034400273, 0.06546491, 0.0069241878, 0.00011558247, 0.050497066, -0.119810306, 0.016351156, 0.021944985, 0.041057423, 0.025537869, -0.06540622, -0.032034297, -0.022458976, -0.029772205, -0.027870553, -0.009650673, 0.004334475, 0.057349075, 0.06219041, -0.027881522, -0.045629255, -0.021200517, 0.08237861, 0.03565658, -0.0070814476, -0.017090205, -0.03294201, -0.11487679, -0.021024449, -0.03501404, 0.12113554, 0.024146164, 0.074799635, -0.022222536, 0.0055600554, 0.005413487, -0.096634224, -0.059110537, 0.003135027, 0.051589515, 0.003814949, 0.007951224, -0.016783884, 0.061876614, 0.057257015, 0.0565576, 0.035994183, -0.08768001, -0.03894587, 0.058062512, -0.050881494, -0.010287996, -0.046873756, -0.07111961, 0.10083502, 0.017346513, 0.028722202, -0.021611689, -0.029629843, 0.007369973, 0.0023915675, -0.07279052, 0.08975132, 0.06063937, -0.027926002, -0.0843173, -0.037794642, -0.000016701832, -0.021700207, -0.030522661, 0.011052664, 0.02598223, -0.055586025, 5.3210993e-33, 0.0016951093, -0.07746609, 0.052942276, -0.014654917, -0.025827304, -0.04157314, -0.04652123, 0.03248946, -0.02382773, 0.12980635, 0.010397279, -0.08060332, -0.011098087, 0.023216572, -0.06938393, -0.06604429, -0.023222275, 0.05023063, -0.036604155, 0.052058153, -0.025728924, 0.03802553, -0.007031048, -0.048959497, -0.036528483, -0.027517123, -0.011554726, 0.030852789, -0.06116499, 0.06310424, 0.082117364, 0.020803839, -0.02279793, -0.123610236, -0.008656864, 0.017309126, -0.04104132, -0.04831887, 0.00444449, -0.029541945, 0.012982778, 0.0057664886, 0.023664985, 0.04711954, 0.022054886, -0.029664189, 0.04911025, -0.009663424, -0.068643875, -0.0133390725, -0.010796139, -0.016866393, -0.0441809, -0.032256898, 0.08943102, -0.0020334253, 0.030598875, -0.026213298, -0.056109335, -0.11507237, -0.09898888, 0.09020879, -0.08970877, 0.093979366, 0.089437954, 0.041356485, -0.05887716, -0.06781923, -0.028999556, -0.052760556, -0.05826197, -0.009320102, -0.073673904, 0.05063148, -0.014524738, -0.022828408, -0.046535406, 0.11746902, 0.05394743, 0.054690134, 0.01826546, -0.037322067, -0.0504209, 0.040360883, -0.010330996, 0.038647626, 0.0027467594, 0.013570683, -0.03107847, 0.031050276, 0.09713425, 0.029676463, 0.020822981, -0.053702336, 0.032986965, -1.4321398e-8, -0.030332664, 0.04112978, 0.04328097, -0.026173392, -0.035316706, 0.030390155, -0.07855615, -0.0770721, -0.023096573, -0.06787902, 0.020071872, 0.037653215, 0.03334669, 0.018685294, 0.07426232, -0.019379163, 0.022382611, 0.04261784, 0.03125693, 0.16561149, -0.058250356, -0.008572391, -0.054660507, 0.044894964, -0.008739794, -0.056200907, -0.05965597, -0.06521968, 0.060535245, -0.069381304, 0.047452252, -0.011879281, 0.03529539, 0.06037299, 0.015556185, -0.03279143, 0.031248719, -0.14048043, -0.03146615, -0.0841552, 0.031004258, 0.030710205, -0.011537482, -0.08116136, -0.037899327, -0.040594265, 0.023319885, -0.103118435, -0.027137168, -0.007145335, -0.031967133, 0.06544996, 0.068396434, 0.059607584, 0.052322824, 0.036203153, -0.039613508, 0.013964925, 0.018944398, -0.0018188809, 0.082743265, 0.016541986, 0.013581551, -0.027118873} + testVectorSimilarAllMiniLm := []float32{ // I love sports + //0.00026362416, 0.014767619, 0.021504184, -0.059653055, 0.03719659, 0.020660795, 0.092176445, 0.015168006, 0.095546804, 0.11516282, -0.09549848, -0.01851005, -0.046594534, 0.07528346, 0.06421866, -0.021190012, -0.0252041, 0.015633091, -0.07253562, -0.01764868, -0.10012506, 0.078775324, 0.0020236252, 0.031180974, -0.038821135, 0.032203797, -0.013239691, 0.037366807, -0.09310408, -0.04295193, -0.05908556, 0.018990973, 0.036237795, 0.06874592, -0.05112845, 0.03658637, 0.022754509, 0.0353726, 0.021830885, 0.05724633, 0.011880491, -0.026705178, 0.036960617, 0.03842726, -0.022787241, 0.04585101, 0.0070750383, -0.01414724, 0.068151, 0.07114068, 0.07641735, 0.050189193, 0.018739313, 0.021789955, 0.13246177, 0.0589047, -0.054787394, 0.020005632, -0.030710634, -0.051219832, 0.08436909, 0.078757755, -0.028502226, 0.020508872, -0.025190346, -0.0010402476, -0.017338276, 0.04105685, -0.020553743, -0.03194955, 0.019584093, 0.033613738, 0.021315971, 0.008128055, 0.06039774, 0.07441746, -0.04150956, -0.032176223, 0.020150306, -0.00719365, 0.06232175, -0.13403073, -0.03232526, -0.009085634, 0.013260246, -0.10003216, -0.013193607, -0.0078267325, -0.038769018, 0.06066836, -0.076722346, -0.029614426, 0.017493809, 0.01837467, -0.08357013, 0.027240746, -0.028053757, -0.019113902, -0.04018055, 0.10493346, 0.040009994, -0.0053541283, 0.0077534714, 0.12980479, -0.04236032, 0.03734012, -0.10051369, 0.058608532, 0.063245036, 0.053538296, 0.0020166421, 0.048067957, 0.009343895, 0.026091954, -0.038032845, 0.114335306, 0.016989402, 0.019588996, 0.03145787, 0.04192349, -0.022790771, 0.05861672, 0.013750055, 0.046368998, 0.0033301187, -0.051242463, 0.022568645, -6.632319e-33, -0.11337348, -0.10842799, 0.041520387, 0.044486426, -0.0894609, 0.058622535, 0.0658847, -0.05292673, -0.038936578, -0.052812826, -0.02045152, 0.12859431, -0.033341423, 0.043710552, 0.058155958, -0.043240115, -0.030041886, -0.022881871, -0.027349714, 0.047143236, 0.060535014, -0.034994554, 0.020302448, 0.093966834, -0.04291792, -0.041587766, 0.046562556, -0.13454455, 0.0031562252, 0.0427703, -0.015481101, -0.017057464, -0.041422457, -0.052689534, 0.013223983, -0.08027451, -0.003286757, 0.0035501893, 0.024575593, 0.038704127, 0.049501404, -0.06224471, -0.097368464, -0.05147645, 0.0423321, 0.031157337, 0.034721, -0.0054011475, -0.026126081, -0.07133564, 0.0055619734, -0.05057349, 0.08191999, -0.030462967, 0.076860376, -0.052729983, 0.012050074, -0.026305396, -0.10325818, -0.03754571, -0.0112336185, 0.08668051, 0.005386818, -0.031071356, -0.07923152, 0.070210755, 0.025536593, 0.0292512, -0.014211757, -0.04075855, -0.0064887227, 0.060551602, -0.04304162, -0.016149169, -0.023897642, 0.020720597, 0.09592182, 0.024485867, -0.0544361, -0.022464413, 0.01669725, -0.02431244, -0.0054614064, -0.051382773, 0.078943565, 0.016240496, -0.026315168, -0.11011876, 0.0070557143, -0.020919513, -0.071293965, -0.07014779, -0.0037458618, 0.029441752, -0.07556642, 4.397621e-33, -0.031258456, -0.06779216, 0.017081454, 0.022765968, 0.013745364, -0.08850574, -0.006786497, 0.014137459, 0.02915892, 0.116907135, 0.00012337089, -0.092243455, -0.05332957, 0.028895931, -0.03073552, -0.043562047, -0.014896422, 0.020126741, -0.07177722, 0.053788517, 0.011570388, 0.02808956, -0.011651408, -0.0022306582, -0.03428677, 0.005092116, -0.030958654, 0.01933354, -0.0070974817, 0.06328831, 0.03959439, 0.072823934, 0.032349728, -0.04443758, -0.045816477, 0.06829776, 0.020676386, -0.026065974, 0.004364585, -0.056256767, 0.06205877, -0.0035178047, 0.023697682, 0.04677135, 0.015567848, 0.018631024, 0.0565867, 0.005380097, -0.039715122, 0.0045714476, -0.03616421, 0.012553079, -0.019506471, -0.051700003, 0.034315363, 0.01979811, 0.012532529, -0.019153126, -0.07433593, -0.10320265, -0.08770017, 0.071043946, -0.08891842, 0.092727125, 0.05241821, 0.00939262, -0.023062192, -0.10742699, -0.11583867, -0.037183635, -0.088501595, 0.0023911784, -0.060512625, 0.003272784, -0.012091187, -0.013224924, -0.02532985, 0.12280879, 0.03587415, 0.07919931, 0.0029871094, 0.021130897, -0.03932042, 0.030678913, -0.022584101, 0.053433564, 0.0077579366, 0.039691627, -0.0678177, 0.00016477818, 0.090540335, 0.049336806, -0.013702445, -0.06380929, 0.046602044, -1.3468986e-8, -0.0033818658, 0.023209086, 0.02295639, -0.04178909, -0.0064045484, 0.033418067, 0.006244826, -0.02013353, 0.032246806, -0.068502314, 0.014048806, 0.061602373, 0.011759671, 0.044154536, 0.1007985, -0.047942605, -0.0057106917, -0.012898089, -0.013123747, 0.095827736, -0.050624825, -0.021023307, -0.039489638, 0.06314523, -0.03398741, -0.06995603, -0.047796965, -0.08669301, 0.07790514, -0.040282667, 0.044843372, 0.017738959, -0.0032962894, -0.008149323, 0.0150587065, -0.03218106, 0.070969, -0.08693049, -0.017444534, -0.050325327, -0.05504765, 0.030505411, 0.006771687, -0.04201647, -0.05674048, -0.04334798, 0.075069934, -0.06142525, -0.0354682, -0.049470313, -0.006548964, 0.058630697, 0.055025827, 0.012652485, 0.009741044, 0.06715426, -0.03244854, -0.031035516, 0.020393472, -0.052886397, 0.08834582, 0.023245005, 0.017296163, -0.022257758} + -0.00026503325, 0.015075458, 0.022475332, -0.059545167, 0.03813807, 0.02036686, 0.09369303, 0.013479007, 0.09444989, 0.11541978, -0.0963961, -0.020584984, -0.050419528, 0.075373605, 0.0655055, -0.019593518, -0.025856186, 0.014249504, -0.07262168, -0.017659204, -0.10128802, 0.08123276, 0.001960049, 0.030197127, -0.04030792, 0.031813275, -0.013740257, 0.038139883, -0.09525042, -0.039035097, -0.057990596, 0.019133989, 0.036438618, 0.0692773, -0.052344423, 0.037260283, 0.024247859, 0.033440772, 0.019159568, 0.05415563, 0.01177267, -0.027721975, 0.034039557, 0.039277744, -0.023239268, 0.0451868, 0.0061478717, -0.016408429, 0.07066666, 0.07111524, 0.07556712, 0.05038648, 0.020277819, 0.022456465, 0.1325548, 0.059136048, -0.05531785, 0.018664911, -0.031223211, -0.051543325, 0.08502753, 0.07717791, -0.03141322, 0.019849299, -0.027327877, 0.0021815347, -0.016466007, 0.039307635, -0.021545054, -0.033264067, 0.021569112, 0.036584433, 0.022011343, 0.0065075983, 0.057799615, 0.07239335, -0.042931464, -0.032916073, 0.019829186, -0.0062569706, 0.06385773, -0.13391589, -0.03106666, -0.008038394, 0.013604029, -0.10168154, -0.013549945, -0.00952448, -0.04212564, 0.06250119, -0.0775961, -0.02820033, 0.017832201, 0.019059727, -0.081784345, 0.027441157, -0.025310619, -0.017331742, -0.038811512, 0.10541721, 0.037507366, -0.009457436, 0.008557198, 0.12868558, -0.04227677, 0.03947787, -0.10547857, 0.059047405, 0.06209266, 0.05312278, 0.0020824817, 0.046173614, 0.008053163, 0.02955848, -0.037380483, 0.11394235, 0.017202245, 0.018510431, 0.03151664, 0.041667342, -0.022609226, 0.05896484, 0.015905662, 0.048379254, 0.004305682, -0.05372106, 0.022164864, -6.6429206e-33, -0.11220114, -0.10858408, 0.039742693, 0.046229597, -0.09005804, 0.05902126, 0.067520306, -0.05464574, -0.040991906, -0.053009246, -0.020272592, 0.13008188, -0.034473047, 0.045024805, 0.05872863, -0.041491453, -0.029205026, -0.024941018, -0.02506414, 0.04546409, 0.058141332, -0.037465636, 0.02166979, 0.09254683, -0.043003555, -0.039547645, 0.045002047, -0.13505226, 0.0028707853, 0.04407949, -0.014593066, -0.0191936, -0.040957324, -0.053792424, 0.0151997795, -0.07883926, -0.0016470567, 0.006253884, 0.021143146, 0.038236517, 0.049254715, -0.060134068, -0.10026592, -0.052788425, 0.04056701, 0.03189313, 0.03803019, -0.0042322385, -0.02455686, -0.07049825, 0.006753516, -0.050416023, 0.08184922, -0.028139638, 0.075519495, -0.052056056, 0.009816157, -0.023980962, -0.10322146, -0.037656397, -0.009970141, 0.085995145, 0.005993517, -0.029136078, -0.08064201, 0.07106184, 0.025150485, 0.031328112, -0.014743739, -0.04096777, -0.0028031087, 0.061052192, -0.042787857, -0.0144993, -0.02153063, 0.019266605, 0.09636558, 0.02224293, -0.056445498, -0.020088082, 0.017258525, -0.025676494, -0.008547281, -0.05112441, 0.080092765, 0.015569566, -0.023282446, -0.10977467, 0.0074417675, -0.019759273, -0.07135393, -0.069584064, -0.0035379876, 0.027057227, -0.076043546, 4.3862808e-33, -0.031652126, -0.06616955, 0.0150874425, 0.022637032, 0.013131262, -0.08941653, -0.0076044444, 0.015282084, 0.02934565, 0.1176537, -0.0020199588, -0.088932365, -0.053280387, 0.026544793, -0.03064991, -0.043374334, -0.013557422, 0.020844249, -0.07300656, 0.053101294, 0.011403258, 0.027736254, -0.010869594, -0.001730684, -0.033353385, 0.0054579754, -0.030351598, 0.01873584, -0.004821236, 0.06365363, 0.03860583, 0.07416209, 0.030442996, -0.044226587, -0.041673765, 0.064933546, 0.01810162, -0.0246678, 0.0036679886, -0.057921153, 0.06229866, -0.0018806888, 0.020416206, 0.046337333, 0.01503754, 0.017524645, 0.058918297, 0.0045170584, -0.038851157, 0.0044274684, -0.0348882, 0.013280185, -0.019361598, -0.051770102, 0.033446517, 0.020245772, 0.012953861, -0.018346183, -0.07367345, -0.10268045, -0.08835269, 0.07038049, -0.085578784, 0.09332974, 0.05431345, 0.009700606, -0.023735302, -0.11070074, -0.11515401, -0.03891971, -0.08780945, 0.002889208, -0.06239181, 0.0057181176, -0.01211082, -0.010898119, -0.02565034, 0.12286157, 0.036218476, 0.078943744, 0.0021716207, 0.020247554, -0.036964614, 0.030607672, -0.021478105, 0.0517363, 0.008547375, 0.038200203, -0.06693413, -0.00016272871, 0.089662544, 0.050362132, -0.01513772, -0.061934553, 0.04440066, -1.35450104e-8, -0.0028327182, 0.019114418, 0.02205391, -0.04002227, -0.011225868, 0.034393378, 0.0062507456, -0.020345204, 0.03193227, -0.06912925, 0.015212565, 0.061344862, 0.011303045, 0.0465152, 0.10057801, -0.047982763, -0.0033142806, -0.016255168, -0.014693881, 0.09813001, -0.05183757, -0.02020912, -0.04057556, 0.06430269, -0.035023723, -0.06742086, -0.046748966, -0.08615735, 0.078306004, -0.03901071, 0.043716334, 0.019368738, -0.004605733, -0.007337338, 0.014569598, -0.032543905, 0.0694859, -0.08584012, -0.01661318, -0.050223257, -0.057307478, 0.031399705, 0.008558557, -0.041517843, -0.056730315, -0.044458736, 0.07214165, -0.06065704, -0.034326863, -0.047764428, -0.0062505794, 0.058440637, 0.056496006, 0.013550112, 0.008445802, 0.06661782, -0.029131785, -0.029807873, 0.02028776, -0.05448606, 0.08661132, 0.023147527, 0.017579598, -0.023752382} + testVectorDifferentAllMiniLm := []float32{ // I like painting + //-0.06104912, 0.014903604, 0.023220936, -0.01141732, 0.027808411, -0.010105668, 0.10245573, -0.03140693, -0.0032928686, 0.003528042, -0.038120035, -0.04353736, -0.016490754, 0.062137403, 0.011410672, 0.017264983, 0.043102834, 0.037559688, -0.02188843, -0.012970595, -0.15947011, 0.0038327319, -0.024398685, -0.07338034, -0.0017653315, 0.046304513, 0.04577522, -0.027437776, 0.051729005, -0.055260208, -0.06721608, 0.057081044, -0.0033073595, -0.03316924, 0.024256205, 0.009002279, 0.011349048, -0.000117086776, 0.02645607, -0.004051567, -0.018992186, 0.008883905, -0.01230414, -0.007309912, -0.032141462, -0.06314189, -0.015241016, -0.01634787, 0.08650361, 0.023629162, 0.06708592, -0.026958518, -0.090697534, -0.050381236, 0.04565158, 0.005777015, -0.0324541, 0.03229667, 0.037340544, -0.06021884, 0.016757673, 0.04195077, -0.00770658, 0.044655874, 0.05610711, 0.015894733, -0.015977234, 0.025372434, -0.02812958, -0.0130566815, 0.05823168, 0.06602701, 0.003994167, -0.025433818, 0.06626987, 0.0033359502, -0.011535443, -0.023443788, -0.05026671, -0.07602762, 0.05790737, -0.04659444, -0.080175474, 0.034962934, 0.025993438, -0.0141365295, -0.028945746, 0.025945073, -0.053902704, 0.0070016035, 0.0433819, 0.035048436, -0.1470021, 0.01501351, -0.009748734, 0.010287717, 0.06772383, 0.014026755, -0.048807736, 0.11228072, 0.045069013, -0.012645957, 0.007921526, 0.010034467, 0.016563386, -0.006073092, -0.12123507, 0.05525804, -0.013579102, -0.061740678, -0.05267618, -0.0003591893, 0.004494855, 0.012835809, 0.00533403, -0.00721462, 0.061292794, -0.0459014, 0.045775536, -0.0032799237, 0.02531823, 0.02629354, -0.044764943, -0.018534377, -0.09482492, -0.09119269, -0.011551239, -5.4327393e-33, 0.0036783344, -0.010845926, 0.029552344, 0.0049977517, 0.06934922, 0.11348828, 0.056901287, -0.007733366, -0.042495232, 0.030546589, 0.038755443, 0.027421096, -0.0534008, 0.17277125, 0.0740854, 0.018232523, 0.043840013, -0.005648031, -0.06901591, -0.025457025, -0.05079258, 0.030278452, -0.0039731786, 0.09745195, -0.03832895, -0.026916958, 0.021845479, -0.057387885, -0.034460317, 0.058870774, -0.020467605, 0.013974356, 0.026962498, -0.050069775, -0.09905852, -0.10022592, -0.012498439, -0.048017528, 0.042293523, 0.04559824, -0.021347342, 0.038896643, -0.021155266, 0.04823061, 0.068692975, 0.03146007, 0.036269076, 0.06251356, -0.08241346, 0.032110527, 0.00026685378, 0.016679738, 0.0005253596, 0.05812038, -0.018468177, -0.027817009, 0.015808474, -0.04744563, -0.048090644, -0.045153014, -0.027596455, 0.112269044, -0.053117704, 0.01417182, -0.036934905, 0.07634702, -0.03352828, 0.026245464, 0.0064985957, -0.040864687, -0.08454131, 0.049722508, 0.008241364, -0.02996971, -0.077078156, -0.037095975, 0.08067038, -0.04231268, -0.053951003, 0.0003239718, -0.03716217, 0.027564721, -0.025196295, -0.056551423, 0.05358913, 0.017710084, 0.0073814616, -0.023918899, -0.030804103, 0.009322982, -0.0017035705, -0.007327673, 0.07522949, -0.008559504, -0.07406333, 5.031474e-33, 0.02799851, 0.010999593, -0.01292696, 0.06240963, -0.030641427, -0.08516889, -0.0064154738, 0.045437276, 0.017906802, 0.10718362, 0.0119746355, -0.048877418, -0.052952833, 0.06058984, 0.05082508, -0.017761692, 0.03349654, 0.033249862, -0.052110836, -0.017998504, -0.005293347, 0.06748351, 0.008741267, 0.006965011, -0.059996407, 0.042521887, -0.01430512, -0.034077592, -0.012919886, 0.100451216, 0.018008068, -0.0834566, -0.017417636, -0.06274758, 0.0039939065, -0.004832375, 0.04107382, -0.03657637, -0.05154561, 0.07501335, 0.02395177, -0.0713759, 0.052228212, 0.1131274, -0.011126797, 0.028643396, 0.025041306, 0.032150097, 0.05874275, -0.0035177465, -0.020878633, -0.011141557, -0.0065096123, -0.026106728, 0.045089606, -0.031360388, 0.03449681, 0.009146472, -0.017424135, 0.04627687, -0.032403734, 0.14236169, -0.068190776, 0.050516434, 0.009074026, 0.0036590837, -0.00418862, -0.065089434, -0.07351525, -0.03417449, -0.000024609128, -0.0013141176, -0.05706758, 0.07600855, -0.03375627, -0.03610806, 0.07825163, 0.11928668, 0.024086706, -0.027423277, -0.04527719, -0.04953151, -0.02931958, 0.0420605, 0.01633893, 0.0091801975, -0.067127444, 0.00354469, 0.017033542, -0.12725323, 0.08841526, 0.048890807, -0.028501153, -0.04903171, 0.022001712, -1.30600295e-8, -0.022528647, -0.011734431, 0.12333282, -0.09133731, 0.022337161, 0.009945905, 0.022030097, -0.0014934536, -0.022058787, -0.0036821188, 0.12384589, -0.013949616, -0.023324745, 0.0017868017, 0.119030595, -0.04387312, 0.08359736, -0.07160269, -0.005210772, -0.007918405, -0.07666402, -0.032551356, -0.0418511, 0.02645245, -0.07917314, -0.042891733, 0.028861264, 0.026427755, 0.023928558, 0.044784512, -0.038739603, 0.0948305, -0.036231473, 0.08252756, 0.01915809, -0.1070295, 0.017238509, -0.10477518, -0.06830786, -0.056523018, -0.026663298, 0.11125745, 0.047247633, -0.071547836, -0.002340591, -0.015267708, 0.15121834, -0.031948503, -0.069323756, 0.021878792, -0.028747914, -0.047735233, 0.099869385, 0.061902456, 0.09593885, -0.044290904, -0.050721165, 0.055774312, 0.052201502, 0.06001058, 0.07514168, 0.1018974, -0.0012816885, -0.03997642} // Quantize to int8 with scaling based on max absolute value + -0.07459795, -0.0036357427, 0.10344767, -0.02957443, 0.024534674, -0.010317706, 0.12891407, -0.031686064, -0.011855273, -0.011631835, -0.045536313, -0.04173978, 0.014325103, -0.006820153, -0.042745847, 0.081708886, 0.0032785663, 0.018128239, -0.11241298, -0.0701146, -0.080661304, 0.054724418, -0.077253535, -0.03769372, 0.022452751, -0.022737162, 0.016582014, -0.053428907, 0.10224209, -0.07625492, 0.027221438, 0.027978001, -0.016469117, 0.010159522, 0.01584792, 0.02984694, 0.023159249, -0.019402092, 0.035658613, -0.0012141381, 0.025697114, -0.06869701, -0.012642167, -0.06691865, -0.012499855, -0.0007137546, -0.013339532, -0.016676815, 0.029304655, 0.008842667, -0.025772143, -0.10655647, -0.086969316, -0.038206637, 0.0005408013, 0.01566182, 0.042610563, 0.06274439, 0.093975, 0.012461853, -0.033833805, 0.026355995, -0.018735882, 0.08214099, -0.00813476, -0.024960287, -0.029287435, -0.011752455, -0.031737417, -0.025373232, 0.09504904, -0.012184118, -0.06727466, 0.019844383, 0.0252968, 0.0479847, -0.011147, -0.03145276, -0.03036906, -0.058147073, 0.052154876, -0.013574374, 0.011107711, 0.07306001, -0.0045128977, 0.0024352993, -0.08108868, 0.08246334, 0.053491123, -0.05678999, -0.001039209, 0.047789287, -0.14025138, 0.04996376, 0.036255937, -0.027742542, 0.054067086, -0.04205478, -0.043733742, 0.12992564, 0.029462995, -0.017760452, 0.013864652, 0.013207492, 0.040398724, -0.052277934, 0.011208582, 0.02569698, -0.03573965, -0.024361987, 0.0054076384, 0.019099778, -0.014007003, 0.042919263, 0.008015992, -0.083415344, 0.06390634, -0.012149615, -0.021805173, -0.040232457, 0.029453617, 0.051060364, -0.04106353, -0.028186833, -0.14582399, -0.08209801, -0.032885104, -3.3129585e-33, 0.04317872, -0.015179548, 0.028228972, 0.019618584, 0.08538256, 0.05553127, -0.001089586, -0.013136522, 0.018747143, -0.015125277, 0.03628249, -0.057955053, -0.12264477, 0.07205428, 0.068179235, 0.09500875, 0.030803274, -0.029928269, -0.067025386, 0.013936646, -0.032514114, -0.0032202331, 0.004133362, 0.014336419, -0.05252771, 0.036434688, 0.06846874, -0.10649512, -0.00080334157, 0.011769146, 0.0016460244, 0.0057864105, 0.050556816, -0.09684867, -0.109302774, -0.063831106, -0.019218182, -0.064683184, 0.010395815, 0.07914081, -0.010902619, 0.032425135, 0.0049673696, -0.0007414359, 0.02184415, 0.079413936, 0.03387684, 0.03307936, -0.022764912, 0.0449972, 0.059696432, 0.031108867, -0.08596792, 0.03417844, -0.03572083, -0.003328892, 0.011035539, -0.019110395, -0.008628456, -0.0062794047, 0.004295993, 0.055653173, -0.025304485, 0.069029994, -0.06755905, 0.005820481, -0.05182164, -0.0012875412, -0.008127819, 0.02017094, -0.09890171, -0.004588147, 0.03164047, -0.060888223, 0.007695157, -0.057196293, 0.055448163, -0.042828467, -0.08114355, -0.0036861824, -0.035136558, 0.056899816, -0.03759483, -0.012502428, 0.031821042, -0.07493797, -0.01057634, 0.021683386, -0.07162313, 0.007956869, -0.03845529, -0.00074685924, 0.027632337, -0.08157677, -0.07804552, 1.48204105e-33, -0.03165973, 0.041388612, 0.08623291, 0.041831877, 0.05317078, -0.07514822, 0.0039769714, 0.09873683, 0.06895161, 0.06537179, -0.020403648, -0.0136354705, 0.027364718, 0.039866798, 0.07224751, 0.00004120712, 0.07714875, 0.032805007, -0.00021281301, 0.0058218036, -0.024003422, 0.040813055, 0.057028107, -0.012195326, -0.03129729, 0.034900714, 0.01582385, 0.037040167, 0.030853676, 0.07903258, 0.071694665, -0.097146176, -0.038988214, -0.027833315, 0.022041753, 0.009385031, 0.07447928, -0.11263658, -0.028474702, -0.013990957, 0.05430063, -0.0869082, 0.036223486, 0.1343045, -0.0061596856, -0.012056845, -0.024382673, 0.018193068, 0.010967213, 0.05812516, -0.060131732, -0.04541531, -0.07966173, -0.015034394, -0.032179546, -0.05935209, 0.04953889, -0.008035148, -0.06928248, 0.02086842, 0.017073058, 0.074477345, -0.08599521, -0.03138777, -0.050922647, 0.023714188, 0.025719997, -0.052466083, -0.042045634, 0.011180476, 0.04398527, 0.07303308, -0.043384995, 0.017309643, -0.04184067, -0.030134749, 0.051943142, 0.058808792, 0.034678698, -0.08569526, -0.008200412, 0.024245376, -0.011175224, 0.002166839, 0.09478579, -0.011244079, -0.0022903266, 0.043870196, 0.031899977, -0.11630824, 0.061978973, 0.07322082, 0.016802007, -0.06106449, -0.031055382, -1.3560151e-8, 0.025527768, -0.033168238, 0.051803157, -0.08845124, -0.010908753, -0.018720092, 0.043603744, 0.015692104, 0.007937755, 0.022989975, 0.07828886, -0.019803535, -0.020645497, 0.0054351473, 0.11235825, 0.010573282, 0.007315352, -0.044072036, -0.012171655, -0.03161611, -0.006209986, -0.004772915, -0.055332735, -0.0116514675, -0.06667689, -0.014245001, -0.003074604, 0.028866546, -0.03650874, 0.060700018, 0.03929413, 0.12655672, -0.017765807, 0.027076198, 0.020298006, -0.034795865, 0.035146493, -0.060159877, -0.013837136, -0.06596605, -0.021909382, 0.11527764, 0.119108595, -0.03797796, -0.017065054, -0.012996933, 0.1304784, -0.05534349, -0.0014606715, 0.047797706, -0.10322347, -0.077236205, 0.0981735, 0.078745574, 0.08450114, -0.058746666, 0.0074427095, 0.019684453, 0.016092828, -0.026871026, 0.03337777, 0.046438698, 0.033300474, -0.07242167} + _ /*testVectorVeryDifferentAllMiniLm :*/ = []float32{ // The cat sat on the mat + 0.13048664, -0.011919865, -0.028140409, 0.051155645, -0.055907402, 0.030133465, 0.03008591, 0.02468966, -0.018288225, 0.058831353, -0.024911208, 0.0602037, 0.03983082, 0.03314179, -0.06126634, -0.049416166, -0.05492195, -0.040052682, 0.05646088, 0.039157756, -0.034732893, -0.013254967, 0.03194261, -0.06353775, -0.060165808, 0.07827359, -0.028283782, -0.047334697, 0.04041871, -0.006649382, -0.06677346, -0.004115102, -0.025345739, 0.053302642, 0.017430358, -0.09785545, 0.006133795, -0.06525492, 0.04565119, 0.023530664, 0.07664405, -0.010146744, -0.0039734305, -0.062314127, 0.03381293, 0.018628646, 0.103016935, -0.10929713, 0.0636556, -0.019380782, -0.07295992, 0.045463637, 0.06734996, 0.028013198, -0.11000589, -0.048611697, 0.091219835, -0.048649758, 0.00083015923, -0.05613279, -0.017406508, 0.03163069, 0.08366649, 0.07424378, 0.05664864, -0.053970065, -0.07304853, -0.013076917, 0.00742499, 0.077441074, -0.014492867, 0.055916082, 0.028261166, -0.013176955, -0.024463816, -0.048578363, 0.14258409, -0.021817485, 0.064296976, 0.05238156, -0.028108971, 0.04367377, 0.06332979, 0.034892768, -0.017096704, 0.053062625, 0.050363973, 0.059073053, -0.017692136, -0.07121594, 0.06788449, -0.0046298644, -0.039573852, 0.09269569, -0.07235346, -0.056801807, -0.02299147, 0.03056096, -0.0041623097, 0.045079544, -0.027446633, 0.036960624, -0.03391003, 0.10662218, -0.038328383, 0.016436612, 0.043561827, 0.006481182, -0.0028152433, 0.009053033, -0.01703272, -0.01705024, -0.05990183, 0.080696255, -0.001700385, 0.043370113, 0.034264, -0.060818765, 0.015088729, -0.06739484, 0.056619875, -0.100070976, 0.007800275, 0.03303524, -0.05412017, -0.07451364, -0.053214327, -5.4238463e-33, -0.051442936, -0.021370977, -0.009331001, -0.09380494, 0.07620661, -0.01278296, -0.011350921, 0.0038817138, -0.036102414, 0.013259333, -0.03710652, -0.0128600355, 0.0086069, -0.026467202, -0.046073675, -0.06307329, -0.0014069504, -0.022493, -0.068869404, -0.0040173787, 0.06818951, 0.03297601, 0.038807873, -0.0022290314, -0.008278547, -0.041061826, 0.020795876, -0.053864088, -0.016272655, -0.0073406366, 0.01939233, -0.031721435, 0.007990043, 0.031795308, -0.059140626, -0.032928057, -0.005922369, 0.052353214, -0.020875119, 0.0369506, 0.034405846, -0.018235022, 0.10980524, 0.07118593, -0.013605745, -0.009611244, -0.019753048, 0.052293062, 0.030535933, 0.028512249, 0.088509314, -0.015859347, 0.037009, -0.13584425, -0.035720762, -0.04343665, -0.021737041, -0.102774665, -0.015280403, 0.0741961, -0.013669474, -0.15776323, 0.05220253, -0.006133216, -0.0074898843, -0.071369864, 0.068588205, -0.09694736, 0.078042716, 0.06365276, -0.07227003, 0.07175319, -0.0030972795, -0.07124749, -0.015773376, -0.05093138, -0.02679844, -0.005969917, -0.052887253, -0.01022139, 0.03876654, -0.009207067, -0.012384491, 0.034041427, -0.028069397, 0.041060187, -0.044471942, 0.04092741, -0.052609615, 0.062520005, 0.0101666795, 0.08044257, 0.028806355, -0.0991476, 0.06529705, 3.2358428e-33, -0.054091122, -0.036610797, -0.06272632, -0.006313029, -0.058314133, 0.014827997, 0.07250375, 0.059187092, -0.036162548, 0.07683908, 0.034274895, 0.0839659, 0.07337762, 0.04061792, -0.0030480006, 0.050480414, -0.08435812, -0.022244683, 0.056114707, -0.041436315, -0.05246669, 0.05456836, 0.009522461, 0.011393692, -0.023598816, 0.071093746, 0.045307755, 0.018453652, -0.015017816, -0.09705776, -0.019676553, -0.051059015, -0.033472463, 0.023759516, 0.036451098, 0.049354088, -0.05500444, -0.060264703, -0.0853677, -0.04508235, 0.01756655, 0.029194582, -0.04452763, -0.03271438, -0.050797865, -0.0072234333, -0.03452147, 0.031807538, 0.07032544, 0.029583305, -0.021873087, -0.04918509, 0.062241625, 0.049187277, 0.059380133, 0.09057832, -0.06268607, -0.009115403, -0.012920506, 0.086204566, 0.02683542, 0.04548301, 0.08562212, -0.038422823, -0.014889057, -0.06449867, -0.018943133, -0.09259397, -0.020308863, 0.0092616435, 0.02293208, 0.11730157, -0.018272737, -0.031652577, 0.038939744, 0.054753765, 0.044844113, 0.025942102, -0.052019764, -0.07446514, -0.016064938, -0.007188292, -0.016875593, 0.02832708, 0.03083801, -0.003977766, -0.006918571, 0.013167047, -0.03156549, -0.03552535, 0.07299735, -0.0408696, 0.043344863, -0.03202345, -0.056658793, -1.4476629e-8, -0.12428533, -0.016611718, -0.10717607, -0.028831389, 0.0046796156, 0.0067803576, -0.028829733, -0.16836958, 0.006484609, 0.0040715523, 0.016060539, -0.049067084, 0.028722703, 0.0112348385, 0.017697943, 0.064410634, -0.104054235, -0.0056282836, 0.022228984, 0.035177495, -0.017269313, 0.0155253755, -0.09587546, 0.044928845, 0.019448906, 0.0032065203, -0.041159008, 0.034010924, 0.03173974, -0.04484175, 0.064728014, -0.063133635, 0.05313335, -0.04023208, -0.013310149, -0.015457827, 0.042068712, -0.068351455, 0.07385936, 0.005606699, 0.07342076, 0.030856185, 0.07226707, -0.011680676, -0.04156735, 0.050358485, -0.013742303, -0.012211849, 0.048219405, 0.04635257, -0.0049878517, 0.03864635, -0.020845456, 0.008952367, -0.06626546, -0.06983379, -0.0031024548, 0.01958658, 0.027793504, 0.08162553, 0.03814222, 0.04463311, 0.00037500626, 0.052174974} + _ /* testVectorOppositeAllMiniLm :*/ = []float32{ // I hate soccer + 0.03073619, 0.0672733, 0.013577526, -0.026341103, 0.10580283, -0.0036627285, 0.046719737, 0.044254195, 0.099522404, 0.0766663, -0.058527403, -0.069810726, -0.025450015, 0.037854023, -0.039023753, -0.026138123, -0.070040375, -0.06750584, -0.036355037, -0.017851872, -0.1593227, 0.029997755, 0.015753811, 0.030398702, -0.06641918, -0.0011088259, -0.014853733, 0.020238021, -0.052554082, -0.048208605, -0.031731807, 0.024576645, 0.028920354, 0.053032063, -0.022498446, 0.007894143, 0.08485788, -0.008321334, -0.005665564, 0.044620167, 0.0076246504, 0.00900192, 0.01826108, -0.0043945927, 0.02515705, 0.0133490665, 0.074276835, -0.034294184, 0.04227429, 0.029806538, 0.05010392, 0.07406747, -0.030679824, -0.0012436415, 0.06943961, 0.05486093, 0.024835903, 0.12017188, 0.00986629, -0.0057805195, 0.025960289, 0.02491845, -0.036308248, -0.006121572, -0.024595771, -0.009754752, -0.0043792534, 0.012299738, -0.00615844, 0.026369447, -0.018828334, -0.007300376, 0.029197754, 0.070713826, 0.051681012, 0.031854596, -0.050213013, -0.06493851, 0.005761724, -0.045813337, 0.00016284757, -0.063146286, -0.0340701, -0.004217967, 0.04291752, -0.0659886, -0.008198367, 0.05061145, 0.028742235, -0.0828165, -0.056082364, 0.018941933, 0.031164382, 0.066764474, -0.023944732, 0.07805574, 0.036568664, -0.041789353, -0.07328429, 0.124633275, 0.01930349, 0.017790968, -0.034636285, 0.11360519, 0.021229597, 0.023149515, -0.016532049, 0.06865896, -0.00989037, 0.0034763212, -0.051746085, 0.035849486, -0.043063357, -0.06617757, -0.08545945, 0.09180195, 0.07481251, 0.041540276, -0.014785054, 0.010921551, -0.004722372, 0.012415321, -0.04348268, 0.10357827, 0.019236527, -0.030660965, -0.03471962, -5.9457107e-33, -0.07585321, -0.07472732, 0.004377012, -0.013422198, -0.049570005, 0.07230977, 0.07238058, 0.03114329, 0.00443432, -0.023772564, 0.009045917, 0.0009968879, -0.0421516, 0.01815693, 0.12020621, -0.021062732, -0.006317674, -0.025352525, 0.013961705, -0.013043824, 0.032002166, 0.05488193, 0.041364618, 0.020203808, 0.04155481, 0.022235561, -0.01502922, -0.101650015, 0.0035190687, 0.03131111, 0.031652696, -0.044982232, -0.038202118, 0.002288888, 0.017260142, -0.09593779, 0.0050037885, 0.068771504, 0.013339659, 0.007469385, 0.017196719, -0.050488964, -0.07150565, -0.029121289, 0.088554546, 0.06189982, 0.043262195, -0.03151824, -0.0020789385, -0.062973365, 0.026048549, -0.04694116, 0.12505011, 0.0479491, 0.066019185, -0.061377, 0.006827211, -0.04084253, -0.0857057, -0.059387665, -0.023615954, 0.042772718, -0.0015311298, 0.036895398, -0.023395091, 0.019212814, 0.013395291, 0.036047608, 0.0031558275, -0.10449527, -0.03468707, 0.074604675, -0.06484455, -0.038725987, -0.020947153, -0.065849006, 0.13778889, 0.043382823, 0.024641823, -0.027533723, -0.013809081, 0.0067156847, -0.019839168, -0.10511476, 0.084299006, 0.041206397, -0.015014497, -0.048522163, 0.02894298, 0.0013893154, -0.0806037, 0.0014659471, -0.031838287, 0.039756056, -0.07123865, 4.1817668e-33, -0.04675924, -0.07774842, 0.07364093, -0.023340756, -0.101991124, -0.043180257, 0.042417333, 0.06979184, 0.020866552, 0.11663095, 0.04600535, -0.076858565, 0.013309526, 0.014863771, -0.026620844, -0.08709046, -0.0060877125, 0.03480367, -0.04669644, -0.005893547, 0.0015182677, 0.0992499, -0.0076940823, -0.038822077, -0.05699561, -0.01921386, 0.014193468, 0.009300198, -0.058080476, -0.014631267, 0.11277131, 0.04404066, -0.044560153, -0.069191106, -0.0030806996, -0.0044606877, -0.1162025, 0.032916006, -0.03535747, -0.02109387, 0.003408774, -0.04108975, -0.00017888265, 0.016206583, 0.04313857, -0.037254684, 0.03860412, -0.03733544, -0.04383569, 0.011839245, -0.028374508, 0.015605336, -0.0665902, 0.030938603, 0.07518519, -0.02658369, -0.03681646, -0.008281727, -0.073420756, -0.0929924, -0.05277689, 0.016222117, -0.10501961, 0.10831985, 0.08959705, 0.038623516, -0.036617033, -0.026401922, 0.015167218, -0.024735728, -0.047426492, 0.0077552143, -0.09818886, 0.03955498, -0.02474939, -0.017510554, -0.09239306, 0.16793069, 0.060626842, 0.025161369, 0.019927513, -0.016395349, -0.024691224, 0.034714386, -0.02748802, -0.0005494354, -0.00073686166, 0.058974225, -0.0005001918, 0.04652181, 0.07503444, 0.0034789906, 0.073712364, -0.011112546, 0.035833098, -1.3438642e-8, -0.03320686, -0.004973912, 0.08796283, -0.025374409, -0.061817136, -0.032757282, -0.08640939, -0.01688524, -0.00832073, -0.011403044, 0.01591934, 0.047195833, -0.005397019, 0.020348232, -0.0058946204, 0.055142153, 0.0078276405, 0.011290255, 0.012327588, 0.10957547, -0.07111819, -0.0064812573, -0.066065304, 0.044631306, 0.0248234, -0.10448463, -0.018851314, -0.07580424, 0.08861452, -0.097415626, 0.027773611, -0.027997456, 0.026699513, 0.059258867, -0.06438902, -0.025169916, 0.06324586, -0.09262697, -0.04779945, -0.08794006, -0.01658876, 0.039915014, -0.029976174, -0.0820367, -0.02387641, -0.00091172056, 0.017283639, -0.06622199, -0.054849457, 0.00007067042, 0.018533865, 0.07052602, 0.038967937, 0.0398982, 0.034082312, 0.045195032, -0.02215715, 0.05723465, -0.0017157646, 0.027635671, 0.10046033, 0.0054611294, 0.030630669, 0.009891892} + + inputVectorMiniLML12v2 := []float32{ // how to cook Italian food with pasta + -0.051910903, -0.09873738, -0.006625238, 0.06688455, -0.069010854, -0.0058009275, -0.0018220361, -0.089612775, -0.021037431, -0.059018116, -0.023123166, -0.019236505, -0.04234267, -0.040841237, -0.03628091, -0.09963261, 0.10062731, 0.078003265, -0.02176751, -0.022565274, -0.025203416, -0.08540319, 0.025021799, 0.0075420192, 0.09082782, 0.030083112, 0.059367593, 0.021078212, -0.02452513, 0.003302518, 0.0042612096, 0.017507778, 0.11180687, -0.08006367, 0.02988426, -0.044762578, -0.027003055, -0.028466944, 0.023978112, 0.0014585649, -0.025926486, 0.07698701, 0.017326837, 0.036951452, 0.053695433, 0.009532915, 0.0005221323, -0.022048343, 0.035520785, -0.0008733863, -0.07657538, 0.00034925254, -0.04164522, -0.032891076, 0.035336506, 0.04119879, -0.00046274962, 0.0050994437, -0.039689254, 0.09140239, 0.046760205, 0.012512438, 0.0445274, -0.011328778, -0.019819109, 0.012645455, -0.011327148, 0.10074061, -0.058920052, 0.040605333, 0.058282312, -0.07203747, 0.00734831, 0.0037602065, 0.030506657, -0.027270721, 0.07971705, 0.009461679, -0.077020355, 0.010658337, 0.01875286, 0.01916014, -0.017117795, -0.040193427, 0.030163754, 0.076309055, -0.025336668, 0.072964326, 0.062311277, 0.06449047, 0.088159844, 0.016829979, 0.008986058, -0.05030687, 0.06471198, -0.02045803, -0.080121756, -0.05225282, -0.013904329, -0.011656363, 0.041437216, -0.09131365, -0.00064404524, -0.025794258, 0.011534211, 0.06434632, 0.08564298, 0.04148573, 0.028485132, -0.009569608, -0.031954475, 0.042558648, -0.047317136, -0.034791153, -0.040797252, -0.023711126, 0.13431557, -0.024826514, -0.017625038, 0.0443526, 0.004700405, -0.027227333, 0.017165866, 0.0437459, 0.008692525, 0.03313914, -0.043812666, -0.038743198, -0.10353606, -0.033346694, 0.071863, -0.015681814, 0.018698772, 0.023332464, -0.0034010764, -0.12611155, -0.02790519, 0.050382882, -0.002957438, -0.041436642, -0.0855947, -0.011777269, 0.02329986, -0.050933, 0.050358374, -0.008804646, -0.00961855, 0.014914757, -0.028489167, 0.004302857, -0.013052722, -0.020815808, -0.015050326, 0.05495065, -0.017739965, -0.058537986, -0.07766797, 0.017509526, -0.04823639, 0.040218547, -0.08552997, 0.039317776, -0.06163686, -0.026327021, -0.055980943, -0.0127132535, -0.029052394, 0.12370856, 0.03687506, 0.014132912, 0.00536813, 0.015092475, 0.02448913, -0.003390164, -0.10670174, 0.088625334, 0.00872236, -0.0606897, 0.023005104, 0.0008256416, 0.036255904, 0.04178777, 0.05237479, 0.03228049, 0.037539877, 0.04491029, 0.055563428, -0.026074436, 0.0013321448, 0.015828524, 0.0005862741, 0.0077792914, 0.018128915, 0.034375284, -0.09072214, 0.033476412, -0.030136509, 0.022641685, -0.040670134, -0.0048842877, 0.019821255, 0.0088937925, 0.07374109, -0.048620984, 0.06591753, -0.03173762, 0.010652225, 0.0012024406, -0.0057228915, -0.008416755, 0.0028963112, 0.035541102, -0.012294424, 0.09181212, -0.044719532, 0.030504381, 0.1285454, 0.1047518, -0.045882702, -0.022767575, 0.032155607, 0.001171402, -0.08036969, 0.0018821534, -0.027624926, -0.05752739, 0.036017593, -0.043700874, -0.06403938, -0.0948449, -0.06286356, -0.0784886, 0.044559658, -0.023434594, 0.0058916803, -0.040606126, 0.04325154, 0.016717562, -0.07538447, 0.11082793, 0.107878305, 0.039541118, -0.004494072, -0.047299962, -0.092755795, 0.0019849637, -0.004614413, -0.0040730704, -0.09682122, 0.020659043, -0.031615082, 0.072006665, 0.009564792, -0.08851862, -0.05365843, -0.048634794, 0.07988213, -0.048007984, -0.07480467, 0.10373426, 0.05015706, -0.0766858, -0.038576398, 0.053575326, -0.05612505, 0.01964998, -0.057316985, -0.005387602, -0.037251245, 0.03878229, 0.010341385, -0.055018876, -0.070961654, -0.051198334, 0.06566611, -0.054627527, -0.03900447, -0.049635783, -0.075480714, 0.021840394, 0.063512824, -0.042248912, -0.03237607, 0.011128944, 0.053680476, -0.0010072244, 0.049485147, 0.0002466048, 0.0858916, 0.016595792, -0.045581765, 0.019391466, 0.05054912, 0.028918978, 0.053485557, -0.005822028, 0.02334028, 0.13944173, 0.024322052, -0.0051856474, -0.051886994, 0.03253299, 0.026580987, -0.02957318, -0.024856338, -0.04353269, -0.0984795, 0.00700661, -0.0003095741, -0.0134982215, 0.004381332, 0.03992673, 0.080635175, 0.05076666, 0.040150974, 0.030279681, 0.08862587, 0.03331206, 0.038470622, -0.0042904257, 0.042530272, -0.08509026, -0.05889182, 0.04476452, -0.062312596, -0.04046271, -0.013083442, -0.10779484, 0.04956914, -0.05278548, -0.06437094, 0.1434806, 0.01537219, -0.009702637, -0.07441775, -0.05449095, 0.005450132, 0.04088098, 0.038402222, -0.011447451, 0.036152072, 0.05482206, -0.09191833, -0.037204962, -0.038913056, -0.030865766, 0.02238842, 0.0771639, 0.017221063, -0.029624173, -0.02301016, -0.041128132, 0.0035721455, 0.06480318, 0.0015180971, 0.004151661, -0.10884463, -0.07291863, -0.030071592, -0.08745532, -0.0127715375, -0.03635802, -0.06322675, -0.023912076, 0.004302191, 0.00741882, -0.03747852, -0.0020934397, 0.051305164, 0.0675664, 0.064163275, 0.059868738, -0.03475481, -0.013358178, 0.0570977, 0.050238594, 0.11000183, 0.06728645, 0.097570665, 0.009692781, -0.09072435, 0.0080292085, 0.032666184, -0.07838803} + testVectorSimilarMiniLML12v2 := []float32{ // The best way to cook pasta is in salted boiling water + -0.045823455, -0.048972037, 0.008796681, 0.03675673, -0.018018363, -0.008431158, -0.016497428, -0.033608768, 0.008252222, -0.028790843, -0.07066961, -0.071192496, -0.046460014, -0.03826459, -0.011072058, -0.07634837, 0.08092094, 0.11833273, -0.03303441, -0.037160423, 0.0016054236, -0.04034145, -0.002679883, 0.0014089398, 0.10667613, 0.037194006, 0.050429232, 0.002006806, 0.010691076, -0.0034122497, -0.006953284, -0.022374237, 0.0830232, -0.08019036, 0.029277263, -0.019638153, -0.015017912, -0.03271017, -0.011948459, -0.02568863, -0.033758372, 0.0743138, 0.035590354, 0.016362678, 0.067350455, 0.03327311, -0.026179174, -0.029174369, 0.007209186, 0.00210063, -0.017214412, -0.016033836, -0.061859045, -0.0025561315, -0.0019381652, 0.0139426, 0.007565983, -0.054263845, -0.038763415, 0.014299047, 0.00791813, 0.033968106, 0.07396812, -0.05343529, 0.03954224, 0.007414248, 0.004553288, 0.10683855, -0.008179807, 0.00010134202, 0.06811742, -0.022223402, -0.021678122, 0.022250831, 0.035378795, -0.04226118, 0.08874044, 0.030355878, -0.087566815, 0.08271158, 0.015504485, 0.0048547112, 0.022500731, -0.04612983, 0.0048025604, 0.13337834, -0.022485744, 0.06045236, 0.050551575, 0.051888563, 0.013191204, 0.03696745, -0.00893193, -0.06639712, 0.03663068, -0.018916937, -0.05034685, -0.026395056, -0.0028380028, -0.035320904, 0.016370362, -0.124574885, -0.06215982, -0.070426986, 0.07447499, -0.0011591052, 0.12163509, 0.060571462, -0.016597614, -0.03377691, -0.011419524, 0.015554542, -0.023570629, -0.02606497, -0.046508815, 0.008105875, 0.09336857, 0.018458785, -0.04565364, 0.07352897, -0.016676387, -0.020727212, -0.015379027, 0.05519825, -0.0064194435, -0.02506149, -0.024846278, -0.07868531, -0.09123554, -0.03649118, 0.112386584, -0.026244901, -0.013627047, 0.040297102, 0.030084614, -0.107071966, -0.02903426, 0.08028816, 0.0022323574, -0.008662374, -0.12674773, 0.013387756, -0.012425549, -0.10675291, 0.06875062, -0.025511835, -0.04582033, -0.041924775, -0.0035265998, -0.017813066, -0.03344154, 0.024959989, 0.026571665, 0.022366218, 0.025739715, -0.077138074, 0.0003010389, 0.0034371465, -0.05230219, 0.036877822, -0.04658454, 0.042617954, -0.0693144, -0.057646465, -0.10243886, 0.041172728, 0.0028804566, 0.081431456, 0.0067226905, 0.020060776, -0.011637629, 0.061662514, 0.024131997, 0.048320804, -0.08478032, -0.012642317, -0.008818161, -0.055937026, 0.024178013, 0.0073237657, 0.017797189, 0.079335585, 0.06755656, 0.016181726, 0.062904224, 0.022081358, 0.003998886, -0.052394446, -0.025659528, 0.047495376, -0.053127926, -0.03385437, 0.012771871, 0.026191313, -0.08528029, 0.048483543, -0.0066156606, -0.0020714896, -0.05996379, -0.025547512, -0.0058149225, 0.016969811, 0.07883078, -0.022595964, 0.0995315, -0.0036591904, 0.026017684, -0.006296987, -0.039697986, 0.060058054, -0.03122911, 0.057665654, -0.08690926, 0.07919429, -0.024974797, 0.049945682, 0.10048715, 0.12044665, -0.0005642919, 0.018785072, 0.049286474, -0.036919102, -0.060869526, 0.031649202, -0.0028853682, -0.07313864, 0.013786784, 0.0263413, -0.066373646, -0.10617026, -0.066231765, -0.04374511, 0.015530247, 0.014093576, -0.003866945, -0.09774392, -0.0012303882, 0.058588855, -0.06619392, 0.09135929, 0.087400705, 0.062126312, 0.013326723, 0.012896491, -0.06952124, 0.07605913, 0.015795466, -0.031484835, -0.09897389, -0.054433957, -0.038068864, 0.0489534, 0.020751808, -0.07990933, -0.07811257, -0.015996572, 0.052960336, -0.010777821, -0.05311615, 0.071119085, 0.08740517, -0.09585122, -0.07334201, 0.0077761374, -0.023199271, -0.034925584, -0.007578449, -0.029778326, -0.082436405, 0.033143148, -0.0071716835, -0.06251193, -0.0590953, -0.022333357, 0.0058513354, -0.055191353, -0.04559454, -0.025147686, -0.05500208, -0.012213878, 0.1029891, -0.07050941, -0.044585887, 0.039899044, 0.08199365, -0.06841102, 0.049963333, -0.037333492, 0.084873155, 0.027173353, -0.02553306, 0.046098754, 0.034392446, -0.0066606477, 0.035723314, 0.0024793365, 0.10378375, 0.10555831, 0.002591817, 0.008807137, -0.036919322, 0.01323392, 0.042454924, -0.06457571, -0.063791126, -0.00391344, -0.12065874, 0.06489235, 0.02927541, -0.03909657, -0.002311476, -0.021337079, 0.038917556, 0.0450457, -0.0062642433, -0.013796476, 0.05479075, 0.034091115, 0.025995301, 0.028928991, 0.07991342, -0.09445184, -0.060534507, -0.006793155, 0.0037917916, -0.013997857, -0.028023, -0.04237354, 0.053840186, -0.009557819, -0.07496013, 0.120900944, 0.037161388, -0.06813784, -0.027949555, -0.048233833, -0.013430864, 0.046379764, 0.027088545, -0.058503374, 0.07756782, 0.03389115, -0.07997618, 0.038452774, -0.0038008622, -0.04292955, 0.08029038, 0.06754243, -0.005182194, -0.054250535, -0.004233686, -0.04773332, -0.022486128, 0.06172529, -0.015875045, 0.062239487, -0.040033054, -0.03687552, -0.009383547, -0.05711749, -0.015387162, -0.026188063, -0.07323029, -0.038482346, -0.01786518, 0.009514836, -0.027326003, 0.05050068, 0.021799957, 0.04796301, 0.075394444, 0.033295367, -0.0041842894, 0.08678971, 0.0654416, 0.06455565, 0.025113864, 0.035733853, 0.08730936, 0.021946603, -0.082417324, 0.018029293, 0.05751288, -0.025935091} + testVectorDifferentMiniLML12v2 := []float32{ // La cocina italiana usa tomates frescos y aceite de oliva + -0.016220659, -0.09935467, 0.008660855, 0.07030659, 0.0021901277, 0.011052067, 0.035248544, -0.097585, -0.007999923, -0.06742765, 0.04634157, -0.02623863, -0.103894494, -0.06808304, 0.012728221, -0.11537897, 0.111009866, 0.001960284, -0.022099037, -0.015415656, -0.001317669, -0.07959768, 0.006150931, 0.011437951, -0.012029995, 0.049631983, -0.016517097, 0.04146258, -0.024801342, -0.022423545, 0.04741652, 0.018011123, 0.116775684, -0.018636214, 0.012734692, -0.0680838, -0.061250072, -0.03184254, 0.07582365, 0.033315886, 0.06391058, -0.046497613, -0.050318506, 0.048717048, 0.04140539, 0.018501462, 0.04253094, -0.017311942, 0.011219875, 0.021979462, -0.050367385, 0.004708854, -0.054685783, -0.17822848, 0.0085408585, -0.0032780357, -0.014178343, 0.018422557, -0.0343337, 0.13896483, 0.01449159, 0.028212816, -0.045814738, -0.005936207, -0.077096924, -0.051417217, 0.04614229, 0.11666386, -0.09496357, -0.012096931, 0.07048602, -0.02086566, -0.03741655, -0.06327926, -0.08160803, -0.013542454, 0.04306303, -0.08271032, -0.09766073, -0.027537055, 0.0566967, -0.06286115, -0.05188471, -0.022778701, 0.039508488, 0.031161178, -0.033511437, -0.002422655, 0.079336725, 0.03513377, 0.07300767, 0.007265792, -0.015581321, 0.0028285172, 0.058281373, 0.02420161, -0.08582896, -0.06331777, -0.043617304, 0.023134256, 0.02243348, 0.015723603, 0.0147757055, -0.06258828, -0.04923862, 0.03101264, -0.025885612, 0.022344034, 0.011860691, -0.0003335158, 0.0040477924, -0.042919178, -0.057693955, -0.09686156, -0.035663553, -0.082863286, 0.051911924, -0.047240175, 0.005454689, -0.0034121803, 0.020569142, 0.017134015, 0.07511835, 0.05115754, 0.013380866, 0.033873815, 0.01620104, -0.0023570275, -0.09196833, -0.0014959304, 0.08963653, -0.026844222, -0.062102057, 0.07171999, -0.05826423, -0.03637349, -0.0044514476, -0.05218895, 0.04854477, 0.013980256, -0.068532124, 0.024546107, -0.013663807, -0.017021505, 0.085067295, 0.002521376, 0.0043717166, 0.001629899, -0.060161173, 0.016043432, -0.01847703, 0.023679245, -0.008264348, 0.042934895, -0.029080065, -0.094744, -0.017515283, 0.067782514, 0.02891836, 0.01652027, -0.08569398, 0.03233421, -0.018413654, 0.051742084, -0.06740074, -0.048360057, -0.050872248, 0.08875556, -0.039181985, 0.053425554, 0.05680936, 0.0028156522, -0.0107310265, 0.06101587, -0.13254094, 0.10005518, -0.0033396864, -0.021697085, 0.017970584, 0.021006092, 0.089760184, 0.051655754, -0.025095679, 0.017972987, 0.004884003, 0.06757514, 0.074933976, -0.03098221, 0.04802393, 0.09361553, -0.06433186, -0.0044979914, -0.0044086706, 0.109693006, -0.112889804, 0.041902833, 0.031774793, 0.057648424, -0.0015410798, 0.057752363, -0.034886636, 0.016793612, 0.085741326, -0.019375322, 0.024687592, -0.0279623, 0.05677805, 0.008400566, 0.028377907, 0.008936888, 0.0062903417, -0.052559316, -0.010766674, 0.06931427, -0.028589267, 0.082267486, 0.11428923, 0.047580175, -0.011497563, -0.028835429, 0.07281495, -0.04680824, -0.033933688, -0.036411334, -0.010545987, -0.014107668, -0.0050741956, 0.010143487, 0.032605138, -0.05313003, -0.034505755, -0.048697785, 0.050538648, -0.03371668, -0.021636216, 0.006760371, 0.068658374, -0.079462, -0.037203245, 0.07626454, 0.01796628, 0.01784508, 0.017769668, -0.021318795, -0.09202141, -0.008927685, 0.05931885, 0.08680648, -0.06677875, 0.0072763953, -0.0731491, -0.007810972, -0.03200062, -0.010485246, 0.046040434, -0.034030054, -0.028938534, -0.0023542328, -0.057952613, 0.101116896, -0.028160913, -0.08849822, 0.0005527313, 0.1368902, -0.09780493, -0.04680615, 0.00076712534, -0.012492446, -0.0140410215, 0.08345586, -0.00657292, -0.022158619, -0.030584814, -0.070165746, 0.10256318, -0.036169104, -0.05215854, -0.032208208, -0.024322381, 0.037281316, 0.08439495, -0.011331628, -0.07574866, 0.0024992677, 0.033196732, 0.012086587, 0.018624738, -0.006327251, 0.06113059, 0.011406221, 0.037068825, 0.059208773, 0.038174126, -0.047353897, 0.031413928, 0.011863657, -0.031558264, 0.046155278, -0.018749943, 0.05713538, -0.035996944, -0.0009305992, -0.010524117, -0.032535262, -0.0013887427, -0.0028290171, -0.03905174, 0.010376951, 0.011456825, -0.07524657, -0.029027361, -0.071920395, -0.0049846494, 0.043845776, 0.0025125442, 0.0070259934, 0.05095529, 0.086760886, 0.04281686, -0.06453373, -0.030849371, -0.031460192, 0.008569573, 0.07088742, -0.012822424, -0.05630776, 0.021779228, -0.046120666, 0.076305225, -0.071468696, -0.055071387, 0.06933444, 0.04105632, -0.034743175, -0.004758986, -0.025505846, 0.01577515, 0.041470606, 0.023653042, 0.013749612, 0.06461667, 0.03876097, -0.039384257, -0.09146143, -0.043480955, -0.0097408015, 0.043017514, 0.00010163882, 0.018273935, -0.010653128, 0.040588498, -0.02291133, 0.016208671, -0.000090141046, 0.026245043, -0.008417565, -0.06717346, -0.027953463, -0.049035445, -0.044991978, -0.06398368, -0.040256802, -0.07224328, 0.000048183494, -0.025308855, -0.02935209, -0.01108117, 0.058955867, 0.01234267, 0.021576118, 0.05630341, 0.0848581, 0.017747251, 0.0007565953, 0.03140621, -0.048973333, 0.12761253, 0.07283497, 0.10975908, -0.01752341, 0.0043081795, 0.038509134, 0.113696754, 0.0005089173} + testVectorVeryDifferentMiniLML12v2 := []float32{ // Los gatos son animales independientes que disfrutan dormir + 0.038773198, -0.036970805, -0.040083647, 0.12536122, -0.022882015, 0.021315014, 0.056678787, 0.025049616, 0.024813903, 0.0120835835, -0.009899034, -0.048745543, -0.041406658, 0.087393805, -0.021416144, -0.052861497, 0.022678588, -0.102469474, 0.013729935, 0.03281509, 0.031755466, -0.03882857, -0.010749077, -0.021000965, -0.075888835, -0.093058735, -0.10089965, 0.026339324, -0.038298577, -0.034268197, -0.011916607, -0.05282156, -0.0097302925, -0.03714732, -0.042928465, -0.012586547, 0.08159514, -0.04487888, -0.10419695, -0.03229646, -0.0036334635, 0.004389561, 0.0055646715, -0.042237964, 0.01841084, -0.018299868, -0.0075984234, -0.055042952, 0.038215913, 0.056524035, -0.03925228, -0.022899278, -0.06978747, 0.0041959854, 0.017737325, 0.008212123, 0.024602672, 0.026904304, -0.0031825406, 0.055223215, 0.010032535, 0.021778084, -0.062488597, 0.051115684, 0.08609055, -0.029345125, -0.05604581, -0.024819657, 0.030328661, -0.083305, 0.015078142, 0.048427783, 0.065866776, -0.043694526, 0.061878808, 0.05049823, 0.11316554, -0.009657738, 0.01038636, -0.008226882, -0.074874066, 0.051842116, 0.0010682624, -0.018500084, -0.007793899, 0.037624277, 0.013798036, -0.0008041487, -0.121672936, 0.002208962, -0.014646392, 0.06758675, 0.067200616, 0.00030347783, 0.07803238, 0.04055297, 0.029012896, -0.096418925, -0.05405886, 0.00078464684, 0.102228515, 0.059593525, 0.0010504355, 0.053548552, -0.004270924, -0.043248408, 0.00007081115, -0.036929227, 0.0050680013, -0.047567394, 0.042051334, 0.036021914, 0.06584176, 0.035997465, -0.017107543, -0.007167671, -0.009233795, 0.068519495, 0.13686547, 0.03798997, -0.024766635, 0.007569431, 0.08181811, -0.057480723, 0.059518553, -0.09046198, 0.040360674, 0.04758537, -0.015695887, 0.008975096, 0.018818675, 0.03348437, -0.0029156366, -0.0826115, -0.03414977, 0.075200155, -0.018440584, -0.008461531, -0.024238706, 0.05820139, 0.09112688, -0.011143741, 0.0924394, 0.039072007, -0.0019073823, -0.0015219722, 0.020245807, -0.03852994, -0.0018924066, 0.13905354, 0.059265796, -0.07032625, 0.088940024, -0.044101488, -0.037938565, -0.038970795, -0.096906945, 0.06065275, -0.052466627, -0.0549799, -0.07480819, 0.08147781, -0.059444565, -0.053134196, -0.030579034, 0.0030518244, -0.051207107, 0.012473973, 0.021719923, -0.0025529366, 0.060841836, 0.01127315, 0.024454847, 0.029041816, -0.055770066, -0.05234835, -0.15113078, -0.08126239, -0.014738014, 0.0008157413, -0.004089376, -0.03101916, -0.0681169, 0.017773433, -0.09014856, 0.013449419, 0.0056449883, 0.039252147, 0.10341589, 0.0040195226, 0.09164564, 0.0038045452, 0.025956742, 0.030964458, -0.0052038906, -0.05787555, 0.036561474, -0.041673362, 0.018559413, -0.024530534, 0.028568763, -0.105419025, 0.065974005, -0.030239968, 0.0007669755, -0.04023858, 0.00836797, 0.03223592, 0.10354197, -0.01940403, -0.022762368, -0.016102314, -0.01095632, -0.05618028, 0.04429432, -0.0015736, -0.047137555, 0.048684523, -0.020686593, 0.0068162037, 0.09283696, -0.05570012, -0.009637794, -0.072636396, 0.0039794818, -0.06543572, -0.0045029335, -0.0056883167, -0.11353394, 0.012839105, -0.04187773, -0.072709315, -0.060162254, -0.004728383, -0.01575908, 0.014364104, 0.07705717, 0.023280906, 0.014816097, -0.008449041, 0.021050738, -0.030262483, 0.0009896689, -0.03061819, -0.03897136, 0.021356732, 0.0057799136, 0.019732544, 0.04772141, 0.09109385, -0.02758158, 0.08450835, 0.0019627432, -0.0518876, -0.036501236, -0.0346496, -0.010474969, -0.016603539, 0.09255842, 0.014738831, -0.07837869, 0.059753705, -0.008167218, -0.0068896995, 0.03158536, 0.012978418, -0.005657255, 0.03726972, -0.009025997, -0.00067213573, -0.013078166, 0.025796369, -0.09590421, 0.080058955, 0.08846313, -0.08338213, -0.051093694, -0.0020557758, -0.041386902, 0.018341815, 0.007146327, 0.04678766, -0.049797256, 0.010238786, -0.010890589, -0.025385847, 0.032748237, 0.01260644, 0.047849182, -0.06069717, -0.03124979, -0.014880721, 0.022713883, -0.08094806, 0.13349012, -0.028162898, -0.041476548, 0.01300787, -0.007883088, 0.07724416, 0.09798705, -0.034681067, -0.093124226, 0.00940398, 0.01345664, -0.055553894, 0.10547937, 0.027961044, 0.010783356, -0.10902568, 0.063008755, 0.081457786, 0.052674264, 0.03940576, 0.088089146, 0.037512045, 0.0050444775, 0.033438344, 0.038408346, -0.01707878, -0.0683014, -0.050477028, -0.06901548, 0.025798075, -0.043122537, -0.09262322, 0.006595632, -0.10320502, -0.06196482, 0.0461267, 0.059487905, -0.05268651, 0.12966771, -0.004013772, 0.063538395, 0.076720044, -0.009429152, -0.014522561, -0.029081186, 0.020279573, -0.005414016, -0.013558259, -0.07305859, -0.02520155, -0.016227841, -0.0050740833, -0.05179067, -0.05909998, 0.021111416, 0.0041141994, 0.0077062077, 0.008531419, -0.07948051, 0.0044141305, 0.040164765, 0.017026428, 0.073208705, 0.0183194, 0.034370814, -0.07752393, 0.06577042, -0.03973599, 0.018428847, -0.0027913842, -0.07654784, -0.029300457, 0.020145413, -0.020056032, 0.014325992, 0.0224245, -0.08291738, 0.058386978, 0.006810452, -0.07158955, -0.08822739, -0.001527183, 0.06993493, 0.055112574, 0.03373282, 0.008156818, -0.004246021, 0.03737389, 0.08129671, -0.032555513} + /* + inputVectorE5_base := []float32{ // passage: I like soccer + // -0.011958026, 0.030307064, -0.007887139, 0.01722607, 0.038534943, -0.022430697, -0.026132884, -0.033142433, 0.021482924, 0.009504165, -0.0060249427, 0.021261819, 0.12276235, 0.025307411, -0.025519311, 0.0061364886, 0.039991066, 0.0036183652, 0.04592214, -0.0040358077, 0.048802823, -0.0227532, 0.036444094, -0.012249692, 0.031186212, -0.046524655, 0.038336482, 0.030804439, -0.034197345, 0.010494412, 0.03782962, -0.011478989, -0.017874207, 0.03167291, 0.0289014, 0.039300643, 0.017847026, -0.035510726, 0.03406083, -0.0061665205, 0.012501271, 0.021648094, 0.037638564, -0.0346889, 0.020774813, -0.009928407, 0.039323762, 0.04643056, -0.085040696, -0.03425487, 0.010951664, -0.013535336, 0.012447383, -0.016171254, -0.0139716985, -0.025343323, 0.03490798, 0.03002813, -0.040967003, 0.062379465, 0.023238318, 0.054974902, -0.002189211, 0.06130506, 0.023304243, -0.049567766, 0.0029698894, -0.059021756, -0.034891035, -0.022081135, 0.0040910537, -0.020119345, 0.019406002, 0.0025520916, -0.053518824, -0.014377506, -0.06245792, 0.023377912, -0.00034166925, -0.028806232, 0.030304352, -0.000058279442, 0.015660949, 0.017937783, -0.00679218, -0.040572707, -0.04936459, 0.024466086, 0.018407157, 0.05099504, 0.006352729, -0.028010411, -0.017278505, 0.023403965, 0.016537556, 0.004795049, 0.009511753, 0.030436907, 0.042840727, -0.05547062, -0.012230076, -0.060316946, -0.029791867, -0.023875352, -0.05980641, 0.0012566427, -0.038075835, -0.03393869, 0.04794415, -0.035379488, -0.014183572, 0.02161441, 0.04541673, -0.050488982, -0.014411983, 0.014105348, -0.0064411345, -0.030230425, 0.009055341, -0.068990506, 0.00067824766, 0.05086135, -0.016276393, 0.028881475, 0.047895856, -0.008750243, 0.033570364, -0.011002792, 0.014680399, -0.01845509, -0.010655362, -0.03344013, -0.0074369395, 0.021630531, -0.03286602, 0.040637985, 0.016266877, 0.043290883, 0.019578101, 0.022898754, 0.021177689, -0.06682793, 0.003861729, 0.04821865, 0.029733738, -0.07064259, 0.012339871, 0.004719177, -0.024328275, 0.041960366, 0.040101033, -0.018612338, -0.032063887, -0.045253437, -0.010295391, 0.011473953, -0.028311405, -0.018768951, -0.055094857, 0.05314353, 0.009966136, 0.03916462, 0.015762309, -0.028708009, 0.053543996, -0.048986144, -0.049527504, -0.020265007, -0.009117578, -0.0076973145, -0.016750384, -0.029601097, -0.039145295, 0.03533294, 0.0255441, -0.023199353, -0.016359307, -0.02113683, -0.023283841, -0.08325627, -0.07313017, -0.050771523, -0.016241698, 0.042224262, -0.02552561, -0.014620982, -0.01625523, -0.053811338, 0.014689236, -0.0055549773, -0.04801497, 0.03473068, 0.04986305, 0.019639945, 0.008318083, 0.028989632, 0.040326945, 0.0079059275, -0.026271982, -0.061957255, 0.044592403, -0.01114613, 0.08179064, 0.04956784, -0.029515289, -0.09644403, 0.056631126, 0.114686206, 0.020510357, 0.016488833, -0.028519372, 0.024155365, -0.03473419, 0.029131202, -0.021258732, -0.044616938, 0.029033478, 0.017687649, -0.057969425, 0.0131431995, 0.03983494, -0.05939801, 0.04309643, 0.014374554, -0.032193035, 0.0028290187, 0.036912974, -0.019642817, 0.041691016, 0.015768858, 0.04685592, 0.032134276, -0.024442708, 0.00984151, -0.052853532, -0.03846759, 0.051593076, -0.010422243, -0.012666271, -0.11438187, -0.024178289, -0.003587725, 0.036188032, -0.0038264121, 0.0066097802, -0.027692905, -0.016199544, 0.03846315, -0.014150815, 0.0076250546, 0.026392212, 0.011652573, 0.039833326, -0.028475057, 0.017774055, 0.0020180636, -0.022364728, 0.009762616, 0.001206238, 0.050957266, 0.039193586, -0.029124247, -0.012584547, -0.034364156, 0.04037913, 0.024618959, 0.11325102, 0.0018559712, -0.015164406, -0.023929384, -0.050403513, -0.02985087, 0.007471126, 0.039197065, 0.018191937, -0.056301672, -0.032336053, 0.029786102, -0.035923317, -0.011465854, -0.008267741, 0.062592685, -0.05859189, -0.038909152, -0.02184389, 0.017960737, -0.06702139, 0.017330393, -0.02045269, 0.034080446, 0.045633815, 0.04775217, -0.016344815, 0.03229773, 0.039331622, 0.01951498, -0.0062657315, 0.014706745, 0.019732462, 0.0037528155, -0.012875424, -0.027694292, 0.050494347, -0.059757438, -0.058635592, 0.0016346759, 0.0871034, 0.032273144, -0.04251994, 0.02832441, -0.029124625, -0.03728655, -0.031078411, 0.0343639, -0.0014505114, -0.061482146, 0.0045560463, 0.051552642, -0.0037096057, 0.019257037, -0.050360393, 0.02928614, 0.009843059, -0.036926456, 0.07196923, -0.036053143, 0.03366069, 0.031238558, 0.02560205, 0.026632385, -0.022133803, 0.042786747, -0.033731878, 0.052207142, -0.020707138, 0.031529877, -0.044450212, -0.013456233, 0.08568665, -0.00081644667, -0.02003199, 0.021588285, 0.0026055488, -0.029747095, 0.06013567, -0.022931121, -0.012431405, 0.012857053, -0.01718952, 0.002096346, 0.03593464, -0.07324744, 0.047731087, -0.015740419, -0.011462194, -0.015159013, 0.020702807, -0.0215751, -0.05011585, -0.011426334, -0.027642546, 0.017088719, 0.014727402, -0.056648064, 0.0062071476, 0.025943218, -0.01868443, -0.013408927, 0.0065880106, -0.03564702, 0.08870428, 0.024992038, -0.0038430295, -0.009734496, 0.04343017, -0.02732613, 0.020087045, -0.012099698, -0.0044926433, 0.057946313, 0.08903274, 0.021570072, 0.0022063775, 0.037124068, -0.07167553, 0.042304944, 0.01070144, -0.051059254, 0.022598706, -0.02150138, 0.0009080106, -0.022425653, -0.019748975, 0.040372543, 0.024974043, -0.01785009, 0.030777445, 0.0055054496, -0.0022003911, -0.012879983, 0.0467983, 0.053360026, 0.0033298414, 0.044878736, 0.020130066, 0.026952952, -0.021446148, 0.054405946, -0.025465596, -0.029250707, 0.064970456, 0.011942952, 0.015155908, -0.026763845, 0.020983782, -0.013301199, -0.041656666, 0.028263526, 0.027830219, 0.0012042793, -0.0587478, 0.019012222, 0.038457863, 0.036172606, 0.020426106, -0.059371877, -0.02605348, -0.047123305, -0.016243298, -0.053901114, -0.035894796, -0.008763406, -0.012896026, -0.018411089, 0.0069237235, -0.05239428, 0.0058221878, 0.05576516, -0.027112924, 0.023379736, 0.02523077, 0.007840566, 0.01673873, -0.0053589023, -0.044434663, 0.009979374, -0.05675191, -0.0025956524, -0.050903123, 0.010826342, 0.02825829, 0.012864764, 0.01809844, -0.020823903, 0.040511884, -0.04556962, 0.05043266, -0.037257634, 0.013505942, -0.10754183, 0.007943708, 0.06293569, -0.022786718, -0.026691074, -0.053530954, -0.043461498, -0.0032980759, -0.017570991, 0.02095033, -0.022333795, -0.056434132, -0.03205476, 0.027811386, -0.0040934333, 0.024853777, 0.026750294, -0.0041964324, 0.053251438, 0.01368262, -0.031356234, -0.023997698, -0.011443603, -0.035529144, 0.025756065, 0.05426319, 0.055783194, -0.06751367, 0.052771978, -0.001163354, -0.025435619, -0.0094826305, -0.004368645, 0.014050108, -0.018753959, 0.007929673, 0.039231367, -0.015217665, -0.054106224, -0.00197224, 0.028045125, -0.012068425, -0.029741693, -0.053945053, -0.009594979, -0.056149833, 0.013052865, -0.03205511, -0.030209674, 0.009464428, -0.045879893, 0.097339064, -0.049096934, 0.0013631302, -0.056097332, 0.008439613, -0.0024833214, -0.000080165606, 0.032083366, 0.004394703, 0.014256693, 0.009234113, 0.006218169, -0.00065661897, 0.0019616361, 0.05598386, -0.014698286, 0.016096152, -0.047371324, 0.016248556, -0.019956674, 0.035569888, -0.04756759, -0.024157768, -0.07951636, -0.06725852, 0.023488104, -0.008263438, -0.002235841, 0.04820385, 0.011171201, 0.013623897, -0.026841022, 0.039405063, -0.026351277, 0.022011783, -0.02672378, 0.010632148, -0.010184964, 0.033703543, -0.013914806, -0.0046339296, -0.0106464, -0.021348465, 0.054253392, -0.051397342, 0.051802486, 0.015218245, 0.01848678, 0.033428963, 0.010762269, 0.007567297, -0.04296342, -0.035615895, 0.038454268, -0.01442382, 0.033056036, 0.014602311, -0.00014383248, -0.012507564, 0.06465915, -0.056539357, 0.013666485, 0.022790365, -0.037845343, 0.019764101, 0.039207146, -0.002680678, -0.039380405, 0.035813987, -0.039841857, -0.04179874, 0.017561007, -0.030872619, 0.014273279, 0.0034460898, 0.018781945, -0.005178211, -0.036320653, -0.23085965, 0.021572905, 0.010433873, -0.010695635, 0.013612118, 0.04890353, 0.00030080267, 0.0025248863, 0.058565266, 0.010541146, 0.026825141, -0.04439922, 0.055104773, -0.0024773215, -0.065505855, 0.016405374, -0.03593307, -0.08552887, 0.038311735, 0.012431675, 0.016173188, 0.039414596, -0.052900486, -0.014783654, -0.024906965, -0.047445536, -0.051366676, 0.017399853, 0.010473932, 0.053782824, 0.01833324, 0.0008487331, 0.04801879, 0.02003207, -0.033826556, -0.010806506, -0.025118528, 0.018070918, -0.062576234, 0.05563618, 0.02813292, -0.08111438, -0.065599725, 0.01603358, -0.010923405, 0.06967761, 0.02589161, 0.013075893, -0.017140126, -0.029358357, 0.03535794, 0.0033946577, -0.07314114, 0.040783703, -0.026152983, 0.04191336, 0.007840777, 0.02851859, 0.011058812, -0.008062927, 0.033796143, 0.029090602, -0.03966101, 0.04532138, -0.029240023, -0.015626792, 0.0092925, -0.027162936, -0.01894042, 0.05531998, 0.0047139567, -0.02460106, -0.035175372, 0.03859947, 0.027832452, 0.005627423, 0.014982945, 0.005642442, -0.055896822, 0.045672826, -0.021191733, -0.057305407, -0.017706782, 0.013513723, -0.03189893, -0.018529939, -0.007428619, -0.007421373, 0.037145972, -0.00014752017, -0.042229403, -0.04151492, 0.0010500661, 0.0041933814, 0.023691593, -0.033762515, -0.0015845222, 0.015373465, 0.0029374156, -0.045784667, -0.029205076, -0.015522001, 0.02115609, 0.029595627, 0.03635629, -0.04078311, -0.04145895, 0.012769062, -0.021677615, -0.015670706, 0.031175781, 0.06753832, 0.021757241, -0.014063302, 0.02671573, -0.0056406097, 0.027441602, -0.0007340817, -0.0077687562, -0.054123353, -0.0068473085, -0.015049716, 0.03151872, -0.008527758, 0.0337959, -0.008781811, -0.009766847, 0.043002788, 0.035290938, 0.030503968, -0.04023449, -0.045888938, 0.072369106, -0.035482645, -0.020194748, 0.050942432, 0.04386771, -0.0027256217, 0.056511577, 0.051694393, 0.0743731, -0.05052871, -0.0063161966, 0.039637823, 0.0004969995, 0.030269105, 0.05144867, -0.027645307, 0.0174659, 0.014801657, -0.029951716, -0.07194129, -0.017560085, -0.024117788, -0.02717623, 0.014556567, 0.012747637, 0.010832007, -0.019112395, 0.033589344, -0.050964493, -0.05356839, 0.01859798, 0.014954559, 0.033817884, 0.0072976504, -0.0158079, -0.011930099, -0.01736037, 0.04816999, -0.02025616, -0.0005011032, 0.032013357, -0.008672973, 0.017644571, -0.0048744082, 0.06767118, -0.021132298, -0.041882575, 0.023528261} + -0.0143156275, 0.0065394347, -0.025388611, -0.0012211486, 0.03181308, -0.017359179, -0.02078927, -0.040980175, 0.025587637, 0.027648767, -0.013788454, 0.013968936, 0.16598135, 0.009979124, -0.015647331, 0.01389914, 0.038550276, 0.0049126735, 0.027116578, -0.012888863, 0.042955946, -0.031956777, 0.032853056, -0.014283477, 0.04974678, -0.021385627, 0.03588862, 0.0416443, -0.035934042, 0.0025792157, 0.02012549, -0.003026372, -0.03698421, 0.0131695615, 0.023318484, 0.027204448, 0.02995661, -0.02622485, 0.045866348, -0.009095554, 0.028172191, 0.024291886, 0.030256806, -0.039284185, 0.008101238, -0.017805131, 0.0568264, 0.057628375, -0.07095006, -0.020339914, -0.0055588204, -0.031356405, 0.0049415906, 0.0059679905, -0.011373868, -0.04177303, 0.024849555, 0.035270046, -0.04783184, 0.067726314, 0.0015085762, 0.031438455, -0.013016488, 0.06146949, 0.010213119, -0.048196997, 0.008085803, -0.053961933, -0.043203034, -0.03281923, 0.00028425656, -0.012706264, 0.016071549, 0.004231804, -0.04904037, -0.017404279, -0.060826886, 0.009750942, 0.025450073, -0.016495978, 0.010975522, -0.012345219, 0.017654125, -0.0049276794, -0.009618917, -0.027415734, -0.05695193, 0.037091166, 0.015261405, 0.04235929, -0.0026607078, -0.03605759, -0.030132527, 0.010535205, 0.0008890961, 0.012930888, 0.026127398, 0.028124839, 0.034141295, -0.08347563, 0.006367476, -0.038929034, -0.012725784, -0.026305476, -0.089854315, 0.005709901, -0.037477702, -0.037441555, 0.061280187, -0.046387173, -0.019377483, 0.024601197, 0.015839094, -0.057232544, -0.022520082, 0.00043290356, -0.010097204, -0.018520005, -0.0030561076, -0.06456531, 0.008680698, 0.049794666, -0.021326896, 0.028421024, 0.03543521, -0.013043102, 0.027963748, 0.0013317054, -0.024576003, -0.031564496, -0.01461037, -0.014279593, -0.003830743, 0.019010108, -0.0479, 0.036511812, 0.01850072, 0.028372293, -0.003131044, 0.0029237105, 0.01245746, -0.09381409, 0.0010135958, 0.027189227, 0.032203533, -0.06844485, 0.022666842, -0.0008960947, -0.017871449, 0.026552476, 0.05022251, -0.031788893, -0.028657028, -0.026505077, -0.009350134, 0.02717328, -0.026921667, -0.011153334, -0.026025102, 0.054239083, 0.011318865, 0.039059773, 0.010021746, -0.01830278, 0.051935524, -0.023705626, -0.039543632, -0.02393099, 0.006210872, 0.012620061, -0.019877333, -0.038950227, -0.028754108, 0.03797042, 0.020339463, 0.0026506188, -0.027493423, -0.03710056, -0.031755332, -0.06766831, -0.08607376, -0.04157356, -0.03141719, 0.037383404, -0.026182171, -0.033752084, -0.020505909, -0.03504061, 0.0152358515, -0.004827336, -0.034077823, 0.039227873, 0.038116172, 0.008785029, 0.024883965, 0.024906678, 0.042350814, 0.019003674, -0.017460922, -0.073146984, 0.05480988, -0.0097080935, 0.0723225, 0.042486392, -0.0074558277, -0.10362494, 0.049734354, 0.10209468, 0.026596138, 0.009682365, -0.04673197, -0.0032097755, -0.03270287, 0.021617962, -0.025454892, -0.039851323, 0.016834296, 0.022454338, -0.073197596, 0.013354853, 0.0375542, -0.06954724, 0.045224316, 0.025414322, -0.017419161, 0.016421154, 0.04282987, -0.032194182, 0.040827114, 0.009190477, 0.017561438, 0.05137698, -0.030766547, 0.014328079, -0.044294305, -0.03184668, 0.030274414, -0.0061751152, -0.014376063, -0.12073402, -0.0031241956, 0.0045510507, 0.03739233, -0.007887159, 0.0121797025, -0.056359306, 0.0011343829, 0.03509171, -0.022393154, 0.025477154, 0.018446779, 0.006506074, 0.04158995, -0.021605374, 0.016600553, 0.028185654, -0.038095154, 0.008689428, 0.011083605, 0.031627715, 0.023189068, -0.021560594, 0.021004254, -0.02861289, 0.05925523, 0.022142828, 0.0893639, -0.008433772, -0.008799363, -0.010476722, -0.03264332, -0.031727377, 0.02287101, 0.04320281, 0.0412084, -0.045917008, -0.03974865, 0.054716, -0.045786694, -0.013330635, -0.003010265, 0.045958474, -0.045544814, -0.061454155, -0.020001914, 0.033647608, -0.014042814, 0.020401664, -0.038520187, 0.029221172, 0.030208245, 0.03333216, -0.0036639902, 0.02074849, 0.03915396, -0.005692372, -0.00021794242, 0.017486684, 0.019405367, 0.0074021267, -0.016563127, -0.03124234, 0.038686924, -0.05647636, -0.07544949, 0.008121357, 0.089121304, 0.025394883, -0.05746141, 0.014104695, -0.021084182, -0.03873256, -0.025772233, 0.019959267, 0.02498548, -0.05913122, -0.010724875, 0.05396549, 0.013129879, -0.0038293377, -0.04428832, 0.025250476, 0.0047125053, -0.04350974, 0.079334654, -0.02507087, 0.047020346, 0.025016533, 0.0388396, 0.012032155, 0.0054177856, 0.047768585, -0.03070753, 0.05328766, -0.04127075, 0.040236875, -0.033983395, 0.015677761, 0.07703151, -0.0053362786, -0.060665276, 0.018704152, 0.017226873, -0.039059978, 0.074215926, -0.02697609, -0.0050617303, -0.019568427, -0.017641826, 0.013713223, 0.041957274, -0.0738539, 0.044904906, 0.0032601086, -0.017376734, -0.015252018, 0.043906044, -0.01211134, -0.049551282, -0.01145524, -0.022226403, 0.023613155, 0.019789, -0.051770497, 0.015873933, 0.022969056, -0.0101753855, -0.009545111, 0.014601525, -0.024766866, 0.084153436, 0.026289355, -0.0324008, -0.0039866595, 0.057349313, -0.020213667, 0.055152446, -0.015137755, 0.00073646643, 0.041838128, 0.07675961, 0.036182333, 0.0019608126, 0.028210111, -0.064777315, 0.0304087, 0.009027945, -0.05648751, 0.05234015, -0.027673027, 0.024226952, -0.039328277, -0.015182952, 0.053729765, 0.029417176, -0.022912798, 0.0139812855, -0.0030220877, 0.010149, 0.0091792885, 0.048068196, 0.03873581, -0.0016975933, 0.037703194, 0.023357077, 0.04206181, -0.011787242, 0.07000972, -0.019430356, -0.021596974, 0.07868515, 0.016417563, 0.015226183, -0.0051322957, 0.020020852, 0.00042281445, -0.03930115, 0.013778754, 0.030394414, -0.010889359, -0.07471781, 0.022536833, 0.029292343, 0.0147314435, 0.039585948, -0.05444345, -0.021838421, -0.033334848, -0.019014802, -0.036701318, -0.05488091, -0.01752014, 0.004399855, -0.012815111, 0.007928676, -0.05745905, 0.011860698, 0.07125338, -0.007385023, 0.019941036, 0.035907887, -0.027795926, 0.012429591, -0.034484815, -0.03995127, 0.0030248966, -0.04074755, -0.042472184, -0.04906908, -0.00006781172, 0.020182539, -0.012671679, 0.018227639, 0.0040071555, 0.03994012, -0.032109864, 0.07397447, -0.036778517, -0.010844071, -0.08190833, 0.014581257, 0.06344988, -0.0071892487, -0.03212407, -0.026634842, -0.054824293, -0.0015353209, -0.007825078, 0.047159884, -0.021719404, -0.05019747, -0.03552404, 0.019502155, 0.011001953, 0.013463218, 0.0139252, -0.0064885076, 0.054999195, 0.015643079, -0.010025921, -0.03437521, -0.0081634335, -0.024085352, 0.017753303, 0.03793053, 0.035451435, -0.07615163, 0.0551546, 0.021295233, -0.02429551, -0.012506303, -0.007136872, 0.032412156, -0.031329576, 0.0022026908, 0.03959281, -0.017088514, -0.053298086, 0.00038762006, 0.034088083, -0.017323814, -0.042610407, -0.04327862, -0.008408326, -0.079850465, 0.032122597, -0.028287137, -0.014272152, 0.004185647, -0.027969623, 0.089913085, -0.057193834, 0.029519005, -0.042634945, -0.0037906347, -0.024482146, -0.0030035132, 0.047534194, 0.025332704, 0.00077951705, 0.0068772277, 0.024449278, 0.00035606296, 0.004076698, 0.04098024, -0.034340892, 0.0010540178, -0.034639683, 0.02883131, -0.028854866, 0.03966202, -0.051365197, -0.0007278118, -0.06965556, -0.04663717, 0.008092501, -0.0021099725, -0.011901275, 0.053080004, -0.003399067, 0.010908232, -0.031692576, 0.044614397, -0.024101485, 0.024072217, -0.013126169, -0.0018893053, -0.010828244, 0.024901608, -0.028367976, -0.0053928923, -0.0025824143, -0.011641346, 0.043385364, -0.04014641, 0.04868164, -0.004000026, 0.018755931, 0.035018254, 0.0059041837, 0.008994129, -0.031459194, -0.02274239, 0.036613908, -0.016827391, 0.015550911, -0.00025892133, -0.012373543, -0.020208294, 0.06623975, -0.04927514, 0.026912093, 0.019870967, -0.01779586, 0.04016542, 0.03810415, -0.009875994, -0.035668094, 0.03708096, -0.024679963, -0.043241087, -0.0013369685, -0.042150524, 0.0163814, -0.0061832964, 0.026095323, -0.0014001649, -0.022797497, -0.21089749, 0.019719876, 0.018455222, -0.010020203, 0.0044883927, 0.04808439, -0.026595607, -0.006164029, 0.040239923, 0.038157336, 0.029104011, -0.047988523, 0.06315468, -0.010183212, -0.04837111, -0.007865147, -0.011987574, -0.089631274, 0.041119248, 0.026009563, 0.002176904, 0.03379191, -0.0425288, -0.02849656, -0.02966118, -0.029129876, -0.03279072, 0.0057679764, -0.0044693826, 0.021303201, 0.00097106484, -0.013650896, 0.04434278, 0.025867885, -0.017756466, 0.0046212743, -0.03583145, 0.0067417724, -0.06862165, 0.08294223, 0.04228908, -0.061839394, -0.055606052, 0.019061508, -0.018139092, 0.05815597, -0.0028854592, 0.023767611, -0.013314796, -0.02765131, 0.020997183, 0.012732602, -0.06690671, 0.059274003, -0.017440867, 0.040435713, 0.017170139, 0.03320682, 0.0025004568, -0.0076839235, 0.045791924, 0.05072653, -0.043648038, 0.048741344, -0.04915739, -0.014182644, 0.021789912, -0.021787368, -0.030386165, 0.04727281, -0.0007539115, -0.016433183, -0.029856881, 0.01911444, 0.03804372, 0.018653333, 0.00077236374, 0.0082214195, -0.035124093, 0.04710185, -0.04346983, -0.07829862, -0.024061905, 0.016701732, -0.0360064, -0.0010016677, 0.0034449862, -0.02611881, 0.053520333, 0.0033698892, -0.024494652, -0.011311927, -0.003567324, 0.00073792913, 0.02573941, -0.010848351, 0.01396745, 0.012363075, 0.0009043423, -0.0556413, -0.02080378, 0.0189676, 0.027505431, 0.020498887, 0.037452303, -0.03932328, -0.076556005, -0.002532913, -0.022174077, -0.013030712, 0.03150133, 0.051448185, 0.019169958, -0.010406027, 0.032284748, -0.039707784, 0.031594746, 0.003489562, -0.048002426, -0.07031704, -0.017045217, -0.015587627, 0.0194493, -0.025602734, 0.006353371, -0.00766311, -0.010537843, 0.044719897, 0.018709905, 0.028079519, -0.05949373, -0.03697841, 0.0903292, -0.02829106, -0.020140776, 0.05954756, 0.040059425, -0.0061180536, 0.03755168, 0.030019425, 0.07791948, -0.03345321, -0.013533833, 0.024399104, -0.0021146634, 0.03954399, 0.05471891, -0.02268942, 0.015345771, 0.030209646, -0.032571092, -0.07277484, 0.016174177, -0.021372838, -0.021197252, 0.006256353, 0.025179246, 0.005631436, -0.023647498, 0.04335252, -0.031614833, -0.05423546, 0.027718965, 0.013332389, 0.028540803, 0.01783707, -0.0077716894, -0.02641114, -0.02955577, 0.040763635, -0.01762036, 0.0043096873, 0.04232738, 0.004150487, -0.0028467919, 0.00478532, 0.057293575, -0.019122522, -0.046193384, 0.025970653} + testVectorSimilarE5_base := []float32{ // query: I love sports + // -0.0030619893, 0.018846497, -0.019348852, 0.013449728, 0.043865077, -0.026698941, -0.023708135, -0.043304674, 0.03460693, 0.003518519, -0.0063237255, 0.014284168, 0.114999615, 0.02315353, -0.037004355, 0.0039856634, 0.0427376, 0.019971736, 0.040793233, -0.015498223, 0.06633076, -0.019149054, 0.04462353, -0.030128941, 0.03124896, -0.052103814, 0.04134896, 0.0207947, -0.030169627, 0.009617589, 0.04457201, -0.031035336, -0.026230037, 0.037205867, 0.027807675, 0.059005275, 0.019389216, -0.029499575, 0.040671926, -0.0060667433, 0.014724775, 0.017504197, 0.04110241, -0.04634276, 0.020660877, -0.00089741725, 0.04276391, 0.04832227, -0.08090728, -0.017126946, 0.016708646, -0.018377632, 0.008590144, -0.019577429, -0.032790188, -0.022852149, 0.037745085, 0.04592767, -0.033632137, 0.04156522, 0.017443374, 0.058445543, 0.01484478, 0.052088365, 0.038055662, -0.036941763, -0.0026431065, -0.045894604, -0.051328875, -0.027574018, -0.011623039, -0.016764803, 0.01765341, 0.005916862, -0.074325316, -0.019089947, -0.06731897, 0.021459239, -0.0022634072, -0.02410411, 0.04055451, 0.012201235, -0.0025827049, 0.014158706, -0.0145072155, -0.039632242, -0.044712037, 0.016506173, 0.02744178, 0.048724912, 0.025706122, -0.021375293, -0.009063378, 0.014982314, 0.009168109, 0.0012155873, 0.012000821, 0.019612236, 0.045494664, -0.03288082, -0.0034627505, -0.07386959, -0.022377089, -0.026729468, -0.058692127, -0.016375983, -0.025769055, -0.018109493, 0.054212403, -0.031862743, -0.021553818, 0.023134423, 0.061740182, -0.055197593, 0.0019266978, 0.022909299, 0.021553503, -0.04497155, 0.008370178, -0.04958218, -0.006918276, 0.03696674, -0.0166237, 0.02140027, 0.03258646, -0.00035201333, 0.017592795, -0.013455702, 0.04227269, -0.034689136, -0.024220606, -0.022216948, -0.002588387, 0.031466026, -0.029130217, 0.02641367, 0.019294547, 0.05021628, 0.008910032, 0.026842803, 0.030475352, -0.06298538, -0.0009676381, 0.04131772, 0.035789847, -0.046193585, 0.019165112, -0.0015315653, -0.009648247, 0.04295029, 0.049314976, -0.023836644, -0.009811698, -0.05506732, 0.0055604023, 0.02204842, -0.039337955, -0.00507496, -0.025969466, 0.045649067, 0.020167055, 0.026419727, 0.027454061, -0.03822238, 0.033561394, -0.042078283, -0.04388901, -0.010183174, -0.019660747, -0.011556442, -0.033892635, -0.025348729, -0.030097894, 0.044389457, 0.024993101, -0.024007041, -0.0006286102, -0.029251037, -0.0299114, -0.06624563, -0.07185939, -0.045532167, 0.0062183156, 0.04142129, -0.014240593, -0.01739343, 0.0023828042, -0.029352723, 0.021590045, -0.0012363109, -0.04250213, 0.013275279, 0.044610944, 0.034370042, 0.005115279, 0.028976878, 0.022919577, 0.02124366, -0.03233229, -0.045509633, 0.054186475, -0.005179471, 0.08006191, 0.045112293, -0.037467625, -0.06647447, 0.04652339, 0.10425537, 0.03819509, 0.017477876, -0.027230833, 0.044067655, -0.02789647, 0.034581363, -0.026311222, -0.044918705, 0.057209108, 0.03231272, -0.03948859, 0.026202178, 0.04508553, -0.06488362, 0.03975224, 0.0030100276, -0.047809195, -0.011839324, 0.03794665, -0.008035162, 0.025086552, 0.04287702, 0.060221363, 0.027325658, -0.03162005, 0.026727865, -0.040857498, -0.04697381, 0.046072524, 0.0012381792, -0.020548817, -0.100162685, -0.033930458, 0.023867132, 0.018011352, -0.009202412, 0.002986883, 0.0099459505, -0.020638477, 0.020407302, -0.01118804, -0.016121808, 0.022072522, 0.010762071, 0.04212661, -0.030929204, 0.015148933, -0.0023829178, -0.02945049, 0.012255783, 0.010815648, 0.039999567, 0.029181784, -0.035802033, -0.009673657, -0.023969037, 0.02129203, 0.018971724, 0.11531746, -0.012767477, -0.031890966, -0.03976976, -0.041559093, -0.027171716, -0.016252214, 0.026641473, 0.0070370915, -0.056828953, -0.039767474, 0.028986583, -0.038576093, -0.01489857, -0.0070311762, 0.07589258, -0.04809731, -0.05105979, -0.024624633, 0.026899714, -0.07854341, 0.021617584, -0.023642791, 0.018384539, 0.052928112, 0.053620443, 0.0011905662, 0.03264652, 0.029835586, 0.00046326278, 0.013334619, 0.026072191, 0.02914155, 0.018211298, -0.013272013, -0.011977657, 0.042079784, -0.07129907, -0.029610934, 0.0038646616, 0.08561227, 0.040608343, -0.055875428, 0.017681096, -0.034519486, -0.030220319, -0.04741432, 0.034277774, -0.015728658, -0.05497179, 0.008644499, 0.061751332, -0.0039309068, 0.03831514, -0.041711934, 0.022673888, -0.0076737176, -0.034901697, 0.060571272, -0.01936283, 0.031891864, 0.013273141, 0.02975129, 0.0150003815, -0.039031655, 0.052466173, -0.043856718, 0.053395536, -0.034478147, 0.0057925265, -0.05613825, -0.010462209, 0.090845995, 0.007906203, -0.028943963, 0.017230138, 0.0038260494, -0.039789036, 0.03364486, -0.028788602, -0.020769987, 0.015104484, -0.0015116611, 0.013871471, 0.03960399, -0.07397894, 0.03817224, -0.021603437, -0.030369487, 0.014149463, 0.02869222, -0.029180437, -0.029642025, -0.019888127, -0.024413453, 0.024650214, 0.014985529, -0.057959944, 0.035237692, 0.03483113, -0.01935332, -0.019850297, -0.010054143, -0.047196068, 0.07603568, 0.021893023, 0.00053217105, -0.012640712, 0.042135708, -0.049653202, 0.023285313, 0.0025403274, 0.0036130778, 0.06907215, 0.07045927, 0.024333872, 0.019731624, 0.03376653, -0.0677391, 0.04144332, -0.003615578, -0.05303046, 0.023093343, -0.004589583, 0.004141793, -0.032812472, -0.017768636, 0.041147828, 0.025714338, -0.0013795958, 0.034718122, -0.0018409103, 0.011016812, -0.0023030657, 0.064484365, 0.03573785, -0.024508623, 0.036429163, 0.029434342, 0.0057427436, -0.011764342, 0.043930437, -0.032919735, -0.03222523, 0.057234935, 0.033344936, 0.043096922, -0.03829704, 0.01981778, -0.016564434, -0.041611277, 0.042322017, 0.04496726, 0.018359335, -0.036420546, 0.030156096, 0.03583065, 0.03127073, 0.03132746, -0.06869759, -0.021127542, -0.036910117, -0.009475217, -0.044585567, -0.031047694, -0.009085905, 0.007960815, -0.024529144, 0.010340939, -0.03263505, 0.023844412, 0.058092322, -0.033627786, 0.023395594, 0.023019249, 0.012652666, 0.013061622, 0.0007509524, -0.03383836, 0.005097929, -0.06857013, 0.016478594, -0.049157493, 0.024621943, 0.02695811, -0.005043638, 0.023652397, -0.014124759, 0.03908455, -0.043953255, 0.0659744, -0.05404904, 0.020576606, -0.10733408, 0.0006560424, 0.054609258, -0.029608227, -0.02709839, -0.06647201, -0.062774554, -0.008701165, -0.031087898, 0.0018629744, -0.042728573, -0.056837134, -0.03150488, 0.02021558, 0.005099145, 0.02022651, 0.037762895, -0.010466285, 0.06882802, 0.030391715, -0.024220003, -0.0130644515, -0.035046086, -0.054943226, 0.013139634, 0.04187457, 0.055248246, -0.06472082, 0.040816862, -0.0032749546, -0.036127858, 0.0077827843, 0.0012423133, 0.01978085, -0.013831931, 0.01820682, 0.043234743, -0.014975462, -0.06386483, 0.010391008, 0.037108693, -0.0007305783, -0.021253262, -0.06039119, -0.0094892215, -0.03319121, 0.011703476, -0.037032116, -0.040090308, 0.024711186, -0.030373972, 0.1117169, -0.049945947, -0.023065712, -0.052143946, 0.009514318, -0.001136675, -0.0116875945, 0.008147342, 0.017093526, 0.014870582, -0.021808684, 0.012917244, -0.00729657, 0.0050954903, 0.05137977, -0.039022468, 0.016997105, -0.03341763, 0.020393556, -0.018503098, 0.024465313, -0.05244907, -0.027874146, -0.09372946, -0.07206463, 0.034454115, -0.005466677, 0.009268065, 0.040224817, 0.011591715, -0.006032282, -0.013835682, 0.041645862, -0.02416431, 0.030142361, -0.008894229, -0.00014007313, -0.02469653, 0.028753936, -0.0026932345, -0.0023791385, -0.010302609, -0.024909342, 0.04969838, -0.07592375, 0.03526181, 0.0032372863, 0.032812472, 0.033751905, 0.028334936, 0.012679262, -0.055550247, -0.034131885, 0.03333461, -0.018796207, 0.039059743, 0.023275282, -0.010866633, -0.015420211, 0.06691386, -0.039273474, -0.0027812375, 0.01889765, -0.047118958, 0.028197436, 0.04291172, 0.0063134506, -0.032647938, 0.039741192, -0.03940806, -0.027325133, 0.03050793, -0.04305133, 0.015697118, 0.0017896608, 0.028319828, -0.017373485, -0.03383761, -0.22829725, 0.016051874, 0.024868792, -0.012598031, 0.0123299, 0.044745874, 0.0046600006, 0.00068816694, 0.04402293, 0.0011009127, 0.025912698, -0.057917073, 0.025038207, -0.026306055, -0.0763415, 0.017719086, -0.03754443, -0.05951767, 0.022408137, 0.008506453, 0.02061808, 0.021715183, -0.051287454, -0.00877075, -0.028382523, -0.040603224, -0.053649273, 0.0036148888, 0.014983062, 0.056722518, 0.036695212, -0.015354521, 0.054933514, 0.021532949, -0.03521357, -0.016936814, -0.0147914905, 0.0027304075, -0.063884676, 0.03914222, 0.015125187, -0.08544885, -0.057713456, 0.018049309, -0.024224406, 0.07500243, 0.029702423, 0.019800637, -0.018249981, -0.038521495, 0.021166766, 0.013291093, -0.080325484, 0.013528786, -0.008633761, 0.025675686, 0.020150442, 0.027789418, -0.016706899, 0.00089047896, 0.038959656, 0.004799374, -0.037886385, 0.033652652, -0.028634442, -0.0307466, 0.01630189, -0.007752447, -0.027496966, 0.02647513, -0.0013860769, -0.033716824, -0.038402874, 0.05538004, 0.015307284, 0.001177634, 0.028782696, -0.013946893, -0.0661733, 0.036104027, -0.00074012706, -0.049353726, -0.017535483, 0.0037718064, -0.04198872, -0.040862717, -0.023099774, -0.02292897, 0.02361115, 0.010211935, -0.058190435, -0.042235382, 0.009005903, 0.007047296, 0.026751624, -0.039904624, -0.011787891, 0.0011131876, -0.02584976, -0.048331957, 0.004408769, -0.012707105, 0.0072209192, 0.015542175, 0.04392512, -0.01965631, -0.03161781, 0.0337645, -0.0049804514, -0.021964999, 0.031222833, 0.026972461, 0.029435266, -0.014165512, 0.04083383, 0.00354552, 0.026489303, -0.013595391, -0.009329064, -0.038532827, -0.000029496134, -0.03139342, 0.02457241, -0.0141687775, 0.054210242, -0.008632142, -0.020174552, 0.029063832, 0.038790382, 0.02864069, -0.01864928, -0.04655779, 0.08188056, -0.03581171, -0.008486192, 0.058028005, 0.039370105, 0.0049215397, 0.041849677, 0.051670667, 0.061034802, -0.033672053, -0.015495464, 0.026406107, 0.0055066203, 0.019371532, 0.03386336, -0.032255776, 0.033035208, 0.011109019, -0.015563156, -0.05516419, -0.028947994, -0.024931537, -0.037637245, 0.023052769, 0.015082346, 0.019250061, -0.01536453, 0.025515592, -0.04029882, -0.07035706, 0.018702146, 0.0023446537, 0.022764336, 0.016498156, 0.0043817107, 0.014851864, 0.0008829631, 0.03326337, -0.020793336, 0.0057160687, 0.027067538, -0.013771819, 0.030484157, 0.00617158, 0.06111399, -0.024243431, -0.04598735, 0.000713331} + -0.0021426065, 0.028506117, -0.017526794, -0.018793339, 0.059588432, -0.040379312, -0.0087621715, -0.029676104, 0.029601708, 0.035935644, -0.0133496765, 0.037435543, 0.13672325, 0.009986487, -0.03455495, 0.024399484, 0.027568541, 0.0143291, 0.019043194, -0.025786411, 0.07097971, -0.025024695, 0.02657241, -0.025420174, 0.02810032, -0.030094193, 0.053658117, 0.024556167, -0.029871013, 0.01091316, 0.031012613, -0.009737866, -0.005303262, 0.024687652, 0.025364032, 0.051481247, 0.011305447, -0.01159111, 0.05308677, 0.0027891833, 0.014545518, 0.025560208, 0.037769016, -0.051184043, 0.027124278, -0.0030116, 0.038642716, 0.05477561, -0.07134693, 0.0019786188, 0.017135406, -0.035066124, 0.0048184237, 0.0021413658, -0.043812916, -0.026636206, 0.02328816, 0.04009615, -0.030664254, 0.051905222, -0.014098271, 0.06371669, -0.004480209, 0.05612505, 0.033107214, -0.02850969, 0.008535243, -0.037525833, -0.065985665, -0.024129365, -0.010917116, -0.045352828, 0.02354873, -0.0056800125, -0.07284971, 0.00008492294, -0.069804884, 0.008013635, 0.0031523868, -0.03908264, 0.04316324, -0.014564494, 0.016314082, 0.008754292, -0.00039351106, -0.0454985, -0.05220713, 0.01607414, 0.024503531, 0.04855709, 0.023938667, -0.032682445, -0.005402195, 0.03298104, 0.013364104, 0.047053754, 0.03209884, 0.031033555, 0.05395942, -0.052371, 0.014409072, -0.050584447, -0.023805125, -0.03601124, -0.06072041, -0.018453369, -0.01667723, -0.020227112, 0.03216963, -0.028539976, -0.018712752, 0.044660114, 0.036621857, -0.04064285, -0.0048561525, -0.006420382, 0.03380125, -0.030182399, 0.019929562, -0.044408835, 0.024021106, 0.02445754, -0.017838495, 0.019417934, 0.019396016, -0.03186347, 0.018171124, 0.010887363, 0.04039068, -0.061295155, -0.045200434, -0.012586684, 0.0062140217, 0.04369624, -0.04149741, 0.03935359, 0.028960858, 0.038114276, -0.0071072984, 0.034933105, 0.045132257, -0.07817419, 0.010174387, 0.02317567, 0.0512262, -0.046624947, 0.035742022, -0.022579996, 0.012864871, 0.030489584, 0.06334743, -0.014791812, 0.013979895, -0.059906088, 0.016558776, 0.010116813, -0.037157472, 0.01932599, -0.017336432, 0.04226779, 0.011569614, 0.02575379, 0.025083432, -0.038817067, 0.033164304, -0.03543514, -0.04520507, -0.0043760007, -0.007242521, -0.006359404, -0.02105409, -0.021162579, -0.038308527, 0.034851257, 0.0068237223, -0.035551116, 0.004907009, -0.046216387, -0.036553092, -0.052897006, -0.09898252, -0.031561766, -0.005128816, 0.036958855, -0.020646837, -0.013258649, 0.017998524, -0.029209103, 0.008452299, 0.021136517, -0.044264045, 0.04067568, 0.043753795, 0.024774922, -0.010178364, 0.037529472, 0.022856783, 0.036331348, -0.05118356, -0.042812254, 0.04496122, -0.0058539393, 0.063117616, 0.04299029, -0.03827401, -0.07373925, 0.037580118, 0.09906612, 0.044383984, 0.02566653, -0.036601942, 0.03283263, -0.022955226, 0.025806366, -0.027154917, -0.036050383, 0.038397703, 0.014491828, -0.038405243, 0.042586125, 0.046094373, -0.05066044, 0.04957105, 0.011426751, -0.028776404, -0.029452082, 0.033422444, 0.0071422053, 0.012601643, 0.020992136, 0.041943267, 0.018481575, -0.040660672, 0.010843929, -0.02437365, -0.04286992, 0.033060174, 0.0177113, -0.026232952, -0.111461565, -0.037368666, 0.028154682, 0.011562994, -0.016833039, -0.007579747, 0.0047503496, -0.017853685, 0.019568942, -0.028434701, -0.004761201, 0.03798927, 0.0027391028, 0.036198996, -0.0537168, 0.02014503, -0.0014796845, -0.020363772, 0.027342228, 0.0051699406, 0.051388837, 0.020923518, -0.037585575, -0.0042198068, -0.033714287, 0.019854862, 0.02149458, 0.10649466, -0.01869411, -0.033484776, -0.040307056, -0.05453723, -0.0005727914, -0.025014007, 0.049886376, 0.008433057, -0.051346187, -0.041419882, 0.04396962, -0.049525388, -0.018309098, 0.004441453, 0.065456584, -0.041756548, -0.03949578, -0.0072174687, 0.029480498, -0.071984574, -0.00419507, -0.0668992, 0.021025969, 0.048755288, 0.060012426, 0.0030790034, 0.025659366, 0.058263715, 0.0033977844, 0.010791784, 0.023907438, 0.02730727, 0.042638548, -0.003106253, -0.015726404, 0.041830007, -0.06647079, -0.05831685, 0.01854986, 0.0780799, 0.03807721, -0.06728864, -0.0061848136, -0.038556628, -0.018404745, -0.025441885, 0.00691167, -0.008677547, -0.06668409, 0.023131195, 0.044816904, 0.007217074, 0.013185188, -0.032754898, 0.006822609, 0.0092148995, -0.03260653, 0.058298662, -0.021404881, 0.021939335, -0.0010621672, 0.02054866, 0.014421813, -0.018227898, 0.039992407, -0.03923048, 0.04858465, -0.027103912, 0.0035577833, -0.06397005, -0.013215655, 0.07943894, 0.0045200894, -0.050267175, 0.022300014, -0.0002333903, -0.027584493, 0.03674835, -0.018619519, -0.010559553, 0.021760514, -0.028975315, 0.019923756, 0.056267213, -0.07768898, 0.031517547, -0.02571657, -0.02584142, 0.002929089, 0.027772225, -0.031877816, -0.019398566, -0.042063255, -0.03126415, 0.02709501, 0.022605404, -0.06134324, 0.015475145, 0.00745332, -0.014457645, -0.04548829, 0.00000727035, -0.067079194, 0.06932425, 0.048112206, -0.022189187, -0.016214767, 0.029382594, -0.048544165, 0.05780167, 0.011591139, -0.0061681787, 0.060569678, 0.082990706, 0.023114264, 0.022303844, 0.03863199, -0.06789708, 0.045879666, 0.021658655, -0.053495545, 0.037751272, -0.01476751, 0.011911629, -0.037331454, -0.022689465, 0.04126727, 0.027109114, -0.012859417, 0.0059286137, -0.013316578, 0.029587226, 0.0074344072, 0.055398133, 0.014932943, -0.029620593, 0.017385364, 0.035810214, 0.014763117, -0.0011076129, 0.030533163, -0.02227942, -0.03895932, 0.05994361, 0.012197344, 0.048204303, -0.023918495, 0.01145748, -0.027455637, -0.03332055, 0.035604242, 0.040828057, 0.015480064, -0.04163014, 0.025578387, 0.01275603, 0.028803617, 0.047159582, -0.0836279, -0.017739516, -0.027314307, -0.0029751717, -0.038284473, -0.024428504, 0.0068212617, -0.010898514, -0.020928958, 0.0092228865, -0.026894545, 0.0005016268, 0.040837068, -0.014831693, 0.05012382, 0.010434221, 0.007975583, 0.022403065, -0.021582102, -0.03246648, -0.0037790926, -0.058888007, -0.019308798, -0.054617334, 0.010705726, 0.025257234, -0.014247689, 0.026835836, 0.0026131961, 0.040793132, -0.02970963, 0.05066446, -0.057971306, 0.021165451, -0.07498206, 0.0046932953, 0.064369716, -0.021624656, -0.022287382, -0.06448667, -0.06871652, 0.0056656008, -0.025666397, 0.005257959, -0.05486289, -0.05384009, -0.025691262, 0.021392291, 0.011339386, 0.016224923, 0.034769535, -0.004684112, 0.052433793, 0.01875641, -0.014472824, -0.017660473, -0.039760925, -0.06142278, 0.0048966063, 0.056737777, 0.028541887, -0.08926853, 0.058841046, -0.003926312, -0.021733789, 0.01450372, -0.0015257563, 0.021179426, -0.01707721, -0.012405496, 0.025904087, -0.028608583, -0.06993624, 0.011569178, 0.017523551, 0.010880845, -0.030457394, -0.029486833, 0.007916608, -0.024429994, 0.020400971, -0.052473836, -0.033593755, 0.038261995, -0.015976671, 0.1005239, -0.03703548, -0.017351901, -0.041434944, 0.027686425, -0.013097795, 0.0125670545, 0.012247188, 0.015960965, 0.009406213, -0.046540365, 0.01607188, -0.02073562, 0.005338549, 0.02828684, -0.036089744, -0.0010818104, -0.042375218, 0.007183511, -0.030207077, 0.021835793, -0.048185367, -0.028818065, -0.09036574, -0.045243867, 0.053618044, 0.0032427297, 0.0032584618, 0.04324319, 0.006774018, -0.008972486, -0.011541097, 0.03171358, -0.011992992, 0.027316457, -0.0052594366, -0.0026795939, -0.022740783, 0.03721574, -0.010971492, -0.028241668, -0.015222259, -0.013533346, 0.055462874, -0.073251106, 0.01800735, 0.00009683943, 0.022056466, 0.039512966, 0.028562134, 0.0009409392, -0.04444035, -0.038695693, 0.035260327, -0.028657492, 0.023599476, 0.022901434, -0.018016396, -0.016506363, 0.057953488, -0.029494612, 0.015375981, 0.02049014, -0.04168208, 0.038992684, 0.022307783, 0.0049580163, -0.03157462, 0.036502156, -0.0563555, -0.00022825995, 0.02924201, -0.046749886, 0.019278025, -0.00067703874, 0.04353252, -0.02790451, 0.0007397082, -0.24680674, 0.031699892, 0.019435676, 0.011009193, 0.0039601787, 0.04428155, -0.01449104, 0.005444395, 0.038702242, -0.0045075533, 0.02543194, -0.040123213, 0.01093675, -0.045070034, -0.0557588, 0.0021415472, -0.028095804, -0.03609805, 0.015262303, 0.0021758503, -0.0033176607, 0.023778591, -0.031071791, -0.035808805, -0.027754083, -0.041073315, -0.08415514, 0.0077781836, 0.043171775, 0.05349504, 0.03316098, -0.022170592, 0.021102222, 0.031137204, -0.0157257, -0.01730986, -0.011393508, 0.008921229, -0.054073855, 0.043627497, 0.021380572, -0.068331115, -0.039561976, 0.03523733, 0.0001583176, 0.071432635, 0.0106479395, 0.027277464, -0.0023154737, -0.031356957, 0.032559037, 0.025753384, -0.07006144, 0.0032614083, 0.012475227, 0.018287512, 0.032302484, 0.031048901, -0.017194754, -0.012926855, 0.045125313, 0.029580213, -0.02343724, 0.02466446, -0.046199292, -0.015467049, 0.03210578, -0.018263452, -0.043799717, 0.020102153, 0.0019322236, -0.0429199, -0.03405871, 0.057236057, 0.023372795, 0.010692343, -0.00008040083, 0.004161806, -0.039188445, 0.030502852, -0.00007192203, -0.054635167, -0.015151856, -0.00042348146, -0.036261845, -0.032504465, -0.017043648, -0.023796799, 0.045662466, 0.011820507, -0.047356002, -0.019456794, 0.009954664, 0.010133952, 0.02176774, -0.023860728, -0.004418547, 0.004892221, -0.016382301, -0.03576424, -0.00077688496, -0.0039416384, -0.0073909448, 0.014251779, 0.03200902, -0.020736316, -0.062107597, 0.043690976, 0.0057150554, -0.037014384, 0.03007782, 0.021529734, 0.037076153, -0.010561156, 0.03500154, -0.009287597, 0.018979762, 0.010503046, -0.022600176, -0.03864317, -0.023162887, -0.011994211, 0.037590526, -0.018494854, 0.054067463, -0.015714012, -0.02484324, 0.017594786, 0.03490957, 0.044518143, -0.032466765, -0.028585553, 0.10140187, -0.048015907, -0.009823204, 0.042389255, 0.026696902, -0.017442988, 0.03531005, 0.040503968, 0.07361406, -0.05684766, -0.013543393, 0.029442392, -0.0024543605, 0.027770955, 0.051819384, -0.024346305, 0.029330468, -0.007298032, 0.00008457184, -0.069918916, -0.034624115, -0.021060506, -0.023815513, 0.017852757, 0.020940999, 0.009269787, -0.0153379375, 0.032498624, -0.027577199, -0.044269204, 0.03812525, 0.008195187, -0.00011462145, 0.01337983, 0.016442668, -0.0020617405, -0.008300439, 0.022865655, 0.00027253327, 0.0075787595, 0.030417947, 0.0024548233, 0.0141333155, 0.004240132, 0.041922037, -0.03870808, -0.060922597, 0.010729094} + testVectorDifferentE5_base := []float32{ // query: I like painting + // -0.02336321, 0.019837355, -0.0038695897, 0.03372489, 0.043691404, -0.015549358, -0.024214335, -0.04676708, 0.0076778354, 0.031174576, -0.0137037765, -0.009892124, 0.09196916, 0.009186889, -0.013382591, -0.026355695, 0.040275555, -0.025541864, 0.054552197, 0.002962163, 0.02714663, -0.012540173, 0.018679062, -0.036588956, 0.0564109, -0.022487119, 0.017805604, 0.03149962, -0.030840518, 0.005271948, 0.039901365, -0.002532756, -0.015220161, 0.0076744608, 0.054024078, 0.042154193, 0.020605853, -0.044263635, 0.032907803, 0.016624568, 0.008122575, 0.016446194, 0.06768991, -0.031053565, 0.06073659, -0.015252329, 0.03940874, 0.03687515, -0.04935728, -0.021226017, -0.008890082, -0.0057282113, 0.019895403, -0.04392651, -0.050262954, -0.027177079, 0.02601091, 0.039524306, -0.04457725, 0.038820863, 0.027403455, 0.072750255, 0.015799824, 0.0345042, 0.029544767, -0.046739537, -0.014038381, -0.048448473, -0.059173018, -0.023618639, 0.002689697, -0.016802372, 0.040454574, 0.009241005, -0.050672166, -0.013881684, -0.045372017, 0.036265276, 0.0051959185, -0.0031371047, 0.04567564, -0.01600656, 0.027201325, 0.03681721, 0.0023231027, -0.023921838, -0.049419664, 0.026716363, 0.022139253, 0.06333812, 0.011309145, -0.020991782, -0.013909552, -0.0042177103, -0.026843915, 0.0009064508, 0.010261989, 0.020796286, 0.034263797, -0.03376896, -0.011088198, -0.0633043, -0.03914894, -0.02938757, -0.09013375, -0.027201645, -0.030567186, -0.03952351, 0.048105642, -0.028582824, -0.0054764473, 0.00993877, 0.07272825, -0.057445437, 0.01661358, 0.0012518872, -0.004690316, -0.0531584, 0.040481575, -0.058833014, -0.0045282757, 0.027275223, -0.0058146245, -0.00895612, 0.043915614, -0.011081434, 0.019683542, -0.0132628, 0.03798402, -0.041439638, 0.0006366584, -0.04926728, -0.0096456995, 0.015090439, -0.0374376, 0.0407033, 0.027019532, 0.06694459, 0.02153688, 0.026709631, -0.0022394604, -0.04558875, -0.005730659, 0.04837227, 0.027511489, -0.060490597, 0.026216889, 0.017121455, -0.017902197, 0.03218961, 0.03606503, -0.021662878, -0.04002208, -0.051584605, 0.018310636, 0.00089577085, -0.04681253, -0.023853278, -0.028394535, 0.032616753, 0.023152404, 0.04951005, 0.014019228, -0.011838665, 0.016076077, -0.041223582, -0.009660948, 0.00025667637, -0.022087123, 0.0024255808, -0.036950924, -0.016759869, -0.032264974, 0.02771159, 0.03899299, -0.0203981, -0.025226314, -0.036597986, -0.021417158, -0.08870188, -0.060968805, -0.038902152, -0.006457136, 0.036518622, -0.021758415, -0.01566722, -0.0063047064, -0.021475215, 0.030303659, -0.017290628, -0.028365074, 0.026076408, 0.028183239, 0.01970376, 0.0166941, 0.043388303, 0.03627586, 0.008949523, -0.016992016, -0.059386145, 0.034679346, -0.0143074645, 0.062117655, 0.051952623, -0.021773575, -0.09078664, 0.035045493, 0.10745923, 0.035388414, 0.03831467, -0.046317816, 0.026125062, -0.05642268, 0.040893715, -0.0023816333, -0.027526429, 0.03280616, 0.022280809, -0.03242884, 0.016098386, 0.016972791, -0.07712321, 0.019172808, 0.025096903, -0.05003449, -0.019927679, 0.035188496, -0.012198117, 0.010461012, 0.02399894, 0.02918795, 0.025135666, -0.026523324, -0.011022996, -0.018351462, -0.052543785, 0.04011673, -0.024458911, -0.018419059, -0.10116024, -0.04576256, 0.016557528, 0.04914916, -0.020344246, 0.00660589, -0.016192477, -0.04129158, 0.01735212, -0.05513587, 0.02219395, 0.013770613, -0.028205829, 0.04352466, -0.016857397, 0.03347387, 0.001306097, -0.01947957, 0.01709819, 0.03224302, 0.039338004, 0.0063237445, -0.045885254, -0.021277227, -0.03221995, 0.037464507, 0.01981262, 0.13106294, -0.0120858075, -0.012333284, -0.0226149, -0.036292184, -0.0482057, -0.018022116, 0.02563716, 0.017306872, -0.037882376, -0.041894127, 0.031004166, -0.040048543, -0.011916146, -0.008020464, 0.053429943, -0.06451053, -0.030747332, -0.044082996, 0.030079609, -0.079014175, 0.030401798, -0.03781568, 0.018312342, 0.07066772, 0.019448936, -0.012776812, 0.052622475, 0.04129895, 0.023234727, 0.014137803, -0.0018127782, 0.01715512, -0.019997384, -0.011221154, -0.005054763, 0.051301017, -0.054110423, -0.04620277, -0.0061338125, 0.09014376, 0.039321613, -0.036645956, 0.05546327, -0.019576404, -0.01673953, -0.05661276, 0.02312617, -0.00006647227, -0.039587036, -0.01047593, 0.06366954, -0.010485849, 0.025704037, -0.053342365, 0.015537648, -0.00029164983, -0.023684874, 0.060930762, -0.024080997, 0.021368923, 0.03693938, 0.01736275, 0.0112395445, -0.026395421, 0.04253799, -0.0051910053, 0.025726814, -0.0295392, 0.046326924, -0.042060863, -0.030081354, 0.08687526, -0.0074656857, -0.037617657, 0.031468093, -0.017084688, -0.032982297, 0.05551646, -0.022739831, -0.008537102, 0.015107541, -0.014384219, 0.015305601, 0.06190898, -0.06396244, 0.052941144, -0.025826322, -0.009342638, 0.020151673, -0.0028878388, -0.030006304, -0.02637738, -0.009359798, -0.01799995, 0.019250551, 0.006403673, -0.063350625, -0.0011826062, 0.039937705, 0.006624355, -0.016013678, 0.022685146, -0.031986896, 0.09164377, 0.012395545, -0.008652471, 0.004198986, 0.055989943, -0.00635101, 0.008216189, -0.012368214, 0.015156171, 0.047026813, 0.05454251, 0.027386, 0.02769734, 0.045759145, -0.053666953, 0.046987575, 0.03664886, -0.07980762, -0.000155167, -0.010758614, 0.0030010503, -0.054545853, -0.03609769, 0.033564515, 0.017801426, -0.033220895, 0.06225049, 0.019120976, 0.0053193257, -0.04761688, 0.04073422, 0.02707906, -0.022142585, -0.0022458306, 0.0012472839, 0.038143206, 0.0051688985, 0.06905968, -0.028650327, -0.044673517, 0.061245665, -0.017076049, 0.021942548, -0.010864585, 0.03777183, -0.02632364, -0.013318944, 0.02518357, 0.04150126, 0.015538562, -0.022774914, 0.02836117, 0.024473874, 0.026770474, 0.014075271, -0.051430847, -0.054649808, -0.039502822, -0.0073518427, -0.034412183, 0.000090333364, -0.0006135918, -0.017111054, -0.0010776316, -0.0013305194, -0.028990882, -0.0068199444, 0.030404016, -0.049215015, 0.024232728, 0.011977101, 0.028903184, 0.021684589, -0.00128397, -0.03319787, 0.012646341, -0.060535956, -0.0016722154, -0.042239737, -0.010442771, 0.032370936, 0.025078246, 0.009979579, -0.024450187, 0.015813034, -0.022161396, 0.026206374, -0.006762047, 0.026650878, -0.09395866, 0.012127471, 0.06171129, -0.035507888, -0.008815436, -0.07196415, -0.05985735, -0.0049316376, -0.03830218, 0.03205016, -0.023963032, -0.06824887, -0.041677777, 0.020483438, 0.022668349, 0.03683897, 0.024355097, -0.0034945607, 0.05584646, 0.025161723, -0.027171569, -0.030951515, -0.018369611, -0.042484332, 0.045719244, 0.065664805, 0.040647022, -0.05687456, 0.025369907, 0.009527679, -0.03497479, -0.0030283704, 0.038843058, 0.0027050264, -0.0139975175, 0.019387884, 0.041531224, -0.010412352, -0.06992981, 0.050758887, 0.003483931, -0.02431717, -0.033363473, -0.06393715, -0.011645766, -0.04558367, 0.0113439765, -0.026066953, -0.022986695, 0.018001778, -0.030688439, 0.123291805, -0.037180997, 0.006867328, -0.064314805, -0.007587616, 0.004321473, -0.012493101, 0.017232627, -0.014330474, 0.014041444, -0.00012393988, 0.017920246, -0.013978413, -0.012532587, 0.042038005, -0.020706484, 0.024609279, -0.037957653, 0.044185545, -0.008043884, 0.036998805, -0.026354209, -0.030536745, -0.06696761, -0.061609253, 0.036700144, -0.034808088, 0.021953844, 0.047163308, 0.0046100775, -0.004692234, -0.027807236, 0.025622398, -0.023100669, 0.037661236, -0.036310002, 0.023564987, -0.008126555, 0.042811286, -0.008606324, 0.009747932, -0.02034657, -0.025317067, 0.055994544, -0.063054375, 0.056265377, 0.045648158, -0.00054970884, 0.038699996, 0.024882939, 0.031383332, -0.033490054, -0.012602169, 0.034663796, -0.013413934, 0.026863666, 0.008428211, 0.010367869, -0.037229124, 0.053942095, -0.07185325, 0.007310202, 0.012046513, -0.047681633, 0.016359309, 0.052312907, 0.014897315, -0.035868425, 0.039696135, -0.048936825, -0.029780833, 0.020474859, -0.032936774, 0.039638516, 0.013465593, 0.016337337, -0.00447386, -0.038936634, -0.21299456, 0.03779608, 0.011606024, -0.014512678, -0.004731043, 0.0054501863, 0.0029132154, -0.011011916, 0.044796247, 0.006730351, 0.02605277, -0.05046375, 0.04275296, -0.0077171344, -0.060943417, 0.039683774, -0.012554338, -0.041799754, 0.047317065, 0.0044012903, 0.022160714, 0.03762341, -0.069575824, -0.015460013, -0.04843733, -0.008987679, -0.02367439, 0.037736468, 0.008398346, 0.058042865, 0.0230108, 0.011144777, 0.05049734, 0.0399619, -0.057358738, -0.024031911, -0.019639026, 0.035289507, -0.061110545, 0.04191741, 0.017410833, -0.07334049, -0.076667614, 0.024337064, -0.011228577, 0.07490411, 0.009270507, 0.014807462, -0.03770079, -0.016920092, 0.020291371, 0.014497489, -0.06778111, 0.018074349, -0.023994667, 0.01633291, 0.04094041, 0.040696014, 0.027904812, 0.0003637607, 0.030394088, 0.020580593, -0.03449209, 0.018459484, -0.0140184425, -0.00970053, 0.03541496, -0.010102599, -0.043176934, 0.011941156, 0.020305585, -0.016892029, -0.033255138, 0.036591794, 0.043757036, -0.008294644, 0.038182084, -0.0285995, -0.086203925, 0.027872283, 0.03600018, -0.04857875, -0.020488959, 0.034320027, -0.023964545, -0.04269864, 0.008019393, 0.01833387, 0.013972221, 0.007736102, -0.06847742, -0.039382104, -0.008265135, 0.0013689012, 0.022547422, -0.04033807, -0.023197278, 0.045107502, -0.013678247, -0.048689377, -0.019036077, -0.022106491, 0.010274364, 0.014232516, 0.04352687, -0.05458684, -0.020508261, 0.013979162, -0.035854727, -0.011772535, 0.039485045, 0.052578148, 0.019514317, -0.018409263, -0.005864355, -0.032291085, 0.024474699, 0.00025018767, 0.019433035, -0.042575177, -0.012676581, 0.0007891536, 0.012900115, -0.020989414, 0.046777315, -0.026972318, 0.008951119, 0.040519398, 0.0421576, 0.020786365, -0.033786137, -0.037780948, 0.071219094, -0.02627985, -0.016484598, 0.039139874, 0.04925283, -0.0012252043, 0.029735524, 0.036646143, 0.04492653, -0.041671976, -0.014155713, 0.026889792, -0.018345768, 0.038769327, 0.026213845, -0.037073806, 0.0019904326, -0.0066414922, -0.022891039, -0.033962235, -0.021629464, -0.025176773, -0.028191123, 0.02251667, 0.021948121, 0.0016576158, -0.017432265, 0.045275614, -0.061207112, -0.0691782, 0.008990852, 0.006646415, 0.02277083, 0.010616498, 0.01873762, -0.011368887, -0.0042976327, 0.043786544, -0.021473864, 0.012977629, 0.040954534, -0.03158096, 0.021799322, 0.0013588985, 0.05741314, -0.029456588, -0.0342592, 0.041268114} + -0.01328194, 0.025625946, -0.01901248, 0.011385735, 0.049433395, -0.029005423, -0.005938906, -0.040633976, -0.009597312, 0.053360537, -0.029441142, 0.0041899076, 0.11862063, 0.007296435, -0.018630203, -0.008510325, 0.04951619, -0.043381292, 0.014321342, 0.0038835327, 0.048267163, -0.012132022, 0.0070304717, -0.046262003, 0.052023906, 0.010114522, 0.011300339, 0.02033603, -0.025960613, -0.001994171, 0.039957702, 0.018793268, 0.013193205, -0.006088252, 0.037021738, 0.04859339, 0.028481845, -0.028464034, 0.04229646, 0.027657615, 0.0027503597, 0.035522316, 0.072134286, -0.039922167, 0.045641463, -0.023198478, 0.03393587, 0.03730337, -0.040109, -0.0058861887, 0.0030701237, -0.028653061, 0.03688837, -0.0121702235, -0.060530532, -0.040732246, 0.020813143, 0.04273604, -0.029206883, 0.040570915, 0.013866085, 0.06368896, 0.0013757921, 0.038088188, 0.018012678, -0.03518204, -0.0037217904, -0.023199908, -0.050325416, -0.025207045, 0.01427242, -0.03432056, 0.035130665, 0.0031037459, -0.050504748, -0.0012585769, -0.028996954, 0.038225338, 0.0059921863, -0.024516828, 0.054294586, -0.025632836, 0.02771841, 0.03339324, -0.011559528, -0.035011455, -0.06374896, 0.026526237, 0.018196976, 0.061699506, 0.019326286, -0.036041316, -0.007369745, 0.0055157044, -0.019761147, 0.031252436, 0.026020037, 0.028701728, 0.0393115, -0.036714118, -0.00412962, -0.04516732, -0.027607704, -0.034986716, -0.0690843, -0.044306908, -0.028961308, -0.044950318, 0.026730243, -0.021608153, -0.018143516, 0.023932185, 0.062937856, -0.04872501, 0.015256001, -0.028588753, 0.014463296, -0.043432023, 0.037434384, -0.048604146, 0.010773134, 0.008593895, -0.013590531, -0.022622114, 0.03506685, -0.028870892, 0.017370539, -0.00019031136, 0.034068495, -0.06522779, -0.019414503, -0.033620752, -0.015099658, 0.04252749, -0.061493773, 0.047229443, 0.03334075, 0.063840546, 0.011897318, 0.015345278, 0.0053038043, -0.0649524, 0.01922586, 0.041615095, 0.039303076, -0.058305886, 0.039631855, 0.011824171, 0.012555004, 0.015444219, 0.05426397, -0.030587701, 0.0012223771, -0.06272976, 0.03374338, 0.0034460472, -0.029485185, -0.001400744, -0.009193461, 0.023318302, 0.026045825, 0.03327249, 0.02053008, -0.020563424, 0.00029667496, -0.046463918, -0.017699536, 0.024571843, -0.01990259, 0.00785993, -0.024214368, -0.012864183, -0.042736903, 0.017303376, 0.014066143, -0.029190795, -0.007402805, -0.048812024, -0.01933732, -0.07957746, -0.07218665, -0.031095712, -0.0025932118, 0.032531187, -0.024569971, -0.009824296, 0.009511447, -0.009653887, 0.026405318, -0.006718449, -0.023405386, 0.036529962, 0.011723433, 0.025253495, 0.0141779035, 0.037706364, 0.03975603, 0.02082436, -0.04202704, -0.039609097, 0.024996592, -0.012987079, 0.039961893, 0.028436849, -0.013636088, -0.089067005, 0.022247732, 0.10007821, 0.041807916, 0.05259758, -0.05706497, 0.023686986, -0.052030846, 0.02399948, 0.0039663482, -0.0031397578, 0.01630258, 0.015658267, -0.026743757, 0.024988476, 0.02657789, -0.07195007, 0.020732144, 0.02762789, -0.044072077, -0.03853845, 0.047287762, -0.0083078025, 0.0034095368, 0.017363906, 0.008688661, 0.022105623, -0.03577601, -0.016012605, -0.007032863, -0.026101582, 0.03389506, 0.0067884675, -0.022141784, -0.11435158, -0.05786312, 0.03174315, 0.025511555, -0.03348318, 0.012130267, -0.015329238, -0.038082894, 0.0098625645, -0.052299596, 0.029751606, 0.03484959, -0.051767323, 0.01395614, -0.03095477, 0.047677733, -0.014147194, -0.017289573, 0.038609564, 0.0309865, 0.046387527, 0.012858601, -0.04728358, -0.0047092075, -0.047270745, 0.017569648, 0.015554172, 0.112099744, -0.0074669183, -0.024415364, -0.030037096, -0.049114387, -0.037434794, -0.016158422, 0.044641998, 0.008968141, -0.032790966, -0.050763197, 0.034997437, -0.0496792, -0.003939191, 0.006437236, 0.038511235, -0.06149173, -0.016451698, -0.030064862, 0.024112875, -0.097929746, 0.004930246, -0.06726939, 0.0004917428, 0.06838667, 0.011391288, -0.018004589, 0.044294555, 0.06542205, 0.031113312, 0.002363749, -0.006072966, 0.00440402, -0.022649108, 0.0004464734, -0.008616083, 0.0515899, -0.05415439, -0.074977696, 0.011571072, 0.08187251, 0.021414796, -0.047028754, 0.020224934, -0.036375348, -0.015670804, -0.047414124, 0.017106371, 0.012636426, -0.04104471, 0.014433915, 0.04903245, -0.010929664, 0.010661506, -0.03612917, -0.0069424566, 0.0071058744, -0.011978801, 0.05216227, -0.04118902, 0.019259546, 0.017131548, 0.019396128, 0.008439738, -0.011663616, 0.032611925, -0.009640347, 0.021665316, -0.03978376, 0.04401317, -0.021201018, -0.03419377, 0.084419735, 0.008321896, -0.051391505, 0.021982552, -0.017023215, -0.02411165, 0.049272697, -0.019742556, -0.009304815, 0.008532808, -0.038368158, 0.027045734, 0.070275456, -0.079429574, 0.039197326, -0.029661857, -0.0056144684, 0.026391799, 0.008314897, -0.023999957, 0.0007704977, -0.016793124, -0.02667418, 0.016448108, 0.0076524923, -0.07448754, -0.015575617, 0.01696047, 0.010161439, -0.01929208, 0.027627992, -0.05295845, 0.09887276, 0.037077684, -0.019352932, -0.006531583, 0.05914919, -0.015408659, 0.039824966, 0.0050281724, -0.00020535447, 0.05417328, 0.050707087, 0.025711138, 0.05400978, 0.041369483, -0.037227765, 0.05137584, 0.047029205, -0.07031777, 0.0021058323, -0.0069994507, -0.005032413, -0.06570421, -0.039079376, 0.024954563, 0.023672968, -0.028578794, 0.033001978, -0.00027618514, 0.041529782, -0.018520465, 0.043354515, 0.0041187657, -0.038791962, -0.024415089, -0.007986599, 0.045676414, 0.025598882, 0.056186546, -0.027125454, -0.042094454, 0.05959069, -0.030197306, 0.04373132, 0.0063635735, 0.015599969, -0.028792465, -0.0016294223, 0.04345317, 0.048029378, 0.01399282, -0.03008586, 0.020830877, 0.0018740434, 0.022456868, 0.025970737, -0.06536566, -0.077812746, -0.027399609, -0.008908558, -0.021992812, 0.0019177799, 0.006690368, -0.033367418, 0.0045110723, -0.008884188, -0.025716104, -0.027926024, 0.009728821, -0.03093465, 0.045049433, -0.01997846, 0.03203735, 0.02621001, -0.016830422, -0.02479378, -0.008552834, -0.047603343, -0.024855528, -0.058107123, -0.020475354, 0.02155505, 0.01740746, 0.011786492, 0.0058626924, 0.01722686, -0.003895735, 0.020586668, -0.0077357935, 0.011018722, -0.056179255, 0.021708462, 0.0692652, -0.01256303, -0.0006939129, -0.073817015, -0.068419, 0.0063132453, -0.028662104, 0.027708555, -0.020320201, -0.07733789, -0.033279233, 0.0073058126, 0.018933147, 0.024045778, 0.016244518, -0.003234995, 0.03169793, 0.037088286, -0.028569978, -0.05069288, -0.034008622, -0.0566676, 0.026115501, 0.07299782, 0.027256124, -0.08685358, 0.022512345, 0.0067319907, -0.015881306, -0.005823954, 0.041962635, 0.011420966, -0.039264802, -0.011885012, 0.025400754, -0.018878426, -0.075025484, 0.059306514, -0.006713105, -0.007404647, -0.034727726, -0.039971862, 0.013952122, -0.033598762, 0.0031635845, -0.04792687, -0.0010193054, 0.02425074, -0.022108817, 0.112343736, -0.03508301, 0.0033069425, -0.041407987, 0.013672128, 0.0058367476, -0.0067801564, 0.01832387, -0.024534633, 0.0091836555, -0.021974057, 0.020660756, -0.02013397, -0.02301306, 0.021951448, -0.016712042, 0.013438689, -0.038699973, 0.035997577, -0.020610653, 0.04191126, -0.013449015, -0.03410887, -0.06873343, -0.049021844, 0.054486513, -0.021261083, 0.018692967, 0.047533993, -0.012778206, -0.022299316, -0.020110548, 0.016027568, -0.027011685, 0.032718796, -0.045546014, 0.01816114, -0.0023318476, 0.04483307, -0.00806707, -0.01417737, -0.025330346, -0.012558528, 0.06845212, -0.06555914, 0.03189295, 0.03947283, -0.009754145, 0.042028897, 0.022965845, 0.022479178, -0.03674385, -0.0061542573, 0.026110219, -0.013889084, 0.038639575, 0.010632204, 0.007424404, -0.05033705, 0.047696836, -0.069605485, 0.03635093, 0.010204369, -0.038898796, 0.012470165, 0.03634944, 0.026387695, -0.04106419, 0.041844364, -0.0604699, -0.012990523, 0.032872498, -0.043727007, 0.04215939, 0.012016194, 0.030052863, -0.0011775682, -0.014945804, -0.22441669, 0.0450485, 0.014150604, -0.010935502, -0.023351884, 0.0055944324, -0.01441563, 0.0016823427, 0.033070937, 0.027447386, 0.018018117, -0.04606999, 0.03092381, -0.037138294, -0.04297994, 0.023218494, 0.0021025832, -0.024560977, 0.031646617, -0.011373749, -0.00051159353, 0.060607657, -0.050682537, -0.032729477, -0.049339946, -0.010723838, -0.055486985, 0.043347966, 0.040909503, 0.054303568, 0.011262296, 0.011506204, 0.028287044, 0.03350595, -0.053338457, -0.00089641043, -0.022020783, 0.034085196, -0.041918237, 0.040985104, 0.01919047, -0.063582785, -0.05304671, 0.04027331, -0.0031959927, 0.07162849, -0.0060893344, 0.024991892, -0.03007293, 0.00013716797, 0.025057614, 0.015181516, -0.06670828, 0.017111368, -0.01348997, 0.016153103, 0.053832408, 0.04366938, 0.022318382, 0.00093564653, 0.042091236, 0.047486793, -0.03813288, 0.0037274482, -0.014291197, 0.009305278, 0.049075104, -0.017234392, -0.052115962, -0.011347522, 0.010436864, -0.0191719, -0.012150249, 0.05504101, 0.040395033, -0.0043961173, 0.023471974, -0.034408208, -0.07398337, 0.027448565, 0.035317034, -0.040618196, -0.018068613, 0.027153758, -0.009964599, -0.04096347, -0.0023795117, 0.020090424, 0.030931182, 0.007980991, -0.07644945, -0.03604017, -0.00012385276, 0.016978076, 0.021576768, -0.019565158, -0.019780073, 0.043109067, -0.0117629785, -0.047605447, -0.028668921, -0.014875895, 0.0102279745, 0.019345, 0.037504204, -0.04815065, -0.040508393, 0.029559324, -0.031159073, -0.020123968, 0.047607265, 0.04116993, 0.0062304945, -0.013517659, 0.0145870615, -0.037662927, 0.032595545, 0.0080543235, 0.02148813, -0.033564813, -0.022075946, 0.0014550597, -0.0078943875, -0.032140758, 0.040566996, -0.019352918, -0.00051804475, 0.036112748, 0.0375287, 0.032922104, -0.035439882, -0.008939565, 0.085518226, -0.045940887, -0.0065854327, 0.029299054, 0.03117634, -0.022450686, 0.029569313, 0.033359263, 0.058139775, -0.041345514, -0.0015337325, 0.02101361, -0.019564282, 0.053600486, 0.036402013, -0.024483278, -0.0029942424, -0.023990886, 0.0016473952, -0.055454075, -0.012425031, -0.02064308, -0.017793616, 0.02228354, 0.022494284, 0.006950523, -0.025109716, 0.05742677, -0.053252954, -0.033841085, 0.012791228, -0.0034574824, 0.029627215, 0.009230981, 0.037006665, -0.018751796, 0.008870444, 0.040718947, -0.02321312, 0.0081126755, 0.050242633, -0.0046777385, 0.029636897, -0.01235328, 0.042894874, -0.051178582, -0.039398175, 0.060457163} + testVectorVeryDifferentE5_base := []float32{ // query: The cat sat on the mat + // 0.0014011491, 0.013367304, -0.0018949462, 0.0163317, -0.009890373, -0.03306576, -0.046470985, -0.015366494, 0.02499652, 0.025193863, 0.026237505, -0.013479148, 0.14914244, -0.027957786, -0.021626748, -0.026066149, 0.016712029, -0.013603722, 0.017084692, 0.005437007, 0.03236993, 0.0016515487, 0.0185474, -0.0034727922, 0.054533876, -0.077043094, -0.015154962, 0.043068897, -0.035229933, 0.032842148, 0.025525024, -0.032853726, 0.010456511, 0.024890052, 0.055536684, 0.019014569, 0.0054135956, -0.026263306, 0.023860937, 0.011549176, 0.008618022, 0.018633684, 0.011152701, -0.036447205, 0.02265292, -0.0393289, 0.014938376, 0.045049056, -0.03871417, -0.03513472, -0.0033367854, 0.024652405, 0.034920603, -0.0025620116, -0.006193254, -0.03141045, 0.0040430697, 0.017203197, -0.02804245, 0.01147187, 0.020343062, 0.035545405, 0.00035795488, 0.024296647, 0.017268727, -0.044770487, -0.0011612853, -0.030962642, -0.049807183, -0.04547858, 0.00967465, -0.0074965884, 0.03842234, 0.036288075, -0.006330874, -0.01996863, -0.0332994, 0.0067987577, 0.0400745, 0.010926516, 0.049415182, 0.0052524745, 0.014447356, 0.04473385, -0.008491698, -0.030038962, -0.027827956, 0.053185444, 0.008270143, 0.07544675, -0.0036433155, 0.0070336675, -0.0692505, 0.021156698, 0.033609826, 0.026272332, 0.032383945, 0.04963458, 0.02150542, -0.051839978, 0.016878353, -0.08026223, -0.0045255367, -0.041490346, -0.051862482, 0.012532418, -0.01972497, -0.024922209, 0.073271155, -0.013819397, -0.010231667, 0.016059723, -0.0029945003, -0.05829412, 0.07102239, -0.03512828, 0.037135337, -0.03393391, 0.041645367, -0.041611813, -0.03205547, -0.011011646, -0.039050993, -0.014148165, 0.029716749, -0.062748194, 0.00077299017, -0.018578429, 0.033037376, -0.006052202, -0.004177261, -0.029116247, 0.0045101894, 0.019119853, 0.0010958712, 0.05696076, 0.02779838, 0.019854074, -0.014462808, 0.005898371, 0.01888174, -0.028894905, -0.002833683, 0.052812457, 0.014120467, -0.083007894, 0.022423195, -0.021937417, -0.005108473, 0.047081456, 0.031765137, -0.04192248, -0.030076465, -0.017435977, 0.02927522, -0.009720868, -0.053538933, -0.0108477585, -0.04783357, 0.02876841, 0.013787191, 0.023873517, 0.043520074, 0.006957512, 0.033956364, -0.021554796, -0.007659794, 0.016090186, -0.05635793, -0.0074122, -0.05211631, -0.030166473, -0.05896725, 0.0320251, 0.030110104, -0.0077746236, -0.012721793, -0.04786805, -0.02821664, -0.04187642, -0.038315788, -0.033937674, -0.02816261, 0.030042905, -0.020513456, -0.034921248, 0.017951189, -0.045335867, 0.008185409, -0.014353506, -0.000013241569, 0.057525054, 0.01552032, 0.016845595, -0.0016082956, 0.035690423, 0.04591188, 0.04018785, -0.03325741, -0.03916765, 0.021440199, -0.023185154, -0.0025837126, 0.011960595, 0.023173545, -0.07561545, 0.024111932, 0.03908016, 0.067069545, 0.021704815, 0.020167168, 0.027274918, -0.022828305, 0.00034235968, -0.032182626, -0.07180642, 0.024814703, 0.018611504, -0.025548229, -0.0070457687, 0.00760893, -0.07588472, -0.0043457136, 0.01314968, -0.024014704, 0.024779266, 0.024171775, -0.0011426926, 0.030512756, 0.022603422, 0.038295306, 0.050093006, -0.044459928, 0.014025893, -0.024295557, -0.03997007, 0.028957646, -0.008944525, -0.009615208, -0.08719084, -0.018769484, 0.047262903, 0.013222113, -0.017260255, 0.06913413, -0.03171311, -0.028342199, -0.0070505138, -0.06890954, -0.014846462, 0.013683853, -0.03262514, 0.02251255, -0.011656284, 0.016823672, -0.04237157, -0.048106264, 0.020299515, 0.038508657, 0.08624993, 0.00063999643, -0.057463676, -0.0097585395, -0.06610854, 0.016929906, 0.010681982, 0.09993145, -0.02174786, -0.026319262, -0.051339637, -0.028812654, -0.023959158, -0.029446665, 0.062068492, 0.017542547, -0.04493268, -0.07045327, 0.056761783, -0.014896972, 0.005481275, 0.027102765, 0.033670876, -0.037559185, -0.026668949, -0.030801967, 0.044653546, -0.05368705, -0.000120775934, -0.02791595, 0.021846617, 0.10197208, 0.08523933, 0.0061850427, 0.0306181, 0.056026105, 0.011170088, 0.056123197, -0.021004766, -0.02168693, 0.0030003989, -0.021216266, 0.0021734412, 0.06940698, -0.03395742, -0.029003609, -0.012504789, 0.11061512, 0.02963338, -0.017280376, 0.0065606516, -0.025563201, -0.027975213, -0.04693429, 0.011874052, 0.017629476, -0.024015252, 0.026468435, 0.055706944, 0.008051698, 0.023408903, -0.037499867, 0.032955665, 0.0031054565, -0.038042948, 0.054970566, -0.0047911927, 0.03947164, 0.014164727, 0.057115853, 0.014694853, 0.0006975369, 0.012836384, -0.017629307, 0.061588634, -0.053451166, 0.010898681, -0.018314473, -0.046855006, 0.04635233, 0.030677108, -0.026724942, -0.011023997, 0.019078234, -0.020474901, 0.018438613, -0.039171986, -0.013422225, 0.016204491, -0.032485668, 0.043497533, -0.0026323344, -0.048951823, 0.018578995, -0.03632627, -0.03804975, -0.00085536327, 0.033768646, -0.04161044, -0.042908285, 0.00091969233, -0.01523556, 0.044045784, 0.0019764362, 0.0020385787, 0.028171167, -0.03129555, -0.0041448097, -0.03890206, 0.0292804, 0.009420434, 0.06552485, 0.075871326, -0.036340475, -0.017599843, 0.023067195, -0.022204392, 0.03175299, -0.027488502, 0.009816051, 0.0038162088, 0.046960037, 0.029081998, 0.020853866, 0.0130747715, -0.04069706, 0.079621725, 0.05647948, -0.072845295, 0.021424187, -0.012105057, 0.01516341, -0.041287836, -0.010984926, 0.033021215, 0.02607316, -0.015874572, 0.017234467, 0.03258657, 0.017525852, -0.030647686, 0.07613524, 0.031415813, -0.012245964, 0.05676405, 0.030367212, 0.041427433, 0.0066008135, 0.06223305, -0.017761115, -0.023914399, 0.051062927, -0.06749461, -0.0124455495, -0.035482075, 0.02117099, -0.031005109, -0.014291871, 0.025532696, 0.02821366, -0.00026745975, -0.06696499, 0.028149446, 0.011957072, 0.0073920204, 0.038176414, -0.04231792, -0.017015932, -0.031089274, 0.002776035, -0.047718085, -0.011561328, 0.019942641, 0.035393853, -0.02572042, 0.040541843, -0.07437129, 0.04170091, 0.026198935, -0.007491934, 0.022278044, 0.007739494, 0.0069590886, 0.011716343, -0.009044844, -0.01275664, -0.003480655, -0.04848418, 0.010512007, -0.04106591, -0.023591403, 0.01906898, 0.0010162791, 0.025859943, -0.025920099, 0.041615628, -0.048299532, 0.050976235, -0.0060050506, 0.036823045, -0.07873227, 0.029711716, 0.039819743, 0.002108701, -0.00301336, -0.035162617, -0.06218988, -0.0013060138, -0.02596647, 0.011691486, -0.018972764, -0.02240691, -0.010759628, 0.020079318, -0.018396351, 0.02016688, 0.049577333, -0.0021639862, 0.024711108, 0.02580031, -0.031011593, -0.010248222, 0.003424608, -0.008941304, 0.033025958, 0.054887228, 0.04943277, -0.063717626, 0.043219358, 0.03773377, 0.005927733, -0.024841208, 0.011278313, -0.01776129, -0.008850875, 0.009365221, 0.02683587, -0.040110502, -0.09478373, 0.02359467, 0.0069164834, 0.0010232186, -0.060407914, -0.052364912, -0.012560525, -0.0789129, 0.015044871, -0.058651544, -0.04596656, 0.04694175, -0.0061805244, 0.07940964, -0.05263277, -0.045515195, -0.021991726, 0.02700103, 0.01095254, -0.02401964, 0.05606615, -0.0042727063, -0.009278634, -0.023526236, 0.027923122, -0.02130275, 0.0054340134, 0.029942304, -0.055933252, 0.015152426, -0.02249711, 0.0008620358, -0.01303875, 0.023951348, -0.043237537, 0.01624884, -0.109458014, -0.06826217, 0.043268997, -0.0681009, 0.00033189575, 0.012142004, -0.0003659245, 0.016338205, -0.029469153, 0.0592244, -0.016937686, 0.023604292, -0.022273649, 0.0080255, -0.022046493, 0.013899912, -0.02484092, 0.022814233, -0.0209147, -0.007021031, 0.032393433, -0.081009515, 0.06295789, 0.016628357, -0.00171994, 0.03480904, -0.019815482, 0.014085537, -0.021044115, 0.00043058113, 0.030984687, -0.011693777, 0.017698854, 0.03557002, 0.010545549, -0.015752647, 0.039775234, -0.062082894, 0.017020974, 0.010750042, -0.022884268, 0.019178113, 0.035777863, -0.029715875, -0.038674925, 0.042223096, -0.044593606, -0.020378403, -0.043003757, -0.015610551, 0.045053802, 0.026618196, 0.04910916, 0.021877311, -0.022418352, -0.22942854, 0.038582124, 0.0028699061, -0.046062674, 0.022051882, 0.024342524, 0.010156629, -0.025125021, 0.020561332, 0.019680446, -0.0022182309, -0.0130437175, 0.03928603, -0.03578414, -0.07289133, -0.004549046, -0.005756155, -0.051579773, 0.030942274, 0.057438843, -0.022075027, 0.046739627, -0.024950445, -0.0056390665, -0.042853486, 0.04080513, -0.03682196, 0.018463627, 0.036960095, 0.049877096, 0.024574889, -0.010192541, 0.03824888, 0.039554324, -0.040809732, 0.0091467835, -0.022252144, 0.023622887, -0.044794895, 0.035027802, -0.0068589426, -0.04471533, -0.062684596, 0.034089666, -0.03491869, 0.06434686, 0.040217247, 0.063133985, -0.034018148, -0.050971214, 0.015967086, 0.013078825, -0.049044304, 0.016098937, -0.02350189, 0.0047111274, 0.06690968, 0.022311974, 0.021937769, -0.02151742, -0.006898109, 0.0043347436, -0.01922632, 0.048439305, -0.044658985, -0.012374066, 0.038637266, -0.027860874, -0.057111673, -0.016147595, 0.06510093, -0.006553218, -0.09095183, 0.06737538, 0.044323687, 0.0140249, 0.014233591, -0.023884036, -0.077266105, 0.051531583, -0.008436543, -0.038720466, 0.0013432632, 0.015657322, -0.050262485, -0.02656529, -0.03838471, 0.0011322021, -0.0047108126, 0.017442128, -0.020635027, -0.012936393, 0.049331207, 0.010161267, 0.033516094, -0.05238897, -0.0014338982, 0.05617009, 0.0016571726, -0.02473275, -0.007392521, -0.0009803934, -0.01825584, 0.03421116, 0.005576713, -0.005841357, -0.07128478, 0.012428209, 0.007506945, -0.027723966, 0.0549684, 0.05266684, 0.02895425, -0.018574659, 0.015117916, -0.0068208766, -0.037041754, -0.016009368, -0.02149735, -0.079626575, -0.034701, -0.018791761, 0.023770098, -0.027641695, 0.017467756, -0.0128899375, -0.0018343348, 0.023888981, 0.016091304, 0.024177795, 0.0043661827, -0.012889221, 0.060076617, -0.012995921, -0.031045392, 0.040396277, 0.009032114, -0.033636134, 0.03455826, 0.043756418, 0.051813524, -0.04948173, -0.010247834, 0.012437501, 0.00042072948, 0.018725263, -0.0010982916, -0.024276104, 0.021394193, -0.014555079, 0.01682316, -0.0461494, -0.0073874346, -0.026122613, -0.044675194, 0.015603996, 0.025974482, 0.01304901, -0.008760618, 0.022504166, -0.014537064, -0.07689494, 0.03977959, 0.0280771, -0.003999882, -0.014973714, -0.02471326, 0.0030095403, -0.0011985261, 0.06425992, -0.0011008594, -0.008191769, 0.026603116, 0.011670841, -0.009516427, 0.028413353, 0.055001874, -0.04570934, -0.02808982, 0.02428901} + 0.03580939, 0.026464, -0.01213044, 0.016563762, 0.02530124, -0.044352267, -0.04654348, -0.022993749, 0.015210911, 0.027667735, 0.03330111, -0.0036994463, 0.15697394, -0.021733057, -0.04199832, -0.022111444, 0.017089467, -0.01606564, 0.017962174, 0.021710994, 0.044846818, -0.019068833, 0.0065790475, -0.010065451, 0.03715019, -0.04625798, -0.011559575, 0.031153467, -0.04459338, 0.028682817, 0.03724103, -0.032878045, 0.022737456, 0.024100604, 0.033280317, 0.03682297, -0.0076631973, -0.012147335, 0.039105672, 0.030559598, -0.0002990546, 0.010758399, 0.014864639, -0.031393196, 0.0073524537, -0.038740855, 0.012294024, 0.049636833, -0.03894044, -0.045460466, 0.009989444, 0.020627737, 0.02807468, 0.010540405, -0.011672303, -0.035691056, 0.011063937, 0.0032596153, -0.023209076, 0.016685717, 0.025446888, 0.038878523, -0.007773731, 0.014241559, 0.023902407, -0.03036726, 0.024713697, -0.012659824, -0.045148183, -0.03888545, 0.01628799, -0.029843975, 0.03647179, 0.026915502, -0.016645748, -0.020296259, -0.011657035, 0.016123721, 0.020854903, -0.01746859, 0.058367085, -0.0031712444, 0.030216852, 0.032242283, -0.010985153, -0.027847422, -0.028791998, 0.041153375, 0.004952971, 0.081713036, -0.0074270642, -0.007857799, -0.06255745, 0.02753631, 0.038288098, 0.046116196, 0.035689488, 0.031689208, 0.035412353, -0.04552838, 0.010243442, -0.07411059, -0.000087340595, -0.059744865, -0.06831022, 0.00086520496, -0.0009814533, -0.02719033, 0.06270721, 0.019254318, -0.02786127, 0.012672623, 0.0124107795, -0.07377766, 0.06517899, -0.046098236, 0.03363099, -0.033679787, 0.036788367, -0.05458852, -0.010070291, -0.018051716, -0.037848346, -0.018067537, 0.025625596, -0.06362779, 0.01417501, -0.015546611, 0.036533162, -0.005295226, -0.023630345, -0.002195789, 0.0005332577, 0.034776136, -0.0091509, 0.057439208, 0.01788, 0.029102128, 0.02306252, -0.0047305897, 0.0025580442, -0.028429758, 0.006613968, 0.042437483, 0.014774628, -0.0781, 0.045868788, -0.016145458, 0.0028423585, 0.045397636, 0.028956208, -0.04634546, -0.020598343, -0.020584192, 0.033640463, -0.022373386, -0.031994388, -0.017073445, -0.033502992, 0.009483382, 0.010629845, 0.02021115, 0.026246035, -0.009641313, 0.030123929, -0.028775882, -0.004712044, 0.0065941545, -0.04839579, -0.0102926735, -0.018178618, -0.010750914, -0.055257183, 0.03201061, 0.017267942, -0.03072623, -0.007826722, -0.05326557, -0.012526019, -0.037781246, -0.050946992, -0.047246642, -0.0038771546, 0.026653083, -0.020063298, -0.015314174, 0.042754706, -0.041565828, -0.0022344878, -0.010714833, -0.022103345, 0.051187444, 0.014740653, 0.039548814, 0.01149469, 0.049038332, 0.019445399, 0.041970022, -0.045199864, -0.043917015, 0.0152815655, -0.025189986, -0.013601829, -0.0026130325, 0.013340662, -0.05572656, -0.0012064921, 0.047727663, 0.097474284, 0.020450385, 0.026813637, 0.036020163, -0.028949462, -0.011850813, -0.020391438, -0.07754665, 0.028734257, 0.0032051539, -0.016704423, -0.00097284344, 0.024431659, -0.059259947, 0.0061001284, 0.019294042, -0.0023858778, -0.011270287, 0.01506763, -0.0039413082, 0.027827077, 0.024580345, 0.045331504, 0.02833132, -0.050609756, -0.0010317031, -0.030075116, -0.026855182, 0.0487532, 0.008766264, 0.017641075, -0.098758586, -0.028243415, 0.05768109, -0.0025808166, -0.033964097, 0.0746999, -0.02833761, -0.04519831, -0.008300649, -0.053321876, -0.014735503, 0.03269136, -0.039433073, 0.01325138, -0.014119967, 0.03177633, -0.056371506, -0.055335753, 0.022787405, 0.01859154, 0.07891876, 0.00028994933, -0.0524354, -0.023372885, -0.055651236, 0.0037950561, 0.009235235, 0.10844323, -0.033116095, -0.0528241, -0.04868141, -0.056374095, -0.024122413, -0.041244283, 0.06497296, 0.009464288, -0.061257303, -0.034937467, 0.03245371, 0.0032306446, 0.022624996, 0.01890434, 0.036395036, -0.026091566, -0.011616044, -0.029937284, 0.017487442, -0.09319224, -0.0012326605, -0.036328685, 0.02097783, 0.100141756, 0.07206761, -0.031655733, 0.027883599, 0.064670905, 0.015687458, 0.072266266, -0.00838589, -0.014358834, 0.011892, -0.024162777, -0.027322581, 0.058809355, -0.013474638, -0.018318852, -0.005112541, 0.10369624, 0.02675732, -0.02679399, 0.008313939, -0.03329075, -0.02555308, -0.051126912, 0.020091591, 0.022356752, -0.01159932, 0.015606944, 0.046943817, 0.0014670388, 0.023102276, -0.016729888, 0.02802712, 0.01342071, -0.04211418, 0.056674298, -0.006001473, 0.0200549, 0.0054992554, 0.069401614, 0.025362616, -0.009296769, 0.007650558, -0.019130614, 0.03953623, -0.04538593, 0.003419102, -0.027430288, -0.04249791, 0.06977905, 0.041870955, -0.025601365, 0.007446937, 0.014489178, -0.02499199, 0.018760003, -0.03277812, -0.0005231019, 0.016266936, -0.058316395, 0.03441423, 0.023035964, -0.054517236, 0.0044611874, -0.03949299, -0.04558529, -0.0044440846, 0.04161528, -0.037361715, -0.020502325, -0.0017410426, -0.01594325, 0.054753702, 0.022343468, -0.007193619, 0.035448212, -0.03469015, -0.010496245, -0.053286742, 0.04317631, -0.0028933224, 0.068097115, 0.06211562, -0.04495123, -0.040376145, 0.01729519, -0.04237833, 0.026155744, -0.0153302755, -0.004206843, 0.0137465475, 0.052489724, 0.02367976, 0.023610795, 0.0013534462, -0.025077831, 0.056820918, 0.07455551, -0.063467965, 0.0010611467, -0.018722137, 0.025492016, -0.03429155, -0.016420515, 0.004465369, 0.008990515, -0.0060544605, 0.00802092, 0.013432046, 0.029833585, -0.015420562, 0.09574206, 0.02463291, -0.0108808195, 0.03206675, 0.046437982, 0.042166494, 0.024767876, 0.052456673, -0.013677502, -0.01657111, 0.062477432, -0.06660967, -0.0066252197, -0.030647157, 0.008580277, -0.039273147, -0.015254131, 0.032707445, 0.010001962, -0.00390804, -0.07887058, 0.027104825, -0.0060108164, 0.002846902, 0.047342442, -0.062021114, -0.022870775, -0.039102513, 0.009996075, -0.029618118, -0.015976958, 0.023103252, 0.010756435, -0.033453755, 0.040475357, -0.06694241, 0.044523485, 0.014688286, -0.0070080757, 0.02024809, -0.0026555085, 0.0061009917, 0.016375905, 0.008010306, -0.0091633815, -0.02127942, -0.05209468, 0.009631742, -0.05877068, -0.0015075118, 0.027666567, 0.011716547, 0.024814917, -0.015922097, 0.03452982, -0.04640947, 0.038921606, -0.00010978665, 0.03539366, -0.048217915, 0.020500245, 0.045533724, -0.016820215, -0.00039086284, -0.023722364, -0.04837017, -0.00078655046, -0.04358872, -0.009101592, -0.025692936, -0.033249557, -0.034673914, 0.0070725908, -0.027088325, 0.007600821, 0.037865784, -0.009135298, 0.02259438, 0.031874422, -0.03270081, -0.01292791, -0.007751594, -0.029729638, 0.014652351, 0.056030236, 0.0566248, -0.07316781, 0.031017551, 0.025670731, 0.018820778, -0.0147480285, 0.009581604, -0.013262078, -0.024448201, 0.0014725673, 0.02755805, -0.04026297, -0.09268791, 0.021130675, -0.02038278, -0.0080389865, -0.020895418, -0.05423976, -0.009172557, -0.04558723, -0.019405287, -0.067900054, -0.015454567, 0.04584516, 0.0016677225, 0.0870152, -0.056501582, -0.049719762, -0.009008757, 0.036108047, 0.0068080104, -0.01943853, 0.06688497, -0.01914955, -0.0067304564, -0.042454768, 0.029280711, -0.025437217, 0.0106997425, 0.03367413, -0.04259089, 0.016552668, -0.020002404, 0.0041435948, -0.012208056, 0.026998045, -0.020413885, 0.010004963, -0.10994021, -0.06124715, 0.04960263, -0.047798377, 0.0044374187, 0.014868299, -0.025270676, 0.0125023145, -0.015938876, 0.06896536, -0.023776075, 0.02499681, -0.007506786, 0.0023561546, -0.012683932, 0.03699016, -0.02234668, 0.020191815, -0.04080457, -0.008946405, 0.039891586, -0.056263376, 0.034420263, 0.019588394, -0.00005678035, 0.035015017, -0.00089369086, 0.0145580545, -0.014729616, -0.005958823, 0.040821683, -0.032156948, 0.033130135, 0.03948769, 0.018908666, -0.023265019, 0.035292126, -0.048499066, 0.024485843, 0.016546693, -0.025340224, 0.01381188, 0.017551519, -0.02268461, -0.045117974, 0.037519675, -0.070608325, -0.028467858, -0.025613727, -0.019415956, 0.04000389, 0.018698791, 0.030743472, 0.0061453683, -0.028788691, -0.23824392, 0.040249243, -0.00812105, -0.046168867, 0.008246149, 0.04379255, 0.043077428, 0.0086352965, 0.0016319028, 0.032195624, 0.0011041295, -0.009473479, 0.03260257, -0.02463606, -0.06025793, -0.016964117, -0.015859643, -0.04838097, 0.02281291, 0.045565292, -0.020017998, 0.06503023, -0.023885109, 0.00198712, -0.034741394, 0.033930648, -0.059787523, 0.01025482, 0.05196191, 0.045966398, 0.016216928, 0.0027357244, 0.02676818, 0.04404638, -0.045609254, 0.0096576875, -0.0077352314, 0.03705382, -0.061694976, 0.032048542, -0.0010938937, -0.041605823, -0.074273005, 0.044966996, -0.030894302, 0.07351186, 0.039867245, 0.038476605, -0.03754253, -0.0454637, 0.022479521, 0.002506334, -0.06758531, 0.006639111, -0.03316554, 0.014796552, 0.071827844, 0.024615897, 0.016674088, -0.0190177, 0.005611636, 0.022339685, -0.022101764, 0.034735713, -0.024878452, -0.011436904, 0.040093254, -0.0194606, -0.055592023, -0.02210319, 0.037381325, 0.00026266873, -0.08415255, 0.091283225, 0.044183005, 0.02084302, 0.005154229, -0.011262448, -0.07213381, 0.040527917, -0.021805655, -0.031175878, 0.00056414824, 0.011552824, -0.03608933, -0.024166537, -0.03188249, 0.021400101, 0.0039549787, 0.00089049916, -0.019165892, -0.008995662, 0.03860156, 0.024446197, 0.016990816, -0.044117797, -0.011416454, 0.05039364, -0.0060887183, -0.005794658, -0.015710851, -0.007489864, -0.008933018, 0.040082905, 0.0007013043, -0.01725433, -0.069105476, 0.007915777, 0.014885353, -0.042281605, 0.059509628, 0.05264684, 0.022334525, -0.010200716, 0.00788102, 0.009153161, -0.036294557, -0.017992603, 0.00089065015, -0.06871859, -0.02642101, -0.016745567, 0.0099127535, -0.022826027, 0.030321853, 0.0010293563, -0.02928382, 0.007381128, 0.014052379, 0.04984958, 0.009443686, -0.003380246, 0.060660675, -0.02953247, -0.024717618, 0.044469506, 0.007059311, -0.02325688, 0.03925251, 0.037816007, 0.04641623, -0.06265945, -0.003307755, 0.0074480423, 0.0012180617, 0.017576814, -0.0029407993, -0.02865703, 0.031086404, -0.010176079, 0.019720366, -0.05335537, -0.008100967, -0.02001287, -0.022041693, 0.014949589, 0.032988872, 0.030353872, -0.027580986, 0.014525319, -0.0198074, -0.041362002, 0.043574646, 0.032198902, 0.011132246, -0.018766023, -0.038428914, -0.010249917, 0.0031612176, 0.041459676, -0.024306903, -0.000118798234, 0.028115695, 0.017392289, 0.022926925, 0.011859289, 0.042196278, -0.054195445, -0.0292163, 0.03231166} + + inputVectorBGEM3 := []float32{ // I like soccer + 0.0021939883, -0.0035830762, -0.05768352, 0.04182663, 0.004256862, -0.000023166154, 0.02446078, -0.026216758, 0.040568117, -0.010088646, -0.0019565837, -0.018847536, -0.0345627, -0.011326201, 0.0060975025, -0.01803004, -0.0001879779, -0.049830753, 0.03473655, -0.0056442395, 0.010878849, -0.009592243, 0.017919848, 0.022733195, 0.0022996825, 0.04537068, 0.0050276043, -0.0136839375, 0.0057085566, -0.010354074, 0.015782343, 0.0143766245, 0.020550303, -0.047364093, -0.022435635, -0.0034299481, -0.009923415, -0.04255546, -0.0037853909, 0.071351685, -0.0019850354, -0.04371562, 0.05942357, -0.06830058, -0.0074094166, -0.03227551, -0.0013437851, -0.0029096897, -0.035358258, -0.007112608, -0.014891319, -0.005059699, 0.049616437, -0.017045587, -0.014131918, 0.048693314, 0.043872803, -0.008077723, -0.034001533, 0.010613951, 0.002406345, -0.0009907058, 0.016123815, -0.010856043, 0.020721711, 0.091508314, 0.03492526, -0.022637026, -0.011952945, -0.022305971, 0.005304962, 0.006747619, -0.007201039, -0.00969582, -0.06277105, -0.0036661834, 0.015298349, -0.021390095, -0.019562757, 0.022050746, 0.07530168, -0.01269271, 0.00907666, -0.011777523, -0.020232067, 0.020519648, 0.011466888, 0.03613913, -0.029657852, -0.000572832, -0.038872078, 0.008946026, 0.012913718, -0.03596284, -0.015260283, 0.0034133687, -0.007411393, 0.0522496, 0.0019120092, 0.03150029, 0.038410764, 0.027536722, 0.0026245408, -0.05539731, 0.04950528, 0.0008945985, 0.03839099, 0.02939588, 0.019675158, -0.0021557983, 0.06821584, -0.017422564, 0.070429176, 0.0017000387, 0.019860893, -0.010478337, 0.008939332, 0.0062418147, -0.0035565384, -0.011660104, 0.01900217, 0.042371143, 0.01647133, 0.011086461, 0.0034192635, 0.009919475, -0.0029418424, 0.029197669, -0.0186931, 0.027950287, 0.009245842, 0.08688329, -0.019286485, 0.026705068, 0.017289773, -0.048226632, -0.0018858934, -0.010021699, 0.005259629, -0.049197055, 0.05353834, 0.048030786, 0.021957178, -0.069665216, 0.0092019765, -0.06221951, 0.021774817, 0.030432729, 0.026819605, -0.020064019, -0.03512274, 0.01877027, 0.015272194, -0.0067045107, 0.03907795, -0.037850775, 0.012629171, 0.035716224, -0.0066677146, 0.034046337, 0.007136802, -0.03526607, 0.00040833955, -0.003090166, 0.033063836, 0.039972626, 0.03583463, -0.0009655921, -0.012520546, -0.019353911, -0.03976014, -0.011461294, -0.0025369104, 0.0047875773, 0.023821322, 0.016663713, 0.02689205, -0.0006068402, 0.030692777, -0.032075536, 0.012924476, -0.04748512, -0.025697539, -0.033204205, -0.057963695, 0.020578515, 0.019550197, -0.03662805, -0.003153344, 0.03745986, -0.01224413, 0.0014329816, 0.019473718, -0.029958643, 0.031769667, -0.03403652, -0.02693279, -0.062205017, -0.00092371815, -0.02147047, -0.029872233, 0.017788779, 0.0051955953, 0.01691759, -0.032863375, -0.00080118136, -0.09616353, 0.0042274287, 0.026673691, -0.014201061, -0.02370686, -0.028041555, -0.008895496, 0.014637253, -0.049272243, 0.024269696, 0.015361698, -0.01836623, 0.0021201477, -0.026919346, 0.019621128, 0.04395561, 0.032975048, 0.01945726, -0.004828616, 0.008969155, -0.021815177, 0.026835106, -0.004437289, -0.014457525, 0.008514681, -0.011760838, -0.032184426, 0.016195493, -0.006794413, -0.009670277, -0.059700057, -0.020136844, -0.029339049, 0.0073133237, -0.0015528717, -0.03866838, 0.07622258, 0.0044270884, -0.013774419, 0.008937657, 0.0047389064, -0.011022109, -0.021927135, -0.0050341273, 0.025021423, -0.017533742, -0.045146838, 0.0013620274, 0.013349475, 0.016531466, 0.0024845689, 0.005147538, 0.009639107, 0.015117685, -0.006213206, 0.0106820185, -0.0017758748, 0.020220248, 0.035434913, -0.04308683, 0.01737002, 0.00776323, -0.02213627, 0.0070778537, -0.0290189, -0.019540071, -0.0017815814, -0.032428734, -0.03523321, 0.017550992, 0.048600476, -0.042091317, 0.017805295, -0.012304887, 0.022085272, 0.040915206, -0.0021747756, -0.014127044, 0.026581524, 0.019117543, -0.002510697, 0.030367719, -0.016525274, -0.07101848, 0.016003354, -0.033288017, 0.03346638, -0.014207855, -0.03724405, 0.0036921, 0.0049256915, -0.1562116, -0.0318396, -0.030876834, -0.008763582, -0.0012992164, -0.013437255, -0.018699072, 0.008840341, 0.0062444136, 0.036834598, 0.0011178424, -0.031274468, 0.022743044, 0.034107376, 0.0095262835, 0.011069528, -0.019201504, 0.0057364847, -0.019079357, -0.03224741, 0.010420159, -0.069698006, 0.056023974, -0.016307577, -0.026983995, 0.03644266, 0.043907296, -0.023908734, -0.069070846, 0.0028375413, 0.013458262, 0.008194312, -0.007568331, 0.018213663, 0.009145801, 0.029192565, 0.03900901, -0.06571306, -0.004048779, 0.048475254, 0.02986055, 0.022391653, -0.048270427, 0.022686537, -0.04796609, -0.032294728, -0.04324824, 0.042526767, 0.01343118, -0.035360076, -0.024475109, -0.027629828, 0.01782737, -0.041496444, -0.015108441, -0.0053227153, -0.043237798, 0.011596896, 0.004322109, 0.0129439505, -0.021492567, -0.09405797, 0.0624444, 0.03409272, -0.00746507, 0.015616159, -0.0052301106, 0.005179418, 0.013869466, -0.008583914, 0.028035093, -0.033497404, 0.007370191, -0.035352122, 0.029667903, 0.02617945, -0.030643536, -0.0031993971, -0.016499503, -0.09909665, 0.0256255, 0.0019406799, -0.028180309, 0.024394806, -0.003868234, -0.03140973, 0.01718984, 0.0021981748, 0.015281165, 0.2633792, 0.05646935, 0.06969483, -0.050444413, 0.047326807, 0.003280329, 0.001595509, 0.014395463, -0.037020575, -0.023145713, -0.03633352, 0.0032782052, -0.00009645344, -0.0072788787, -0.016365422, 0.012324111, 0.0038126234, 0.027789744, 0.0404218, -0.024927163, -0.010098069, -0.0359961, 0.04761012, 0.015371516, 0.014515096, -0.05563522, -0.013123538, 0.051997386, 0.0054952903, -0.022211378, -0.032446004, -0.0039968044, -0.010477468, 0.0069035515, 0.0055288645, -0.013959455, -0.016761642, 0.002760798, -0.027685475, 0.041532613, 0.036594227, -0.011592737, 0.0038488617, -0.01885375, -0.013313812, -0.02183152, 0.009846868, -0.041310236, -0.022591135, -0.014246086, 0.010946489, 0.030660158, -0.035313953, -0.009473992, 0.012385716, 0.0119998185, 0.0033254186, -0.01094377, 0.0020153588, 0.02821331, 0.010883076, 0.035197347, -0.012433615, 0.015303992, -0.0052149054, -0.0075436328, 0.013383003, -0.01479035, 0.036256995, 0.018929357, 0.04074329, 0.02237598, -0.028940795, 0.024042394, 0.026872054, 0.058395606, -0.018199457, 0.07759626, 0.007084527, -0.0026277793, 0.010220107, -0.021390945, -0.051903073, 0.006121293, 0.018480156, -0.028398657, -0.010812457, 0.015506101, 0.04218421, -0.017772043, 0.046843328, 0.015717074, -0.035247974, -0.0024578227, -0.008687992, -0.023393473, -0.004527215, -0.040379018, -0.011506511, 0.011953311, -0.046734486, 0.007467726, 0.01011658, -0.04401683, 0.02349459, 0.0011083112, -0.036157988, 0.012075608, -0.021193773, 0.0028004881, 0.008660387, -0.00017683307, -0.02022933, 0.0280118, 0.04992598, 0.03193019, 0.015044943, 0.022705795, 0.030705312, 0.00041578308, -0.055423867, 0.002291571, -0.018930921, 0.021787144, -0.035058275, -0.024760542, 0.021932403, 0.035200384, -0.03352271, 0.04499975, 0.00077098334, -0.03363989, -0.017651776, 0.015831884, 0.10877343, -0.019413771, 0.047635738, 0.022407152, -0.009673115, 0.037895065, 0.01948267, -0.010167324, -0.023943909, 0.030499136, -0.00060654123, 0.0302065, -0.014713843, 0.029678209, 0.0062539116, 0.0036913783, 0.048581995, 0.0492521, 0.013509118, -0.025198415, -0.009995738, -0.032297306, -0.038964555, 0.001649296, -0.018179154, -0.038281266, -0.00086916226, 0.03924901, -0.0022319106, 0.115455076, -0.035745274, 0.016843826, -0.03927024, 0.0038910294, -0.00041888133, -0.04340636, 0.01094735, -0.019395242, -0.016772147, -0.021232583, -0.021790901, 0.0061283354, -0.04820161, 0.030783538, -0.03842508, 0.012707911, 0.04216517, -0.0038044679, -0.004658158, -0.012993661, -0.02083717, 0.026194649, 0.012477582, -0.005926699, 0.03047603, -0.034044556, -0.0207697, 0.078599475, -0.022590823, 0.006158006, 0.024606515, 0.04637135, 0.029262554, 0.0077504283, -0.019123234, -0.035862383, -0.011057244, -0.08159327, -0.02070507, -0.014828222, -0.00074384996, -0.036882978, -0.06149417, 0.003566633, -0.017514374, -0.030004568, -0.0152245695, 0.028194265, -0.021702863, 0.0023783252, 0.0007628969, -0.011939519, 0.01964286, -0.00612798, -0.0051132008, -0.033022393, -0.04657408, 0.05073277, 0.04224692, -0.022375245, -0.005428572, -0.041046537, 0.032856915, 0.0018954274, 0.0037234349, -0.030571977, 0.0033621779, -0.042608168, -0.030608524, 0.003232139, 0.038910277, -0.020311916, -0.04605376, 0.049007, 0.016188707, -0.04000347, -0.019980678, 0.008622599, 0.054616336, 0.022672364, 0.0021500825, 0.011550037, -0.010238505, 0.020910652, 0.02870737, 0.019450713, -0.029315041, 0.033540037, 0.042116877, -0.019754246, -0.035594583, -0.008853622, 0.004412545, -0.028050207, -0.016843341, -0.014978589, 0.0014834602, 0.04039133, -0.020043213, -0.027763695, 0.03469856, -0.023634871, 0.040073108, -0.047062796, -0.0021138545, -0.07593324, -0.017878009, 0.021620752, -0.018732822, 0.021452539, 0.07520598, 0.009501585, -0.017322473, 0.018435236, 0.020647211, -0.01416132, -0.0015375029, 0.021209754, -0.040529657, -0.0153727615, -0.011413308, -0.003957361, -0.0066761174, 0.023237396, 0.012303045, -0.01685711, 0.004106364, -0.04439903, -0.033124954, 0.03886917, -0.007894089, -0.0027634618, -0.04392842, 0.015988521, -0.008426271, 0.011919383, -0.032913357, 0.022805171, -0.0351445, -0.04383646, 0.03456477, -0.014809226, 0.0063606324, -0.028937042, 0.004456073, -0.009800672, 0.0133087095, -0.017896574, -0.022021106, 0.031564727, 0.010242175, -0.009891154, 0.03240012, -0.008098862, -0.042342294, -0.012947504, 0.0013034147, -0.010814276, -0.019751579, -0.00040167858, -0.019165887, 0.005196858, 0.002393225, 0.015506564, -0.026980571, -0.0051642233, 0.012842757, -0.013840907, -0.038440324, -0.009512099, 0.0025824413, -0.030106494, 0.041405186, -0.02123375, -0.0012376773, 0.011733499, -0.016064165, 0.07667558, 0.0703585, 0.0019879257, 0.02473134, -0.03457274, -0.004577102, -0.0011731475, 0.044761542, -0.0032652612, 0.020286474, 0.05833181, -0.052679487, -0.031892687, 0.026265888, -0.0427203, 0.0037842647, -0.05122356, -0.0041421438, -0.04595175, -0.028235087, 0.010585015, -0.01970493, 0.005279428, 0.003603542, 0.052244075, -0.05049547, -0.018991958, -0.0017268629, 0.009005949, 0.018346297, 0.023087444, 0.01311208, 0.003034398, -0.062948555, 0.044511173, 0.003261606, -0.024728466, 0.027059346, 0.014053984, 0.024773786, -0.018402366, -0.008069446, -0.0057266024, -0.022834666, -0.04182028, -0.0061133076, 0.060241036, 0.02263837, 0.029649979, -0.0014732333, -0.027468255, 0.03977884, -0.0074872933, -0.13935676, -0.0041115535, -0.029767413, -0.0006480699, -0.023230169, -0.0048062345, -0.033268046, -0.022598715, 0.009615599, -0.044401094, 0.008222877, 0.035355218, -0.00709274, -0.005302747, 0.015189921, -0.032436736, -0.0050906716, -0.027239949, 0.034820594, 0.052774113, 0.024717357, 0.027048387, 0.009295997, 0.0021396182, -0.010345654, -0.017070644, -0.011320446, 0.0074012703, -0.043383893, -0.014247113, 0.03739618, -0.102933064, -0.007384805, 0.06040902, 0.044666972, -0.025643494, -0.004963047, -0.0023250408, 0.029769635, 0.022456808, 0.023745593, 0.0078146225, -0.020886935, 0.0016893763, -0.0012048592, -0.00035069996, -0.012765778, 0.0013509808, -0.041246213, -0.0248977, 0.022076244, -0.0067368043, 0.010337638, 0.06404466, -0.05667758, 0.043542445, 0.00854698, 0.03923337, 0.04210805, 0.012305684, 0.0005917237, -0.0073649785, -0.0126916645, -0.0138256075, -0.019411532, -0.024287637, -0.036808185, -0.0021454948, -0.0039074584, 0.04243192, 0.004284737, -0.023965519, -0.0011844452, -0.039550524, 0.008142175, 0.025799805, 0.008003862, -0.019004676, -0.04334295, -0.0028926483, 0.045532525, -0.012454594, -0.004372779, -0.026531832, 0.007015985, 0.014138802, -0.031828564, 0.010489073, -0.05957097, -0.032384228, -0.02184125, -0.04156272, -0.018458009, 0.031826388, -0.03534093, -0.02853421, -0.016329644, -0.03632643, -0.0044052927, -0.051195018, -0.045800183, -0.08454139, 0.023034358, -0.025771402, 0.00040697164, -0.014535312, 0.01830453, 0.024469458, 0.07423155, 0.034682505, -0.027669622, -0.018541118, -0.03808083, -0.023033163, -0.030520037, -0.039741136, 0.035575807, 0.0224079, 0.0088304505, -0.0140475705, -0.014529854, 0.0068758777, 0.035367057, -0.020704439, -0.008182379, -0.008181122, 0.046549883, 0.018830558, 0.00904454, 0.024017144, 0.037454847, -0.035391945, 0.011390855, 0.010776766, -0.008473505, -0.016589178, 0.0007134801, 0.03969672, -0.030921178, -0.016713364, -0.025986342, 0.01842033, -0.0333871, -0.0031703915, -0.04133427, 0.005268342, 0.008531621, -0.035773985, -0.036839355, -0.031633854, 0.069754824, 0.0028188068, -0.016305717, 0.007256893, -0.01975885, 0.011481142, 0.028215384, -0.02167797, 0.02687508, 0.022293068, 0.023113722, 0.021815617, 0.0061693857, -0.0055562346, 0.006088049, -0.0020776584, -0.010876097, -0.024088468, -0.028360138, 0.02380329, -0.016970571, 0.04118966, 0.0025462946, -0.005061881, 0.057071414, -0.053489085, 0.03948553, 0.026191827, 0.041473396, -0.0007571682, 0.033965714, 0.06040664, 0.016160091, 0.012563315, 0.014952288, -0.012756668, 0.022223445, 0.03978125, -0.014885787, -0.017908664, -0.03525362, 0.011520063, 0.022366831, -0.04491104, 0.040882863, 0.025906121, -0.016617877, 0.054068074, -0.02077987, -0.00022689869, -0.02928019, -0.06540712, 0.0015375671, -0.03555031, 0.021186162, 0.006031488, -0.04945094, -0.05134449, -0.029988218, 0.03575276, 0.0060494225, -0.04886984, -0.04832232, 0.019886833, 0.018083373, -0.014252697, -0.0118286, 0.008762879, 0.02178479, 0.04716444, -0.02163782, 0.011886837, -0.024184618, -0.02541746, -0.013272225, -0.0051834295, -0.064432, 0.0015301654, -0.012735508, -0.000111041045, -0.020655237, 0.017903358, -0.014810903, 0.05341555, 0.020900777, -0.01671981, 0.09222434, 0.039329052, 0.003098312, 0.0021789197, -0.00092014, 0.008969115, -0.04360675, 0.02739744} + testVectorSimilarBGEM3 := []float32{ //I love sports + -0.015574532, 0.002240773, -0.058977798, 0.051791035, 0.0012886648, -0.0017816275, 0.023119641, -0.03499189, 0.03301414, -0.025844444, -0.012131262, 0.0074946457, -0.025073802, -0.007849473, 0.021071667, -0.022813503, 0.0068659238, -0.04397895, 0.010917207, -0.040304862, -0.0133208735, -0.025708979, 0.033511493, 0.04166938, 0.00863123, 0.04210675, -0.016261185, 0.0015353571, 0.023082182, -0.019439653, 0.00015153938, 0.04334448, -0.010068235, -0.053117264, -0.061338, -0.012197182, -0.01749229, -0.04764707, -0.011218713, 0.05001317, -0.01038495, -0.031673133, 0.05267197, -0.059254862, -0.02730154, -0.02558281, -0.007912819, 0.012965607, -0.016566541, 0.0039767213, -0.0028908232, -0.016273206, 0.026449349, -0.03707255, 0.00093965133, 0.032958914, 0.02371616, -0.012147021, -0.038313773, -0.0054685366, 0.019114183, 0.02184151, -0.005683751, -0.021009987, 0.027079673, 0.08989151, 0.03559781, -0.036052383, -0.012413238, -0.0043793716, 0.00081079925, 0.0055798246, -0.021835553, -0.0035894795, -0.062021602, 0.030173952, -0.014439605, -0.010033415, 0.016104206, 0.011246751, 0.063678116, -0.029792469, 0.027399806, -0.0133722285, -0.020033013, 0.04249881, 0.01370271, -0.0157596, -0.03892745, -0.010428743, -0.040160347, 0.028949475, 0.0028166804, -0.029548509, -0.026592905, -0.004281466, -0.0022622133, 0.057235207, 0.0101088695, -0.0030310042, 0.037075464, 0.034190148, 0.010768546, -0.021555096, 0.027289897, 0.0037694094, 0.05330261, 0.035699908, 0.014253989, 0.0074354853, 0.038323388, -0.018929694, 0.06386258, 0.026359424, 0.029945113, -0.016560242, 0.0069727763, -0.019998873, -0.004531125, -0.017023008, 0.05456488, 0.038837317, 0.008919205, 0.016745528, -0.008708683, -0.012616728, 0.013596662, 0.03749979, -0.035127494, 0.0076414375, 0.05094869, 0.0671924, -0.023192132, 0.023643691, 0.008968847, -0.039007083, 0.0112441145, 0.021755807, 0.019274786, -0.034405738, 0.0487272, 0.06897014, 0.020477587, -0.0517936, 0.033337705, -0.08320619, 0.012396872, 0.014290769, -0.0021067036, -0.044410598, -0.04525684, 0.022605414, 0.03068902, -0.0066486774, -0.013402149, -0.016480925, 0.0072389585, 0.023489097, 0.012962911, 0.03555421, 0.019740492, -0.054508276, 0.02118935, -0.008661778, 0.027436778, 0.018454537, 0.041578837, 0.014735706, -0.015109264, 0.0021127595, -0.037568755, -0.016133338, 0.015698941, 0.030258698, 0.026200224, 0.005675422, -0.014594819, 0.01348324, 0.027764305, -0.03692898, 0.0056542396, -0.024686052, -0.004101461, -0.014622262, -0.02174525, 0.037558924, 0.06598448, -0.030963134, 0.0056536957, 0.0048306147, -0.03657678, 0.011952231, 0.010564498, -0.07405966, 0.06386917, -0.044404484, -0.011137522, -0.02917923, -0.032087896, -0.00005271527, -0.021333033, -0.008513036, 0.02107052, 0.004352805, -0.05760782, -0.005248675, -0.07925862, 0.0171257, 0.013840874, -0.03263469, -0.01021177, -0.021273358, 0.0010467488, -0.011445571, -0.041458566, 0.025728155, 0.038229145, -0.00855858, -0.016387176, -0.004638632, 0.0064900164, 0.030518629, 0.022850756, 0.036331438, 0.0052443845, 0.025525443, -0.024648989, 0.009839443, -0.0063192453, -0.0054786648, 0.033979353, 0.0016336485, -0.022960544, 0.008425605, 0.009300248, -0.01215645, -0.021217933, -0.0052932673, -0.007490714, -0.0034466817, 0.012593881, -0.06330971, 0.03829386, 0.00047739604, -0.04888001, 0.009818634, 0.018111382, -0.025683144, -0.022101248, 0.0037724867, 0.013912383, -0.036884323, -0.06427593, 0.0059910887, 0.024151681, 0.024552524, 0.00669153, -0.0040711607, -0.0042148703, 0.025309203, 0.00021060635, 0.00543275, -0.011008395, 0.017412292, 0.025637342, -0.05136464, -0.0076367515, -0.00797819, -0.007321414, 0.03142904, -0.04396082, -0.0062425383, -0.0020007163, -0.035971068, -0.034167342, 0.04925176, 0.0355126, -0.056937814, 0.021626072, -0.0009106937, 0.010376666, 0.04682122, 0.0022450883, -0.022295384, 0.036454722, 0.014172384, -0.032544523, 0.013954752, -0.007858724, -0.053158004, -0.0086816875, -0.0034609104, 0.034934353, -0.0039888793, 0.005968112, -0.00019169551, -0.0012400495, -0.15264222, -0.024053926, -0.030204678, -0.011574601, 0.0016655514, 0.005094437, -0.022105088, 0.027183594, -0.012981606, 0.018028077, 0.013210947, -0.037350774, 0.009221229, 0.038115688, -0.007969345, 0.016036836, 0.005142658, -0.008457639, -0.026416745, -0.043858975, -0.003096101, -0.046871454, 0.07235745, -0.02284802, -0.021273144, 0.00827089, 0.029086994, -0.026377724, -0.054404847, 0.01460115, 0.008110563, 0.03695299, -0.0011351447, 0.020986708, 0.023809198, 0.04562549, 0.026142161, -0.044854313, 0.03133466, 0.027229073, 0.017780758, 0.012361173, -0.031484563, 0.016054185, -0.059054866, -0.035040967, -0.012244274, 0.029693583, 0.018707208, -0.048559967, -0.02020758, -0.017346697, 0.0006251031, -0.030988397, -0.016233802, -0.0040621962, -0.06102356, 0.027009968, 0.0073421826, 0.010230838, -0.009472219, -0.09323822, 0.06850878, 0.027605092, 0.0003441266, -0.012208757, -0.014023225, -0.0037269855, 0.014461203, -0.0050596455, 0.024164781, -0.010079254, 0.006203158, -0.046139386, -0.0034068008, 0.028005484, -0.00695841, -0.013887995, -0.016887717, -0.09334614, 0.03214355, -0.017776387, -0.024077173, 0.0034673552, -0.004494756, -0.01099215, 0.03276744, -0.028054234, 0.03613917, 0.26086244, 0.021622656, 0.06769722, -0.035126768, 0.07008626, -0.019833442, -0.008852452, 0.0082648825, -0.043685462, -0.024750106, -0.022934971, 0.025197275, -0.00467053, 0.005800279, -0.041393373, 0.01963846, 0.013405522, 0.029560503, 0.055938285, -0.0039586634, 0.011575281, -0.024882315, 0.017247304, 0.021560183, 0.021565182, -0.066023394, 0.014288538, 0.048367307, 0.041431703, -0.005997279, -0.05215501, -0.002308309, -0.008713221, -0.028433574, 0.030962897, -0.004612465, -0.009943518, -0.015906924, -0.017563999, 0.034008663, -0.00033262096, -0.016283706, -0.00898734, -0.010933594, -0.008352996, 0.003271419, 0.029805357, -0.05342297, -0.021697097, 0.0059503987, 0.0138253085, 0.066825025, -0.034721144, -0.0099205645, 0.020406606, 0.0030097896, 0.013436045, -0.023130598, -0.0007262511, 0.038003296, 0.0060280417, 0.036769323, 0.0007262158, -0.018140122, -0.020079706, 0.011931135, -0.026742084, -0.036382068, 0.02483949, -0.0025942132, 0.04297294, 0.022911549, -0.011241219, 0.036228213, 0.036709856, 0.046927758, -0.038521294, 0.072019204, 0.004379965, -0.036246687, 0.014440913, -0.022035025, -0.056481466, 0.00506687, 0.014237077, -0.0118297115, 0.012182011, 0.027241329, 0.038109116, -0.0006967523, 0.033365153, 0.026770966, -0.044263598, -0.023387935, 0.004537966, -0.016161608, 0.010570735, -0.032018542, -0.026343772, 0.030901873, -0.03191958, 0.011991195, -0.0050720726, -0.020346005, 0.036498368, 0.007444314, -0.027469052, -0.011295934, -0.0034368471, 0.0030576033, 0.028069489, 0.012169565, 0.015715318, 0.03919961, 0.06573574, 0.033672683, 0.010954758, 0.0067543057, 0.02556753, 0.0025236907, -0.0662468, 0.00021581963, -0.010061518, 0.005513269, -0.046557933, -0.05933405, 0.045820825, 0.016324667, 0.00059400883, 0.03869053, -0.0027173292, -0.023473637, -0.015475774, 0.021504687, 0.0914852, -0.016064962, 0.040441487, 0.004007101, -0.00025823066, 0.03716774, -0.00501726, -0.006234236, -0.034194477, 0.025814427, 0.0018643726, 0.043014564, -0.0047320426, 0.0089288335, 0.003135884, 0.00070480246, 0.036145557, 0.015996687, 0.015205675, -0.047368303, -0.014081382, -0.027203687, -0.031408973, -0.014985951, 0.00071102043, -0.04924505, 0.0121240355, 0.041494705, 0.0038727785, 0.13604563, -0.04501138, -0.0024331238, -0.022695098, 0.023261182, 0.0018134568, -0.0536054, 0.012885157, -0.034064144, -0.03078462, -0.010922231, -0.015170957, 0.0011271131, -0.037411407, 0.04989744, -0.03447691, 0.024838576, 0.04189914, 0.023778113, -0.029960996, -0.011650752, -0.024532896, 0.0018576544, 0.025801659, -0.0046357177, 0.006664579, -0.016594939, 0.007921308, 0.06331615, -0.018019263, -0.006437924, 0.030279268, 0.040554408, 0.019139566, -0.03094257, -0.011925011, -0.04681507, -0.0290683, -0.05896417, -0.014891237, -0.038675766, -0.01865828, -0.027476719, -0.031474076, -0.013276866, 0.005452737, -0.032052193, -0.028481437, 0.015231331, -0.023764644, 0.011813077, -0.053050157, -0.02306295, 0.0452806, -0.028039645, -0.027313678, -0.010372108, -0.013761217, 0.039374124, 0.015000894, -0.011345402, 0.006798711, -0.050223812, 0.04749417, 0.009232353, 0.024425201, -0.006354116, 0.0035229945, -0.009436613, -0.017111152, 0.0018988609, 0.043659143, -0.014083623, -0.03494338, 0.04252309, 0.027461031, -0.035765097, 0.010045855, 0.0084831845, 0.044139177, -0.00030133128, -0.015784029, -0.017877001, 0.0044499785, 0.0012823165, 0.043412667, 0.007346751, -0.017788794, 0.0064370865, 0.038343117, -0.022648517, -0.043612763, 0.0043405164, -0.0044111633, -0.008314737, -0.062350303, 0.027528677, -0.00020011481, 0.035122033, -0.033212565, -0.01874096, -0.002864504, -0.022793068, 0.017605765, -0.03400213, -0.009030462, -0.0784863, -0.032075413, 0.005711692, -0.020145979, 0.007818699, 0.07901418, 0.017348478, -0.028818306, 0.0643839, 0.015284638, -0.036071505, -0.024427885, 0.028018126, -0.021768253, 0.003895899, -0.0064637326, -0.00456782, 0.00046348284, 0.045883738, -0.004320262, -0.007889056, 0.020701211, -0.06947695, -0.04258404, 0.026155993, -0.020519722, 0.0023154106, -0.02835029, 0.040738218, -0.021316586, -0.005065237, -0.012036124, 0.045676593, -0.02316414, -0.024542255, 0.0038132584, -0.016855322, -0.0075992798, -0.035075158, 0.034847725, -0.009649562, 0.012455873, -0.008820463, -0.0071380325, 0.019145321, 0.0204118, -0.011469355, 0.036943227, -0.012128939, -0.017930696, -0.005091544, 0.018467505, 0.005349742, -0.014838489, -0.027028603, -0.0011340927, 0.03491698, -0.006658464, -0.00483215, -0.014969222, -0.005223773, 0.0032676489, 0.002717295, -0.023816532, 0.017276485, 0.004483638, -0.012819961, 0.044659924, -0.014344743, -0.025069421, 0.005916097, 0.007910571, 0.06487606, 0.06207659, 0.011250674, 0.025062995, -0.056174874, -0.021657543, 0.020835498, 0.03487869, -0.005345426, 0.016410805, 0.039739862, -0.019994533, -0.05019813, 0.016594453, -0.04209061, -0.034628138, -0.060737368, 0.030559575, -0.032759685, -0.031190101, -0.0015149037, -0.021184126, -0.0003071471, 0.022521101, 0.041943565, -0.022578625, -0.03107361, 0.013565049, 0.013420621, 0.012036331, 0.043343905, 0.02857722, 0.0062402603, -0.051644508, 0.0262831, -0.010549774, -0.011371719, 0.017242536, 0.024430292, 0.033282783, -0.0070629525, 0.0057499097, -0.006155024, -0.016008878, -0.022163853, -0.007787336, 0.024928562, 0.021359388, 0.0613289, -0.024477728, -0.03261467, 0.028711462, -0.019642578, -0.15002751, 0.010766705, -0.029329019, -0.0003063603, -0.03329001, -0.003642709, -0.023780871, -0.015862387, 0.0038491923, -0.037152, -0.0066752546, 0.030474247, 0.00046633984, 0.010690377, 0.029723117, -0.034066956, -0.0075815734, -0.021106802, 0.019549927, 0.05437087, 0.017851911, 0.014916261, 0.0051592453, 0.014702559, 0.008286437, -0.022229552, 0.014926314, 0.008009354, -0.0410211, 0.020519681, 0.003488451, -0.096518055, 0.00074952136, 0.040615432, 0.035523232, -0.011383632, -0.011827184, -0.000082135746, 0.010279187, 0.023189418, 0.021481443, 0.0016477873, -0.004954323, -0.0008634509, -0.0035309885, 0.027179904, -0.01730176, 0.002896736, -0.037615184, -0.030493887, 0.010308606, -0.0039447346, 0.049055338, 0.091876425, -0.043562867, 0.027823132, 0.007838865, 0.009472811, 0.009860938, 0.017830534, 0.013894546, -0.03618261, -0.0066542327, -0.04834348, -0.03154872, -0.006312815, -0.04321677, 0.018698895, 0.013595002, 0.009032732, 0.0051352554, -0.034509722, -0.013558847, -0.052419104, 0.024413286, 0.012553652, 0.04008188, -0.0033687393, -0.05506409, -0.0067777433, 0.03677168, -0.013071536, -0.04505151, -0.030887172, 0.014970227, 0.013962321, -0.038933076, 0.007003616, -0.06590818, -0.010975942, -0.036433555, -0.045689236, -0.0053051626, 0.013567148, -0.05501911, -0.041272104, -0.019747922, -0.019269327, -0.012442926, -0.036326986, -0.026387468, -0.060579848, 0.021046564, -0.036332026, -0.00668541, -0.0035108044, -0.009635599, -0.0036765453, 0.0544955, 0.030038258, -0.021409517, -0.02363916, -0.056263994, -0.023989195, -0.007597134, -0.00758715, 0.036396798, 0.025843192, -0.0055271736, -0.020532373, -0.018320695, 0.0034960688, 0.03530723, -0.02622847, -0.022079749, 0.00063323864, 0.04233978, 0.01729823, -0.0055560744, 0.00851672, 0.036451973, -0.038805563, 0.036825582, 0.009842895, -0.0044880877, -0.03980767, 0.011090381, 0.032909434, -0.042442765, -0.034701973, -0.007009901, -0.00034409438, -0.0053733326, -0.020982489, -0.030486718, 0.017097164, -0.013422783, -0.024685897, -0.06391248, -0.03748386, 0.039241385, -0.0026564836, -0.008653286, 0.050428126, -0.019731121, 0.0006457255, 0.033813164, -0.024108056, 0.042588763, 0.008272119, 0.0030877443, 0.019942675, 0.019109357, -0.035090566, -0.042026576, -0.0046042763, -0.032235436, -0.01964907, -0.022568949, 0.0014768372, -0.012007249, 0.061378513, 0.0009869369, 0.005975816, 0.04588951, -0.0516081, 0.019859714, 0.05151476, 0.008058714, -0.010507304, 0.038593907, 0.057752375, 0.013877598, 0.013322162, -0.008894371, -0.0077994172, 0.04917151, 0.040360224, 0.02624213, -0.024696438, -0.023899632, 0.0023065456, -0.020652777, -0.02494288, 0.043131348, 0.029839352, -0.010502396, 0.040123153, -0.021386826, 0.0025831345, -0.007849951, -0.053164605, 0.006662216, -0.040635, -0.016844565, 0.024598803, -0.037537668, -0.036870707, -0.019300625, 0.027760252, 0.00784872, -0.041345607, 0.009223088, 0.016664773, 0.020059396, -0.016209466, -0.010985868, 0.014871042, 0.0019900678, 0.026657064, -0.036916263, 0.021741055, -0.041898143, 0.018356914, 0.0046703466, -0.0053469855, -0.0332441, -0.0068158424, -0.013142021, 0.0035823141, -0.05324037, 0.0049675796, -0.029564677, 0.050866902, 0.027875505, 0.0031162237, 0.08463319, 0.049700316, -0.018106962, 0.013008837, 0.010905596, -0.0238745, -0.026653048, 0.022847246} + testVectorDifferentBGEM3 := []float32{ // I like painting + -0.037674077, 0.009346571, -0.029451353, 0.036535654, 0.009323637, -0.028890455, 0.004592399, -0.007849684, 0.028009359, 0.015392329, 0.015216605, -0.010394424, -0.0069970675, 0.0033292773, -0.0077668657, 0.00019585519, 0.007938906, -0.0039769025, 0.05045314, -0.037485093, -0.014075023, -0.025665535, 0.01802216, -0.0058315047, -0.024017964, 0.029321233, 0.001483284, -0.026463442, -0.025405081, -0.004159032, 0.0030409726, 0.017734485, -0.0028618185, -0.030510807, -0.010970409, -0.05514468, -0.009071524, 0.0027872075, -0.008373793, 0.013596097, 0.0059142904, 0.0095031345, 0.04339037, -0.045079645, -0.007926477, -0.021852259, -0.011903788, -0.00081165775, -0.0066777975, -0.00048839103, -0.0018890809, 0.030334247, 0.05059955, -0.019304276, -0.023451349, 0.043934904, 0.0026228516, -0.0036762315, -0.043780725, -0.037744563, -0.0015647854, 0.03155115, -0.014793373, -0.006756561, 0.011076819, 0.080861054, -0.007338972, 0.028894836, -0.024085846, -0.03358318, 0.0009066821, -0.015446359, 0.0021108813, 0.001010948, -0.056650504, 0.0028032511, 0.005696127, -0.03434498, -0.029869044, 0.00061731046, 0.0806756, 0.0037674517, 0.023133915, 0.021497177, -0.0076438747, 0.022686666, 0.0048968946, 0.0036799437, -0.015789801, 0.008463397, -0.010747429, 0.033511367, 0.026841497, -0.014111741, -0.038758103, 0.017063254, -0.03077273, 0.036843088, -0.0022913218, 0.019661495, 0.044026367, 0.042963132, 0.0086434055, -0.020321168, 0.020670261, -0.00687922, 0.02029928, 0.055874493, 0.011036356, 0.0013205095, 0.06120187, -0.025503667, 0.026813086, -0.005843776, 0.050215047, -0.029515104, -0.011722272, -0.015982509, -0.018309657, 0.0012840979, -0.009555549, 0.061765354, 0.010839602, -0.033729594, -0.0043446124, -0.010557049, 0.06367697, 0.028471343, -0.012857171, 0.037760943, 0.0422894, 0.072536156, -0.043597013, 0.033267006, 0.035344522, -0.005274359, -0.010540478, 0.013895216, 0.003925247, -0.012107538, 0.032104537, 0.021615438, -0.014979754, -0.08591118, 0.04130088, -0.08364181, 0.034405388, -0.011880315, 0.03450299, -0.0053398735, -0.013739709, 0.02813928, 0.028475255, -0.004188942, 0.002365589, -0.0195753, -0.006762013, 0.0678227, -0.032206673, 0.017273804, 0.023018638, -0.06324025, -0.01438528, 0.01218343, 0.017490774, 0.001416502, 0.037885774, 0.028212178, -0.011900233, 0.011536854, -0.04122321, -0.004278015, -0.0029236749, 0.00500284, 0.022746256, 0.00817634, 0.044924147, 0.0014463979, 0.012193249, -0.035529997, -0.03798157, 0.008763305, -0.026443733, -0.02707098, -0.010402999, 0.024227317, 0.029274106, -0.031989433, -0.0011394164, 0.005561591, -0.014029944, -0.031709794, -0.01566467, -0.031800915, 0.046101768, -0.045196883, 0.009152823, -0.00004539347, 0.0046407203, -0.042403437, -0.011199483, 0.023727164, 0.022370944, 0.003487617, -0.026038654, 0.012483911, -0.076051846, -0.021453975, 0.033358026, -0.021223634, -0.011467809, -0.0285322, 0.017100386, -0.020640334, -0.03455, 0.035802033, 0.003871974, 0.007470693, -0.0060007307, 0.0053217886, 0.022786377, 0.007432347, 0.046314996, 0.025788499, -0.0060654352, 0.006431757, -0.0060703796, 0.008255538, -0.022016801, -0.02439932, -0.00181754, -0.0039752848, -0.0033288852, -0.0032327299, -0.0022162308, -0.042677972, 0.004916345, -0.006016656, -0.019870678, 0.0015050177, 0.024227992, -0.003407821, 0.032047756, -0.0008391831, -0.0077968063, 0.0034911034, -0.0008173814, -0.015643107, -0.003327483, -0.025371557, 0.032115124, 0.0133956745, -0.026034337, -0.0021569768, 0.006103138, 0.013821868, -0.009129153, -0.0156874, -0.00997035, 0.0070017832, 0.00031211623, -0.010622371, 0.005721649, 0.030888006, 0.012819047, -0.03548351, -0.03385123, 0.027543219, -0.00020410774, 0.0017952325, 0.01548388, -0.013288071, -0.018276209, -0.011171758, -0.016165396, 0.0021332903, -0.0038442835, -0.01043714, 0.00076854357, -0.0015411787, 0.04322659, -0.002808739, 0.017491851, 0.000052532843, -0.0075934613, 0.028631985, -0.022894492, 0.03204106, 0.015208864, -0.023944953, -0.025242122, 0.011590667, 0.029505163, 0.0031253984, -0.021129385, -0.011346019, -0.011409569, -0.15280491, -0.03297879, 0.009771399, 0.00046308403, 0.0051794024, -0.00086850236, -0.03459158, -0.0074978704, 0.021019958, -0.005803165, -0.014490819, -0.028399073, 0.006375064, 0.018954458, 0.011489114, 0.014642257, -0.016154122, -0.0027896275, -0.03135127, -0.020124072, -0.03297398, -0.03370805, 0.06592873, 0.00044315663, -0.037536338, 0.016075805, -0.000032153086, -0.03455102, -0.0513792, 0.0029248036, 0.03343953, 0.005697104, -0.010497256, 0.0136357, -0.03820227, 0.0323283, 0.01311272, 0.011281542, -0.0048240907, 0.0073121167, 0.013562925, 0.058805317, 0.007320431, 0.057514075, -0.050802533, -0.039393943, -0.040309634, 0.04685833, -0.0041075195, -0.027085682, -0.030060401, 0.0005224262, -0.011970699, 0.021723738, -0.015582879, -0.0047991564, -0.036687456, -0.0073388848, -0.015649952, 0.042691138, -0.032991763, -0.097086124, 0.035158288, 0.004354924, 0.017387552, 0.00480287, -0.012282153, -0.013850613, 0.027693568, -0.02104666, 0.024773516, -0.004513048, -0.0284496, -0.012812161, 0.017503139, 0.05180739, -0.008116539, -0.008112101, -0.010191732, -0.09362803, -0.0100425705, 0.009413772, -0.006373835, 0.0040121456, 0.00094911666, -0.04380943, -0.00047591046, -0.023446955, 0.031914964, 0.2743724, 0.034773264, 0.051095083, -0.034715515, 0.0662266, -0.030655935, 0.009558737, 0.008740152, -0.031928454, -0.0032858995, 0.022445183, 0.062245984, 0.000982565, 0.015555278, 0.0042635025, 0.048738074, -0.028447509, -0.0024718342, 0.072561555, -0.034781795, -0.022153938, -0.02193735, 0.037537042, -0.0029971076, -0.024443846, -0.0696635, -0.016828503, 0.056901358, 0.030345058, -0.018210907, -0.044126816, 0.029651226, 0.012510875, 0.0010909999, 0.010462829, -0.031490784, 0.0022713146, -0.056350205, 0.031480454, 0.03129296, -0.018918233, -0.02303922, -0.029843207, -0.03287791, 0.0074662045, -0.017589286, 0.061170477, -0.030579656, 0.019048678, -0.021876784, 0.018596567, 0.0264395, -0.039431427, -0.032520674, -0.0036362107, 0.006995213, -0.010323429, -0.028313775, -0.0034902224, 0.04360281, 0.0055665174, 0.009954792, -0.012995816, 0.0027277488, 0.017494362, 0.024023829, -0.013503118, -0.005068262, 0.05903699, -0.007891596, 0.04403722, -0.00081068795, -0.027289458, 0.010148775, 0.022260744, 0.08028562, 0.010259434, 0.041283067, -0.0019500094, -0.0136686275, -0.015405533, -0.018989403, -0.037512247, 0.0010165434, 0.006220941, -0.030573325, -0.027278526, 0.016423598, 0.01678819, -0.032388, 0.037607558, 0.010711203, -0.06396725, -0.024240011, -0.005424715, -0.05957465, -0.01692393, -0.06759216, -0.018491697, 0.062243313, -0.024089703, -0.010414736, 0.0092030335, -0.055741988, 0.029160082, -0.038426556, -0.024974052, 0.037855726, 0.002432921, 0.038914133, 0.0006830094, 0.022165898, 0.0012924576, 0.038076375, 0.048085626, 0.027893653, -0.0014441214, 0.030995958, 0.020293672, 0.004726762, -0.012628167, -0.00097296893, -0.006883955, 0.010825182, -0.049848374, -0.038063284, 0.028186457, 0.019705001, -0.008854609, 0.03805909, 0.001687697, -0.019543756, -0.03014691, 0.021243272, 0.09550447, 0.016122082, 0.053077765, 0.013053531, 0.021589436, 0.019155527, 0.0029450562, -0.01477235, -0.006622878, 0.005774838, 0.03432316, 0.047679704, -0.028664617, 0.018780414, -0.027696345, 0.009306984, 0.009659471, 0.046623986, 0.024466813, -0.03352612, 0.025800116, -0.05552648, -0.05347003, -0.015844066, 0.03033767, -0.0285422, -0.00052877254, 0.03299209, -0.010838805, 0.10567986, -0.0137791345, 0.016833179, -0.0107105225, 0.029699937, 0.0054516527, -0.047688954, 0.0055836984, -0.00033211012, -0.030929528, 0.032769877, -0.021533841, -0.018936228, -0.014871095, -0.0049952357, -0.015891453, -0.0021472538, 0.04081178, -0.012895689, 0.006411439, -0.021026732, 0.021478176, 0.00026927236, -0.0012296229, 0.0083160605, 0.025902925, -0.042881664, -0.064459726, 0.0761709, -0.023918875, -0.0012113224, 0.035952866, -0.006934208, 0.05109269, -0.04636267, 0.0053581465, -0.054300077, -0.024603806, -0.051693328, -0.014462865, -0.057023015, -0.008217879, -0.02929687, -0.0055966964, 0.022682333, -0.025251945, -0.02717093, -0.015189268, 0.016185652, -0.0057435697, 0.003218332, -0.02911054, -0.0026066827, 0.041429028, -0.034180943, -0.023598753, -0.026966391, 0.022107454, -0.008227516, 0.01200534, -0.028023448, 0.0010600715, -0.012003186, -0.0002897665, 0.013989048, -0.039869286, -0.009787648, -0.0053805537, -0.009248642, -0.044326693, 0.007798051, 0.07905495, -0.016877221, -0.03748058, 0.0290568, 0.024535531, 0.017346263, 0.022725958, -0.032499824, 0.0923442, 0.030174043, -0.011469038, -0.01808925, -0.045738112, 0.018727802, 0.057337075, 0.011128364, -0.011137907, -0.031109666, 0.028373457, 0.0011822949, -0.03703424, -0.008473708, 0.018516839, -0.032213476, -0.06376163, -0.022361789, 0.02199165, 0.05462048, -0.01106843, 0.0004228372, -0.008283935, -0.030796, 0.017589958, -0.06067104, 0.01644989, -0.07523976, -0.012842036, -0.0043906556, -0.0043189493, 0.0059554735, 0.1010175, -0.002873117, 0.023135558, 0.06805906, 0.03545234, -0.0124705555, -0.023064652, 0.0055329967, -0.02451978, 0.0011075857, -0.011912804, -0.009404419, 0.014482636, 0.03447239, 0.024254188, -0.019508848, 0.0065075513, -0.040383026, -0.039876435, 0.012338101, -0.013776622, -0.00030091006, -0.059412546, 0.057419628, 0.0064147385, 0.017692208, 0.0043474096, 0.011510838, -0.05343798, -0.014374755, 0.0043955785, -0.0772747, 0.015944429, -0.019060535, 0.003902627, -0.021571843, 0.018467195, -0.028041007, -0.004756734, 0.03896469, -0.015849296, -0.025203045, 0.011670661, -0.038314994, 0.025300547, -0.0012215493, 0.001364786, -0.0018386573, -0.02961639, -0.058413945, -0.03273155, 0.011940802, -0.0008521949, -0.008690411, -0.020149812, -0.005694279, 0.014109378, -0.008714525, -0.014721382, 0.026570555, -0.003542035, -0.023320574, 0.06657613, -0.052682646, 0.025264453, 0.013679165, -0.0076596034, 0.049479816, 0.063822225, 0.000109946726, 0.055931825, -0.058831837, -0.009589257, 0.023592679, 0.061451238, 0.011354577, 0.006771509, 0.03296432, -0.03829631, -0.058708224, 0.036338363, -0.02055793, -0.03077305, -0.042566177, 0.0103040645, -0.038186803, -0.023962729, 0.0014447025, 0.000958736, -0.009992435, 0.02170406, 0.02356006, 0.007250858, -0.049568318, 0.00026475577, -0.03255285, -0.04305614, 0.023812544, -0.0013063224, 0.028458796, -0.040605374, 0.016455086, -0.047775637, -0.03404558, 0.0026435526, -0.007833649, 0.005782471, 0.013826437, -0.011137649, 0.009727315, -0.012547819, -0.025679946, -0.031833705, 0.009345544, 0.009332183, 0.017070528, -0.054728262, -0.054876197, 0.015811617, -0.020427948, -0.15267517, 0.022585044, -0.021628605, -0.029518625, -0.019710025, -0.001937381, -0.07692051, -0.010931078, 0.012356616, -0.021608042, 0.0035224769, 0.05416775, -0.050979163, -0.022253266, 0.028379498, 0.015596006, -0.0055963895, -0.0011299074, 0.0036824425, 0.089928165, 0.036891, 0.005236365, 0.013372251, 0.021125128, -0.019420674, -0.0057387874, 0.012644277, 0.038198367, -0.03533805, 0.0008064248, 0.0049848547, -0.083674006, 0.008703779, 0.013284692, 0.015219609, -0.0029713034, 0.0017884498, -0.007419442, 0.017519457, 0.0037970298, 0.023406668, 0.007957575, -0.04765493, -0.0014334822, 0.008360809, 0.01261427, -0.029990165, 0.022149375, -0.01434688, -0.044660788, 0.03227033, -0.034243315, 0.00063945825, 0.05548174, -0.040742964, 0.017645322, 0.00008426075, 0.036736194, 0.011405372, 0.026939923, -0.016635269, -0.025056742, -0.023125878, -0.037924103, -0.06878921, 0.0021640018, -0.039899394, -0.019005788, 0.038674243, 0.0002062139, -0.020224243, -0.01850297, 0.0027313635, -0.03788336, 0.03926915, 0.0370044, -0.0036955115, 0.00040715348, -0.035195477, 0.015176873, 0.017679857, 0.0074789873, -0.013460666, -0.05968425, 0.03440194, 0.029698743, -0.043776996, -0.015603201, -0.05131824, 0.009342118, -0.0035762463, -0.05030915, -0.0010477792, 0.016619097, -0.03369071, -0.0039239284, 0.00780037, 0.013822729, -0.019014375, -0.025802318, -0.05598404, -0.03324148, -0.0068113324, -0.04459597, -0.01936866, 0.011801866, 0.016960166, 0.01829125, 0.08012454, 0.01684296, -0.031602196, 0.02193943, -0.032308854, 0.023994997, -0.02863876, -0.020962832, 0.07066773, 0.028867712, -0.00014686643, 0.009671903, -0.012318323, -0.0008491809, 0.027866838, -0.03848373, 0.016047483, 0.005869677, 0.018587254, -0.011186068, 0.026320016, 0.034275584, 0.020231614, -0.037422895, 0.004727748, 0.042026658, -0.010776407, -0.0031467027, -0.037557624, 0.027259812, -0.016659804, -0.019334162, 0.007963539, 0.006246179, -0.022235997, -0.023059828, -0.050840344, -0.001115175, 0.019079436, 0.018931277, -0.035058755, -0.0122408625, 0.05174501, -0.020457132, 0.033526186, -0.0012395711, -0.0015534105, -0.0021828755, 0.025715187, -0.004308088, 0.04480646, 0.0019377702, 0.030454338, 0.002528406, -0.0010083686, -0.0281393, -0.034377333, -0.0347005, -0.022989867, 0.0026928913, -0.008780569, 0.0068237367, -0.013573793, 0.06233938, -0.004475617, -0.018968308, 0.026943473, -0.05731867, 0.0052365307, 0.06161817, 0.003159209, -0.015051774, 0.033898313, 0.026367128, -0.013441867, 0.011147675, 0.014699197, 0.02182933, 0.010168702, 0.047309816, 0.0062500685, -0.0248315, -0.01733879, -0.035497952, -0.024899052, -0.04227799, 0.025216274, 0.035840075, -0.008902616, 0.044990752, -0.0068282383, 0.045401663, -0.008756044, -0.013725104, 0.0019296168, -0.034249905, -0.010393843, 0.021963503, -0.014520636, -0.020559274, -0.035184413, 0.040994186, -0.04003199, -0.041267034, -0.022200605, 0.074485675, 0.029953893, 0.012549354, -0.007818248, 0.03685385, 0.03540409, 0.054592233, -0.02761211, 0.009018615, -0.02361103, 0.018474447, 0.012663053, -0.01695293, -0.056983404, -0.013685747, -0.008904628, -0.0027191464, -0.04545175, 0.067227356, -0.0031729785, 0.014436117, 0.020977499, -0.02551583, 0.029027019, 0.024583159, 0.015928457, 0.0055533224, 0.008508477, -0.030893689, -0.028556395, 0.016821753} + + + + + inputVectorGemma300mPrefix := []float32{ // title: none | text: I like soccer + -0.072284766, 0.005397726, 0.0016333675, -0.002900052, -0.010216651, -0.011110686, -0.024665391, 0.09440253, 0.02099492, -0.07621014, -0.053849906, -0.0042059966, 0.030171992, -0.020546654, 0.061806537, -0.010382888, -0.036617175, -0.0068241847, -0.069961354, -0.046741866, 0.019712357, 0.0031614345, -0.0110266395, 0.0073370067, -0.002229468, 0.0054874644, 0.007878278, 0.036105588, 0.053043913, 0.004026623, -0.01111258, -0.019559927, -0.003536538, 0.018815657, -0.018633734, -0.007173371, 0.012583121, -0.060179796, -0.025464926, 0.006711466, -0.06995896, 0.08622101, 0.0037625565, 0.049469423, -0.037021104, -0.03404436, -0.040957376, -0.012207768, 0.0010006137, 0.02593832, -0.011961684, 0.017999124, -0.032288123, 0.046467375, -0.04072433, 0.00822678, -0.038654424, -0.0005636667, -0.012616981, 0.043700878, -0.039935783, -0.0139832515, -0.013264154, -0.023802634, 0.05613767, 0.021834651, 0.0218579, -0.029703151, 0.05201489, 0.25084692, -0.002786525, -0.013651692, -0.024266085, -0.01959919, 0.13597038, 0.043680746, -0.02780931, -0.017122936, -0.033688754, -0.005587911, 0.015591087, 0.05000633, -0.0069926437, -0.03624385, 0.07559606, -0.020228134, -0.008222282, -0.03626746, -0.01902105, -0.04963517, -0.013179433, -0.0073154545, 0.0054324362, 0.0101555595, 0.009972455, 0.00898101, -0.058859773, 0.008622074, -0.003611674, -0.019902172, -0.052971657, 0.037429605, 0.0736365, 0.03660234, -0.03298037, -0.029579647, -0.02719295, 0.015238084, 0.025577832, 0.017799767, 0.024772268, -0.093578406, 0.033095114, -0.06691459, 0.029606199, 0.019289985, 0.00012162749, -0.009077495, 0.0081599215, 0.010422333, 0.02097784, -0.022489045, -0.0077649644, -0.03110919, -0.009205159, 0.035468522, -0.018413996, -0.03375395, 0.040162116, 0.01474268, -0.005173999, 0.03407355, 0.046035457, 0.038916785, -0.0034115366, 0.0703272, -0.04663181, 0.012667228, 0.13247076, 0.042693835, 0.033974513, -0.14347161, -0.022984711, -0.067442544, 0.06623006, 0.028203124, -0.02628976, 0.03402213, -0.011563471, -0.014448655, 0.022927253, 0.036175773, 0.026604658, 0.002336636, -0.021162534, 0.010154343, -0.00764248, 0.018014582, 0.010841352, 0.010476933, -0.021572541, 0.018590132, 0.013387604, 0.055584863, 0.054679304, 0.035661776, 0.052819427, 0.051160727, 0.045018345, 0.032707456, 0.0059863757, -0.061924208, -0.018830707, -0.00052863755, -0.024133276, 0.06157117, -0.034045536, 0.0125173265, 0.1269633, 0.029214201, -0.014637991, 0.0070294826, -0.027827028, 0.017086066, -0.021882145, 0.03760306, 0.0015065481, 0.036514655, 0.0096789235, -0.00817127, 0.016416669, -0.0076154447, 0.011716529, 0.008847849, -0.0022396909, 0.053489253, 0.07005467, -0.007456973, 0.01321893, -0.006405146, -0.021295466, -0.022866474, -0.019756684, 0.03395988, -0.044514835, -0.028554006, -0.016685896, -0.018754048, -0.060571387, -0.025768133, 0.013781783, 0.012229682, -0.029781777, -0.013556484, 0.01099999, -0.023659859, 0.028317204, 0.008381091, 0.008929566, 0.0015608984, -0.053944606, 0.01351252, -0.031112814, 0.003084642, -0.046577815, 0.0069277426, -0.0012321051, -0.040866304, 0.029663036, -0.035606842, 0.021912646, 0.031432953, 0.027957803, -0.0020429024, -0.06237694, -0.019616436, -0.011371905, 0.032032404, 0.026098056, 0.072211064, -0.059172574, 0.020070609, 0.01406972, 0.023169423, -0.00945928, -0.045108467, 0.0018382497, -0.054710194, 0.0011813014, -0.04916152, -0.0016130666, 0.014323211, -0.031356234, -0.024599057, -0.016072037, -0.012878102, -0.0043782713, 0.013182045, -0.04245869, 0.02841295, -0.0106321275, -0.0010273516, -0.019319082, -0.010860112, 0.022415726, 0.013790313, 0.015146331, 0.008879473, -0.024204234, -0.013135595, 0.014587337, -0.02912569, -0.040583808, -0.03049026, -0.004837429, 0.021267066, 0.04279787, -0.01960013, -0.009249379, 0.044974446, 0.050498832, -0.014094448, -0.010754026, 0.061519805, -0.00033583524, -0.009716352, -0.042534873, -0.03128052, -0.027040806, -0.06090959, 0.014590321, -0.0009949628, 0.058484815, -0.049158387, 0.030476857, -0.037420917, -0.15039694, -0.04384937, -0.013042085, 0.02161206, -0.12930444, -0.003747923, -0.015213724, -0.053928036, -0.020495696, 0.02187154, -0.015368578, 0.022176228, -0.051926855, 0.012809839, -0.047222096, -0.019404309, -0.03822206, 0.003724424, 0.044096664, 0.027713746, 0.029540738, 0.023287484, 0.054527685, 0.0060168784, -0.013428098, 0.03169096, 0.042842932, -0.02183568, -0.0024914595, 0.055329, 0.029355811, -0.04904809, -0.07098646, 0.040031347, -0.00469701, 0.01816061, 0.09132025, 0.027602378, 0.04337985, 0.05484917, -0.04298314, 0.012052091, -0.0062593194, -0.03142127, 0.011274735, -0.027307808, 0.010171231, -0.010075596, -0.0006797919, -0.017177345, -0.042027976, -0.0035955182, -0.031174835, 0.032402758, 0.042994954, 0.037828833, -0.0019034034, 0.049786787, -0.038595334, -0.0110225305, 0.0023183194, 0.016054414, 0.04404595, 0.026203219, -0.0006999488, -0.047237165, 0.051212896, 0.03561234, -0.056536004, 0.0042668483, -0.05970605, -0.016679425, 0.021685688, 0.037358955, 0.0016230037, -0.01913361, -0.0272429, -0.022414077, -0.06650035, -0.011646668, -0.009158446, 0.03255755, 0.015386274, 0.009527367, -0.042531587, -0.005613872, -0.016582608, -0.013804632, 0.022138748, 0.012239996, 0.0016413439, 0.058129363, -0.0035809795, 0.09421854, -0.030636724, 0.03495081, 0.034986563, -0.044736348, 0.013624264, -0.0041589346, -0.034072477, -0.020836039, 0.03426133, -0.058233116, -0.024573103, 0.03910751, -0.075991675, 0.0062675085, 0.015297746, -0.008293413, -0.005432822, -0.008332575, -0.042982075, 0.012329446, 0.013564389, 0.013701471, 0.0040457426, 0.07736077, 0.03924589, -0.085962914, -0.00425693, -0.02802651, -0.15478018, 0.013514597, -0.016974498, -0.034629036, 0.03390037, -0.0046992665, 0.0051871007, 0.044533793, 0.0036096426, -0.015717413, 0.050722267, 0.036621418, -0.012230926, -0.031356927, 0.020454483, -0.0038493355, 0.024531722, -0.012546699, -0.036040485, 0.03032287, 0.030646529, 0.04341025, -0.022707008, -0.06866389, 0.00888875, 0.007191607, -0.019313406, 0.03296604, -0.019559411, -0.0025313946, -0.017540175, -0.002979248, -0.0037479827, -0.06842207, 0.03564193, -0.0031696337, 0.017756358, -0.0009869939, 0.0030793387, 0.058801156, 0.029780732, 0.009602143, -0.0063307504, -0.018902933, 0.0026101607, -0.017000744, -0.02311985, -0.0062381322, -0.013603529, -0.027017748, -0.031383604, 0.037041362, 0.04602502, 0.011352354, 0.047884673, -0.026300438, 0.045155182, 0.01288915, -0.008418911, 0.03001358, 0.015858406, 0.0071731173, -0.020072667, -0.018648395, -0.028237995, -0.0137038175, 0.048680283, -0.017113801, 0.015367243, 0.0090566855, 0.021611014, -0.029379902, -0.061030526, -0.01091203, -0.034951407, 0.047173027, 0.043550886, 0.032296713, 0.0062056254, 0.0036735819, -0.06206353, -0.047221333, -0.018505912, 0.01979309, -0.025654169, 0.019666644, 0.049033854, -0.0178469, -0.00092785194, -0.048412576, 0.02037828, 0.0006777725, -0.018262269, 0.019616265, 0.031902812, 0.027776254, -0.033256143, -0.0033425316, 0.002452462, 0.017269192, -0.001331677, 0.035370324, 0.023529826, -0.027023835, -0.0019422731, 0.01686316, 0.024747534, -0.02861206, 0.020708285, 0.0040620835, -0.024339002, 0.008159489, -0.092583306, -0.009389744, -0.037532486, -0.057234652, 0.0193786, -0.06082522, 0.010143947, 0.007089006, -0.02694246, -0.0600224, 0.012934416, -0.039231, -0.022689687, 0.019035695, 0.010050569, -0.010076737, 0.040766153, -0.021353893, -0.027582524, 0.039429665, 0.015433775, 0.033543177, -0.06096551, 0.030208942, 0.05708312, 0.0043151057, -0.011369811, -0.017435228, 0.012296551, 0.0044090613, 0.009212026, -0.04258935, -0.005864361, -0.016122846, -0.022742487, 0.010513342, -0.05406144, -0.04218015, 0.0071235234, 0.0066425386, -0.008323658, 0.04880645, 0.032641415, -0.030716522, -0.043205656, 0.079466276, 0.012698363, -0.042833045, -0.04462951, 0.01264992, 0.034873568, 0.030791353, -0.008116336, 0.034464277, 0.048382267, -0.0036712692, -0.0058463933, -0.034576904, -0.0653417, 0.05049239, 0.013482879, 0.015821941, 0.03606656, 0.010675378, 0.020038849, -0.027075632, -0.054302324, -0.019070078, 0.06247976, -0.024341233, 0.009714004, 0.016535643, 0.046586998, -0.008847124, 0.032899078, 0.039578214, -0.021757266, 0.016223144, -0.031827796, 0.04106485, -0.014500338, -0.014612362, -0.002299456, -0.07494921, -0.019476723, -0.0024711646, 0.008381048, 0.025691727, -0.021136649, 0.0056644054, 0.004344846, 0.013763315, -0.021784319, 0.009127909, 0.03905124, -0.023906874, 0.014203325, 0.00071766775, -0.05625335, 0.033227023, 0.023766758, 0.08979646, 0.026545404, 0.0135549465, -0.0045313337, -0.0010333918, 0.013692505, 0.021435617, -0.007019721, -0.032354616, 0.034956798, 0.021160137, -0.006305416, 0.027432308, 0.013372831, -0.01667336, -0.027916854, -0.007805913, -0.015097294, -0.026559934, -0.04231311, -0.033404276, -0.0021483428, 0.027153645, -0.019514801, 0.030235806, 0.023480268, 0.025592398, -0.043547474, -0.015497528, 0.030951712, -0.024947437, -0.032029968, 0.020475345, 0.051063683, 0.00017679234, 0.016830552, 0.026091257, -0.002926624, -0.028071523, -0.03528893, -0.026590765, 0.015647711, 0.032690685, 0.015433953, -0.03943609, 0.032504868, -0.02342466, -0.029374091, 0.021810967, -0.03249957, 0.04177423, -0.021834299, 0.01007888, 0.024028389, 0.02578589, 0.010398874, -0.04158715, 0.053374745, -0.0028731334, 0.010537261, -0.11361711, 0.018209042, -0.017313527, 0.0027602545, -0.009126336, 0.016062936, -0.027590962, -0.024217375, 0.00774635, -0.0078028007, 0.003247252, -0.015638186, 0.00204718, 0.031247405, 0.007114898, -0.0030919686, 0.03793813, -0.010029599, 0.028899884, -0.08231681, 0.019399555, 0.0026561068, 0.0026528642, 0.040102288, -0.029615173, -0.059408452, 0.009355349, -0.016022373, -0.017495425, -0.0025302814, 0.016439572, 0.05782867, -0.003485428, 0.0367805, -0.006948833, 0.011108409, -0.04917792, 0.047369286, -0.014502422, 0.018015556, 0.031097418, -0.049066756, -0.03154016, 0.04231999, 0.021500124, 0.05503283, 0.015905496, -0.021541344, -0.021336347, -0.04013308, 0.009024654, -0.054575242, -0.041646212, 0.03991168, -0.025831584, -0.06204724, -0.030437011, -0.02147342, -0.050442055, -0.0066135162, 0.04340114, -0.0077996626, 0.118610844, -0.045114797, -0.014646492, 0.0012353956, -0.041998982, -0.04404865, 0.0071968515, 0.04302333, 0.016726015, 0.012614212, 0.032650072, -0.01558332, -0.016620476, -0.030566191, 0.007257365} + testVectorSimilarGemma300mPrefix := []float32{ // task: search result | query: I love sports + -0.176718, 0.0065655434, -0.015737727, -0.048986603, -0.054769356, -0.04374399, 0.0018836671, 0.081751555, -0.00081376376, -0.088127315, -0.025964178, -0.04483953, 0.066195145, -0.023677047, 0.04427913, 0.041875344, -0.029031271, -0.024147546, -0.03782654, -0.037342835, -0.020339787, 0.044593476, -0.05606499, -0.013419152, 0.020975154, 0.0024571046, -0.0055804034, 0.006326032, 0.06655453, -0.018773345, 0.033111155, -0.022212232, -0.029048895, 0.0067018233, 0.014807174, -0.0093465, 0.023769388, -0.022329194, -0.031225536, -0.007523339, -0.123145916, 0.08295743, 0.035762742, 0.015470844, -0.043784846, -0.0027514559, -0.026110543, -0.032118093, -0.01997905, -0.0028206382, 0.02265011, 0.008122826, -0.067956224, 0.0049479157, -0.05672399, -0.028580872, -0.012890688, 0.015290787, 0.030399784, 0.048901383, -0.050761566, 0.008657538, -0.007766293, -0.013096534, 0.013134911, -0.019737089, 0.03410056, -0.00018533367, 0.043890454, 0.07396614, -0.00023029298, 0.022837212, -0.026860923, 0.023894154, 0.098722935, -0.03168921, -0.0013996541, -0.019363008, -0.021898529, -0.0020707834, 0.05328412, -0.021855382, -0.029934067, -0.0593233, 0.07472449, -0.056568157, 0.0010989752, -0.052167013, 0.013247874, -0.037793126, -0.0016169249, 0.0180641, -0.0004073748, -0.024257833, 0.024993, -0.050964475, -0.0059833396, 0.07752758, 0.008227539, -0.025643561, -0.049450293, 0.04587913, 0.05136871, 0.13580774, -0.019055972, -0.02762488, -0.038943958, 0.019996542, 0.039255448, 0.029075855, 0.042878952, -0.09742263, 0.0530866, -0.08815048, 0.02495291, 0.026617542, -0.02524035, 0.012605309, -0.0011400783, 0.0059177927, -0.0030578761, -0.0058284756, -0.02357878, 0.0032501107, 0.0092017865, 0.018531062, -0.010747542, -0.027513193, 0.014294621, 0.012589564, -0.022845475, 0.018128864, 0.020168021, 0.04920988, -0.015140657, 0.027422782, -0.020437172, -0.000016919214, -0.020239394, 0.03538468, -0.022025542, -0.1615484, -0.047489543, -0.04449348, 0.028073419, 0.02791197, 0.01220118, 0.015263171, 0.0064840987, -0.010525848, 0.021311313, 0.06282545, 0.03503889, 0.008540422, -0.039570145, -0.0043557943, -0.021817543, 0.030650264, -0.030232478, -0.015515937, 0.017461844, 0.05513887, -0.035910208, 0.07736192, 0.02621988, -0.04672246, 0.012266893, 0.018626928, 0.026567249, 0.017961549, -0.018576968, -0.011873052, 0.009892483, -0.025851037, -0.016370323, 0.00621244, -0.00025602194, 0.024342733, 0.10677995, -0.0012438333, 0.0064653363, -0.04209072, -0.026827887, 0.06752096, -0.030541636, -0.022152737, 0.009508021, 0.028904704, 0.021878064, -0.037774276, -0.0032937503, -0.013315477, 0.012806136, -0.00041137091, 0.011955586, -0.046606876, 0.070068516, -0.014335653, 0.05536958, -0.018839095, 0.006466499, -0.023879426, -0.017971551, 0.035843488, -0.0202636, -0.035901405, 0.0006983777, -0.027969304, -0.013448626, -0.0065529556, 0.042406578, -0.008270777, -0.039995953, -0.0036624968, 0.022070903, -0.011930705, -0.016115416, -0.0154709695, 0.023655524, 0.02365733, -0.006511068, 0.0029809435, -0.024231937, -0.019486006, -0.06019195, -0.0013960983, 0.022836875, 0.0029984852, -0.021975668, -0.0699394, -0.0024478813, 0.014956977, 0.024472002, -0.06100239, -0.01654814, -0.026883869, -0.0016140054, 0.047862004, -0.012284152, 0.049177635, -0.11209584, 0.045424975, 0.0009039695, -0.046402246, -0.0431078, -0.065898664, 0.0279793, -0.056239314, 0.010088176, -0.06855808, 0.018823033, 0.0115013225, -0.009620737, -0.036559768, -0.052202996, -0.022910621, 0.022708844, 0.022776565, -0.038631853, -0.00579741, 0.015284622, 0.023108605, -0.039870378, -0.06217679, 0.022074208, 0.027154591, -0.010558774, -0.0028227428, -0.016905, 0.009420798, 0.014546596, -0.028018964, -0.02974609, 0.009442417, -0.005438459, 0.038147993, 0.004017, -0.04040984, -0.03688898, 0.020789677, 0.025538977, -0.0009872088, 0.03326257, 0.06006643, -0.00538234, 0.030560061, 0.008617551, -0.03914284, 0.0019224349, -0.04575508, -0.013636969, -0.010104723, 0.037055656, -0.00073556526, 0.035868227, -0.011962299, 0.03308808, -0.036633074, -0.000920722, 0.009247516, -0.0525219, 0.024370283, -0.009990005, -0.041956507, -0.025607888, 0.01632027, 0.0030687305, 0.02673634, -0.04623248, -0.035156902, -0.061402828, -0.0054163956, -0.034762383, 0.025257958, 0.018768094, 0.006253634, 0.04611249, 0.02030183, 0.07886525, 0.010674326, 0.0056098104, 0.046078287, 0.019000689, 0.0038386802, -0.0139676025, 0.018592808, 0.009586513, -0.043284144, -0.028789792, 0.012181708, -0.049577627, 0.05403041, 0.037255175, 0.058859255, 0.057114583, 0.05737245, -0.06648968, 0.018706575, 0.010414232, -0.0076667424, 0.024332723, -0.017212698, -0.012398043, -0.01818305, 0.058075137, -0.036814783, -0.008512903, -0.014932418, 0.0054251077, 0.035470575, 0.0395071, 0.045574468, -0.0022547871, 0.06258979, -0.10549167, 0.05863539, 0.043690424, -0.017549887, 0.012972374, 0.013715516, -0.013908213, -0.04689973, 0.056950506, 0.0142704295, -0.035592355, -0.04522767, -0.0332895, 0.0026995316, 0.044672918, 0.049996786, -0.0028529174, -0.050038915, -0.049257465, -0.032729156, -0.068786226, -0.015675489, 0.020257924, 0.047794025, 0.028507309, -0.019473903, 0.011814735, -0.008566301, -0.015672794, -0.039567363, -0.0105840955, -0.037629392, 0.017670458, 0.055949934, 0.0049481154, 0.0543457, -0.00079134974, 0.023657886, 0.048624933, 0.010029676, -0.0027276082, -0.026320169, -0.04701809, 0.0020421806, 0.046256732, -0.03111779, -0.016974984, 0.026446585, 0.10549835, -0.0010075498, 0.043151964, -0.043466933, 0.009499259, 0.019754667, -0.041408654, 0.010006955, -0.013988053, 0.016418334, -0.015966885, 0.052326523, -0.005808767, -0.049685664, -0.006109466, 0.0063831713, -0.018126061, 0.0036449416, -0.03564184, 0.0029492383, 0.04584059, 0.03531469, 0.0163892, 0.093858026, 0.008696214, 0.007326514, 0.05386455, 0.00921647, 0.0023321914, 0.030257264, -0.03799143, -0.021756783, 0.0045373146, -0.0014294286, 0.037511162, 0.025529768, 0.01540663, 0.082479194, -0.02579298, -0.052758258, 0.014651258, 0.023577958, -0.02251307, 0.030488914, -0.046791382, 0.015067761, -0.02643415, 0.023049025, 0.0008772244, -0.06931479, 0.0285824, -0.0055758874, 0.020385956, -0.0059479643, -0.013777694, 0.061961867, 0.04598737, 0.006877812, 0.0036917552, -0.003882717, 0.021532597, 0.019363705, -0.042121302, 0.022405514, -0.028362842, -0.0109715555, -0.057379927, 0.035553385, -0.008347211, 0.0007431413, 0.03794017, 0.018645355, 0.023176076, 0.01264848, 0.0036555983, 0.030446881, 0.016846592, 0.0042680525, 0.026614223, -0.08606476, 0.02741376, 0.003578117, 0.00039739136, -0.028842697, 0.013635664, 0.014880986, 0.0062592933, -0.050044652, -0.08512495, -0.0251497, -0.010666905, -0.007884419, 0.072685204, 0.02465883, 0.022087116, -0.02060413, -0.1364186, -0.06193707, -0.012055116, 0.028642828, -0.03333006, 0.019119566, 0.045947652, -0.033358313, -0.004992471, -0.054865535, -0.011275105, 0.011983987, 0.007904222, 0.040070496, 0.051174086, 0.021799842, -0.061672464, -0.014848559, -0.02557525, -0.0026141054, 0.023153132, 0.054590877, 0.009181387, -0.013919926, -0.02462138, 0.013758518, -0.04456174, -0.005230621, 0.0051681525, -0.050102107, -0.021413833, -0.01191594, -0.050366964, -0.004386881, -0.015450691, 0.005047555, -0.02260149, -0.020067181, 0.001794196, 0.025516054, -0.034655392, -0.043124497, 0.0019795815, -0.037999067, -0.006490993, -0.007716246, 0.032204963, -0.030888416, 0.05219907, -0.009972949, -0.025824614, 0.029849008, 0.012839344, 0.021904234, -0.015438286, 0.025721299, 0.03067137, -0.0049393056, -0.029385362, -0.04646023, -0.018169448, 0.03074543, 0.0029383143, -0.043196335, -0.004875013, -0.012622653, -0.016219947, 0.054196987, -0.05621703, -0.042509478, 0.0014618393, 0.01426759, 0.007653192, 0.03981336, 0.0047457195, -0.041471776, 0.0040217135, 0.033136297, -0.018681377, -0.058768604, 0.0027230647, 0.061733566, 0.018811936, 0.030499978, -0.0108649265, 0.02335807, 0.0034723878, 0.012546805, -0.006847814, -0.061269663, -0.04363628, 0.00030666593, 0.038399197, -0.014401138, 0.008910173, 0.014058986, 0.0059576617, -0.014405264, -0.02096785, 0.010094083, -0.002117777, -0.035088953, -0.01277657, 0.004121387, 0.022173408, 0.012622781, 0.028040227, 0.0291176, -0.058555517, 0.037597172, -0.040259156, 0.018966658, -0.007135057, -0.02522446, 0.014763033, -0.056830347, -0.0054551745, 0.019256212, -0.02072686, 0.03018062, -0.008771436, -0.004551044, 0.009920699, 0.0048934883, 0.014717028, -0.011439645, 0.03775917, -0.00052603363, 0.0197455, -0.0055617834, -0.035737738, -0.1084198, 0.032486938, 0.053220917, 0.0051782094, 0.011079555, -0.018979354, 0.0040722517, 0.0056972806, -0.015480677, -0.041099332, -0.012772368, 0.032617703, -0.015456295, 0.06860022, -0.0007496268, -0.007850606, -0.010675925, -0.04188396, 0.01469766, 0.014930733, -0.018970024, -0.035494644, 0.002375829, 0.00924593, 0.0795682, -0.029182058, 0.061998405, -0.026893362, 0.026750809, -0.0386699, -0.061214533, 0.006293158, -0.077185504, -0.030334698, 0.06493838, 0.042881012, -0.027823113, -0.016940868, 0.0010850414, -0.0030023253, -0.0075366274, -0.049766712, -0.014976757, 0.046154663, 0.051074523, -0.0000465749, -0.033912018, -0.006995263, -0.042782113, -0.027253663, -0.0032384046, 0.025676876, -0.0038147483, 0.027950704, 0.049071502, 0.0059678475, 0.008998526, -0.01168227, 0.029302694, 0.020630367, -0.019609842, -0.0075584184, -0.12022105, 0.040495954, -0.016619852, 0.057048168, -0.026719945, 0.006425978, -0.014031886, -0.010866478, 0.039579734, -0.04981894, 0.012557895, -0.015555034, -0.035831653, 0.04323009, -0.017334845, -0.012518164, 0.07984875, 0.00018331647, 0.028933983, -0.03509336, -0.024097666, 0.035236795, 0.012592056, 0.04157066, 0.020756386, -0.076989956, -0.010215457, -0.018521475, -0.03937668, -0.047273185, 0.0063751633, 0.01557958, 0.006014699, -0.040826127, 0.0078228945, 0.03406307, 0.0017497061, 0.10087535, -0.03250235, 0.08505852, 0.030628871, -0.054890104, 0.00184642, 0.04964784, 0.015669048, 0.003029117, 0.03232055, -0.014907394, -0.07895684, -0.027249174, 0.037405, 0.005754665, -0.03241643, 0.013659842, 0.017014459, -0.026902376, -0.009280394, -0.037311148, -0.013591536, -0.00666238, 0.020981917, 0.0013518549, 0.012978187, 0.0043681087, 0.024549747, -0.02540837, -0.054034684, -0.058066078, 0.029647382, 0.028120603, 0.013298538, 0.013913252, 0.010167018, 0.012072977, -0.014260227, -0.011998889, 0.038905412} + testVectorDifferentGemma300mPrefix := []float32{ // task: search result | query: I like painting + -0.189905, -0.054302856, 0.020831248, 0.02775958, -0.08679674, 0.0030621062, -0.012906878, -0.0009512873, 0.012469478, -0.037452605, -0.019134713, -0.0030185683, 0.023155924, 0.010810449, 0.013640103, 0.003952236, -0.0067969835, -0.032944046, 0.02320807, -0.07051331, -0.016641084, 0.045248136, -0.032239623, -0.02370519, 0.0044423463, 0.046215355, 0.015320562, 0.01368472, 0.046803523, -0.016666837, 0.048009373, 0.014396208, 0.026051957, 0.018257601, -0.018995646, 0.01726657, -0.0011275966, -0.016545588, -0.014037237, -0.05027021, -0.051595617, 0.010617528, 0.047444824, -0.010230054, 0.023983559, -0.013682806, 0.008274817, -0.08482358, 0.012144159, 0.024952669, 0.026599547, -0.04253916, -0.07242553, -0.00033674334, -0.06493679, 0.016154362, 0.015244355, -0.0047060233, 0.020907296, 0.029029112, -0.036086794, 0.021577153, 0.06852082, 0.000011419103, -0.011964439, 0.00042391437, 0.01622588, 0.06117649, 0.03919344, 0.069235444, -0.022920733, 0.0032567945, -0.055385046, -0.008299763, 0.12611182, 0.016396007, 0.061604634, -0.017348763, -0.039351672, -0.0002511726, 0.016068434, 0.014466261, -0.01795146, -0.02210242, 0.06284842, 0.016422717, 0.0073080827, -0.027887601, -0.005047606, -0.020135367, -0.0054714247, 0.024597183, -0.010635095, -0.046761673, 0.012735961, -0.023473071, 0.009631793, 0.012643822, -0.01115858, -0.014518166, -0.007554396, 0.045325678, 0.029549936, 0.09073912, -0.026296047, 0.012612649, -0.014890393, -0.00291016, -0.0108625125, 0.01496182, -0.047306992, -0.08534146, 0.026593037, -0.016107023, 0.035778847, 0.033154212, -0.013417226, 0.03389385, -0.0023237118, 0.025431113, 0.025109697, 0.060854945, 0.0080888495, -0.002564189, 0.011634001, 0.013853038, -0.07305558, 0.013247203, 0.019107047, 0.010017295, -0.02322088, -0.01112352, 0.016659098, 0.022235803, 0.017635357, 0.03426617, 0.0018381957, 0.011150971, -0.0046320413, -0.008533564, -0.021399502, -0.1642215, -0.032931734, 0.025492677, 0.07273245, -0.026394797, -0.012919645, -0.014278748, 0.04866505, -0.0008331715, -0.005812511, 0.06361204, -0.029894004, -0.010915368, 0.018044678, 0.011098923, -0.06443523, 0.05937764, -0.0091087185, -0.037731748, 0.07707219, 0.052224528, -0.03462039, 0.072725035, 0.0058749206, -0.031012183, -0.0013843459, -0.0016020465, 0.00046772297, -0.021521972, 0.020598054, 0.00014907998, 0.058886066, -0.040748097, -0.044112142, 0.0050505064, -0.0072475555, -0.035578005, 0.031292576, 0.010170539, -0.02344862, 0.029923033, -0.05092102, 0.030136008, -0.027871558, -0.013548197, -0.0011284129, 0.04447071, 0.01908483, -0.040758636, -0.0050431197, -0.033989407, -0.008109632, 0.0028078589, -0.005964677, -0.044320665, 0.08695121, -0.06581071, -0.014559005, 0.0006754284, -0.0207703, 0.0014562753, 0.006021814, 0.0033262006, 0.025546577, -0.034294672, -0.022157641, -0.052147754, -0.034481972, -0.018689197, 0.051176555, -0.012629773, -0.057625394, 0.01853276, 0.025845619, 0.04183072, -0.014905722, -0.012709193, -0.026399568, -0.0015176849, -0.00870039, -0.0023843825, 0.00723293, 0.023719149, -0.02797713, 0.019453716, -0.02975924, 0.006006036, -0.04094432, -0.029385649, 0.008425855, 0.043613557, 0.037576094, -0.006300761, -0.025635675, -0.051340513, -0.015650092, -0.021530788, 0.039177362, 0.07929203, -0.14325929, -0.003475806, 0.025728887, -0.065845564, -0.03128999, -0.0593765, 0.0080624195, -0.055823788, 0.03010428, -0.087225854, 0.0048637777, 0.019308595, -0.023937633, -0.03792896, -0.020528795, 0.012977183, 0.01872807, 0.019077051, -0.028091716, -0.006036729, 0.022920132, -0.008813602, 0.00981394, 0.008174745, 0.022167716, 0.013331586, 0.01701888, 0.052224748, -0.041465946, 0.008684657, 0.021895818, -0.047711246, -0.040310692, 0.057397943, -0.041236855, 0.007911667, 0.021838093, -0.004579466, 0.0023433939, 0.015018914, 0.074861094, -0.024751361, 0.035624117, 0.039900754, -0.01822232, 0.06487644, 0.004835881, -0.0053935177, 0.009328167, -0.022058815, -0.06612124, -0.028995143, 0.023586513, -0.020899288, -0.014397384, -0.030469257, 0.053119045, 0.018539974, 0.014746581, -0.0050767274, -0.043124117, -0.028738571, 0.02584166, -0.052865863, -0.02837898, 0.032055117, 0.037913665, -0.018623862, -0.0022194595, -0.0020972502, -0.026149418, 0.030747438, -0.027549936, 0.026058365, 0.024547432, -0.0176852, 0.02957925, -0.052783605, 0.02594537, 0.052445583, 0.014661957, -0.012167871, 0.017852467, -0.04321671, 0.00559634, -0.006025925, 0.009329029, -0.03378193, -0.048461765, -0.009965817, -0.03232844, -0.016287258, 0.028570518, 0.040578123, 0.04089757, 0.016624473, -0.0091228075, -0.0076028174, -0.022629958, -0.05530809, -0.024680438, -0.011744221, -0.020714726, -0.024899598, 0.03258809, -0.0021902719, -0.019477153, 0.017724114, -0.043114655, 0.030565836, 0.07200336, -0.0139958365, -0.018088685, 0.04667293, -0.07600616, 0.03944098, 0.012371882, 0.01377659, -0.009994407, -0.0032519065, 0.048178624, -0.076111965, 0.07663702, 0.034574185, -0.029811192, 0.015694356, -0.0133601, -0.07539893, -0.016588442, -0.0046641314, -0.00617639, -0.0029704834, -0.07042657, -0.01411732, -0.050324533, -0.07810653, -0.0015109386, 0.03944382, 0.01798586, 0.030200941, 0.039418973, 0.053611957, -0.0060891905, -0.009247284, -0.05610884, -0.026452627, -0.025143502, 0.032464948, -0.008577768, 0.06182524, -0.0061064065, 0.031993497, 0.04613627, 0.007984114, -0.028205479, -0.012368368, -0.009234661, -0.028549464, 0.012541238, -0.021708515, -0.056907747, 0.015648095, 0.09443484, 0.028194426, 0.02801269, -0.002796927, 0.015401413, -0.009387353, -0.048924122, -0.0029509887, 0.04158422, 0.044161636, -0.028189644, -0.04372058, 0.0075099207, -0.0045826267, -0.029747542, 0.016120844, -0.028553538, -0.042564243, -0.06195169, -0.004293553, 0.017685635, 0.037402913, 0.016650058, 0.002564552, 0.02526971, 0.049558762, 0.0148929, 0.032550864, 0.00891755, -0.014785538, -0.031306427, -0.04161889, -0.005493526, -0.05704475, 0.028650155, -0.013743009, 0.021007707, 0.042865556, -0.06892823, -0.07577363, 0.010250175, 0.018683681, -0.01585912, 0.05638925, -0.028040344, 0.028054254, -0.05324003, -0.02835401, -0.029761948, -0.013953871, 0.01800929, 0.009602985, 0.025280487, -0.037758883, -0.021909306, 0.07788558, 0.052966382, -0.040948957, 0.02080413, -0.0046585943, 0.039787147, 0.044573467, -0.06774258, 0.021621695, 0.0008335384, 0.021715373, -0.040091764, 0.060576007, -0.0227573, 0.006898387, 0.05668734, 0.0010268949, 0.02590081, -0.00074422394, -0.0064013256, 0.031980544, 0.020339215, 0.014684404, 0.0062753647, -0.0080067795, -0.0015621854, -0.06117102, 0.010167602, -0.01519366, -0.0032275612, -0.013043986, -0.0042937663, -0.036814604, -0.08193704, -0.045896634, 0.024048537, 0.02703387, 0.046240617, -0.0155930715, 0.011259377, -0.023155902, -0.07172296, -0.05029704, -0.0230527, -0.031759594, 0.022849463, 0.0112401415, 0.062584445, -0.07510336, -0.045729548, -0.017064875, -0.030634373, -0.007639833, -0.035970815, -0.03134226, 0.068768606, 0.07363017, -0.04283125, -0.00067329995, -0.019395802, -0.059890278, -0.018266609, 0.021962892, 0.071512796, 0.00025120532, -0.026365593, 0.060046624, 0.03317017, -0.008745786, -0.0022287073, -0.021264914, 0.0075470777, -0.044957127, -0.08038446, 0.01253501, 0.022953615, 0.014083689, -0.0014427358, -0.036099065, 0.026459547, 0.016523506, -0.03897762, 0.006607671, 0.009139796, -0.037935395, 0.018786864, -0.01451393, 0.023860335, -0.05187726, 0.05473226, 0.02819557, 0.019244624, 0.05156876, 0.01142133, -0.011831102, -0.08391626, 0.014320813, 0.06879024, -0.0010856502, -0.06156981, 0.0031307023, -0.036413312, 0.050030474, 0.049246844, 0.008607475, -0.022122692, -0.006110102, -0.007980099, 0.00962487, -0.022768142, -0.04128197, 0.024498656, 0.024653161, 0.020282926, 0.05155187, -0.01901482, 0.010729483, 0.039281067, 0.015659973, -0.027650835, -0.027464101, -0.04477211, 0.033029187, 0.041071657, -0.026165942, 0.032382686, 0.0057084663, 0.03518667, -0.02700029, -0.009661002, -0.013366043, 0.014081378, -0.008341646, -0.05741408, 0.0062511517, 0.025198726, -0.046945978, 0.016139716, -0.005797706, -0.0036899445, -0.035615947, 0.06911907, 0.010679314, 0.016864298, 0.042853825, 0.028222864, 0.032849014, -0.0026505995, 0.018490527, -0.034107164, 0.022297237, -0.0073127826, 0.053110078, 0.065449916, 0.041058164, 0.021823779, -0.021113796, -0.015468001, 0.01431247, -0.011184501, -0.03244793, 0.008704486, -0.0055382694, -0.039018568, -0.0225737, 0.033382386, -0.011950774, 0.043544307, -0.016272482, -0.002017444, 0.035562012, 0.011835055, -0.10499318, 0.014407153, 0.031233227, -0.02629314, -0.027410932, 0.0027242294, 0.050826155, 0.03287149, 0.040926117, -0.06730318, -0.027012268, 0.0026494754, 0.010218398, 0.02784687, 0.0062189917, 0.024754569, -0.02757616, -0.030109294, 0.035302497, -0.044085264, 0.00767291, -0.056374624, 0.009987366, 0.02657654, 0.07756338, -0.012470898, -0.0068156174, -0.05525278, 0.0023654045, -0.02498901, -0.027593857, -0.017768318, -0.03699703, -0.014197921, 0.092333876, -0.0056037325, -0.05542219, 0.012255174, 0.008928207, -0.0038674322, 0.009741423, -0.03921643, -0.065612905, 0.05074842, 0.05339858, -0.024575463, -0.03433625, -0.024730517, -0.010357354, -0.015540877, 0.077590324, -0.014201184, 0.045420308, 0.021700228, 0.022370148, -0.01598199, 0.022091014, -0.008826, -0.010888861, 0.021698192, 0.024975145, 0.040205836, -0.09837367, -0.012474524, -0.010366442, -0.027369563, -0.04808978, 0.015360504, -0.035136025, -0.0074538486, 0.01963089, -0.03618698, 0.012864401, -0.009739564, -0.034426022, 0.025257912, 0.0086342795, -0.0023661968, 0.03362397, -0.053549908, 0.03817138, -0.029909806, -0.017513692, 0.04609801, -0.028078984, 0.052121654, 0.024857197, -0.041294504, 0.01839261, -0.03335572, 0.00064589526, -0.0055421707, 0.0061826957, -0.0091359345, 0.01942602, 0.008288212, 0.028513521, 0.0023563644, -0.08101017, 0.018640758, -0.047110505, 0.021327078, 0.029772937, -0.073679246, 0.024734367, 0.040876333, -0.02186643, -0.008356093, 0.025163399, 0.029394837, -0.011241934, -0.07067061, 0.023514945, 0.03720482, -0.0028830285, -0.008353445, -0.029916758, -0.041413482, 0.037415426, 0.036660317, -0.011016019, -0.074796185, -0.014593292, 0.026445702, 0.009598844, 0.029317554, -0.0013094156, -0.014406907, -0.035042368, -0.056775432, 0.053708993, 0.038438074, 0.010673802, 0.03972148, 0.020914605, -0.03744723, -0.0055944584, 0.02152291, 0.055107396} + + inputVectorGemma300m := []float32{ // task: sentence similarity | query: I like soccer + -0.12321771, 0.022824064, 0.0055121304, -0.005476948, -0.009367921, 0.013712689, -0.022698374, 0.11511719, 0.025182812, -0.079908505, -0.04638956, 0.01585905, 0.020481788, -0.008693112, 0.018376555, 0.051080413, -0.030498909, -0.00049509195, -0.05575088, -0.057395093, 0.025564281, -0.004488282, -0.015492142, -0.009914164, 0.013617828, -0.0013789881, 0.013404853, 0.051821396, 0.029527253, -0.0013196444, -0.010924045, -0.011264279, 0.005740998, -0.019597946, -0.0035221542, -0.023866538, 0.013538469, -0.047199856, -0.038643423, 0.028522268, -0.071105145, 0.0681064, -0.020070722, 0.039927535, -0.03640084, -0.014261802, -0.026860155, -0.0074400906, -0.0015283837, 0.02103081, -0.021142745, 0.013603733, -0.04461476, 0.05514363, -0.03893873, 0.0066638077, -0.042887174, 0.01429444, -0.0035970148, 0.028575595, -0.060669154, -0.023942245, -0.013997114, -0.0017093577, 0.046667654, -0.0020601929, 0.048401903, -0.06137632, 0.062697314, 0.14808325, 0.010441465, 0.028361585, 0.0038594215, -0.04469151, 0.13423842, 0.0310153, -0.013362881, -0.022903973, -0.013121448, 0.0078025083, 0.030627254, 0.03973142, -0.035701957, -0.035638984, 0.1338159, -0.038115826, 0.05529096, -0.020699969, 0.0045565437, -0.04477859, -0.010443524, 0.00054587494, -0.011510408, 0.012664933, 0.029471243, 0.012673756, -0.049280766, 0.06829018, 0.034648634, -0.022574263, -0.05383786, 0.04805933, 0.05687877, 0.05983964, -0.027078373, -0.02409322, -0.007473542, 0.0404781, -0.005394586, 0.0051933867, 0.008781422, -0.07454405, 0.0066473274, -0.032365926, 0.0067254975, 0.005875436, -0.004049964, -0.0086815925, 0.017125882, -0.019584285, -0.0025491514, -0.029743217, -0.025699811, -0.027057251, -0.025254942, 0.07575045, -0.03309629, -0.074179746, 0.04392139, -0.0016817227, 0.00867535, 0.009644793, 0.08746248, 0.009448568, -0.02514465, 0.044356693, -0.044212468, 0.060101938, 0.04173838, 0.032885026, -0.064920366, -0.08943727, -0.0740428, -0.057719745, 0.05591522, 0.044605024, 0.03631802, 0.03147071, -0.00940167, -0.018166594, 0.015426969, 0.03560209, 0.051894315, 0.0035225234, -0.02430683, 0.05017901, 0.003955761, 0.022228105, -0.029533688, -0.014322637, -0.015690407, 0.041909903, 0.03123911, 0.10701835, 0.06595974, -0.021476354, 0.029652148, 0.08975483, 0.04579224, 0.04168257, -0.019954108, -0.074105315, -0.011851827, -0.002689897, -0.0077179205, 0.016488366, -0.02460326, 0.06291314, 0.01707504, 0.028539255, -0.025035191, -0.01582938, -0.011486634, 0.025247667, -0.054728657, 0.04904634, -0.005351516, 0.07456472, 0.0017613986, -0.029275635, 0.01815601, 0.0136083085, 0.034951538, 0.023815537, 0.029369412, -0.0016458781, 0.08241508, -0.016191151, -0.0005029788, -0.019342225, 0.0091475, -0.0571558, 0.0018099071, 0.017129662, -0.053761676, -0.028048707, -0.038308118, -0.012134579, -0.073870115, -0.046498846, 0.03067407, 0.018465754, 0.009259859, -0.01969478, 0.035371777, 0.004515785, 0.012486229, 0.014211037, 0.030955313, 0.006130703, -0.033880085, 0.028415188, -0.029841837, -0.019551452, -0.046305962, -0.0003389184, -0.020428263, -0.036920346, 0.011644513, -0.026506048, 0.027107896, 0.05876055, 0.0468323, -0.0005687385, -0.04880607, -0.03564457, -0.009156481, 0.05261804, -0.021908766, 0.102491446, -0.08224326, 0.05419275, 0.0030326108, -0.017994141, -0.011029299, -0.0265645, 0.0042140083, -0.06318661, -0.0029825557, -0.07246673, -0.016977008, 0.01188308, -0.015218057, -0.044749763, -0.0058135525, -0.03828502, 0.029198902, 0.0120264515, -0.027850846, 0.025151724, -0.0070469445, -0.007999846, -0.031044375, -0.032761183, 0.024035562, 0.014000189, 0.0019284323, 0.020979438, -0.039897457, 0.016896162, 0.029204104, -0.01325815, -0.062684275, -0.013710887, -0.012875284, -0.0012331284, 0.022988565, 0.0079186205, 0.0060192267, 0.01783291, 0.041763164, 0.0041113473, 0.015751729, 0.05884899, 0.008356878, 0.0147051355, -0.03130989, 0.052787583, -0.039541624, -0.057540845, 0.004576473, -0.020362213, 0.039119106, -0.0335866, 0.020611428, -0.024094008, -0.022075292, -0.07904291, -0.016741969, 0.030172616, -0.108806364, 0.019492703, -0.0042844783, -0.058019683, -0.011671026, 0.046765752, -0.03451943, 0.030001348, -0.04156904, 0.04323121, -0.027447008, -0.023043511, -0.03110686, 0.026867297, 0.04872865, 0.055358402, 0.035706133, 0.020926835, 0.075120576, 0.01374666, -0.003491194, 0.02522124, 0.0403565, 0.0024097424, 0.005241686, 0.037378468, 0.033737376, -0.040098835, -0.016304577, 0.044312652, -0.014464315, -0.007490535, 0.042791463, 0.016222997, 0.04990817, 0.07522978, -0.029063424, -0.017800571, -0.006374426, -0.071620904, 0.00050548866, -0.03150775, 0.016091848, -0.007316904, 0.032517817, -0.020634828, -0.06805583, -0.004445925, 0.011847202, 0.020042786, 0.040393036, 0.031112982, -0.04022227, 0.051720932, -0.027354056, 0.016958864, -0.03141734, 0.0016699112, 0.014959932, 0.015450632, -0.044929385, 0.06100186, 0.032446217, -0.011693143, -0.09302732, 0.0013519578, -0.07128394, -0.010992893, 0.06006141, 0.07027135, -0.012503397, -0.019628063, -0.004051181, -0.015575752, -0.02794168, 0.0050593577, 0.016944395, 0.034895614, 0.005634461, 0.014883453, -0.016175447, 0.01360463, 0.009957782, -0.0068508047, -0.030567292, 0.032338668, -0.00083503884, 0.06257236, 0.0077934274, 0.13019733, -0.029386636, 0.01706133, 0.049117237, -0.009944691, 0.029031567, -0.0034067787, -0.028361622, -0.020064648, 0.021023223, -0.041297078, 0.0011128377, 0.019977288, 0.02634081, -0.014777996, 0.006007697, -0.012777911, -0.0028568064, 0.004024345, -0.004129589, 0.025061166, 0.0516007, 0.012697804, -0.023802636, 0.037224397, 0.03207148, -0.043906048, 0.0358131, -0.005983637, -0.053476203, -0.011206745, -0.0454309, -0.015574752, 0.006953732, 0.048708037, 0.042242724, 0.063127756, 0.01446956, 0.036337584, 0.022033319, 0.03249794, 0.021103887, -0.027244741, 0.047036715, -0.0030890328, 0.045025703, -0.028380616, -0.031676944, 0.03496025, 0.030881668, 0.03801315, -0.023088606, -0.058103837, 0.025140118, -0.01097736, -0.000029534047, 0.04273616, -0.008030801, 0.030773379, -0.0170947, -0.0063718185, 0.006903434, -0.04932383, 0.081497625, -0.014359693, 0.018919952, 0.011267754, -0.010265285, 0.06680713, 0.029079778, 0.0036719248, -0.0030894685, 0.0021278495, -0.0103025595, -0.019103613, -0.053278934, 0.004628443, -0.020637216, -0.025919521, -0.019824494, 0.021056278, 0.019688051, -0.009582765, 0.045784753, -0.029008115, 0.025729576, -0.0020786985, 0.0041243383, 0.02844323, -0.0052059367, 0.0125448555, 0.008598586, -0.013385952, -0.06250317, -0.014588583, 0.029591184, 0.016119765, 0.015811194, 0.0059438846, 0.059809208, -0.007278358, -0.09407869, 0.00063716894, -0.012805698, 0.018674226, 0.04443006, 0.05180132, -0.056063306, 0.033491198, -0.112140894, -0.044935003, -0.024846552, -0.017737472, -0.0011454361, 0.019539684, 0.036826793, 0.0010134777, 0.015898362, -0.036675476, 0.013609578, -0.013190406, 0.013880912, 0.015874494, 0.01297524, 0.034235142, -0.023937047, 0.019767134, -0.013272534, 0.028228898, -0.014248082, 0.080095164, 0.032325353, -0.02313599, -0.0066155046, 0.003267183, 0.029832888, -0.028922392, 0.0035783031, 0.00094675296, -0.054376256, 0.02769277, -0.09301246, -0.0005279491, -0.050708015, 0.026412731, 0.058249053, -0.035072435, 0.04198701, 0.01665575, -0.040349524, -0.053689048, 0.022197006, -0.05191208, -0.010155883, 0.008705657, 0.0069932947, -0.0032210012, 0.02957352, -0.0028149271, -0.01879217, -0.0050257924, 0.031445704, 0.0098605165, -0.052647747, 0.015594799, 0.039669096, -0.008970331, -0.03027834, -0.013811162, 0.03289345, 0.006157646, -0.010574978, -0.024146235, -0.0019740157, -0.030233635, -0.002071066, 0.021190692, -0.04418655, -0.045019303, 0.010577838, 0.014611987, -0.029611891, 0.07457874, 0.040747344, 0.0016370848, -0.03420744, 0.008627148, 0.018853694, -0.029839037, -0.0656448, -0.006840682, 0.026035093, 0.042488925, -0.04037711, 0.040723428, 0.05238118, 0.019430365, -0.0003322385, -0.031335082, -0.03726736, 0.028814876, 0.038058355, -0.01969931, 0.049083527, 0.04334873, 0.01683031, -0.02441815, -0.021151206, -0.026023645, 0.05267304, 0.0055371244, 0.002753653, 0.012441297, 0.026773496, -0.008364757, 0.008765207, 0.03442934, 0.025056437, 0.018266954, -0.025982559, 0.025592547, -0.030820644, -0.011917623, 0.012841847, -0.04023565, 0.008560673, -0.021629302, 0.031185832, 0.019670403, -0.05410101, 0.013168653, -0.018121649, 0.017648662, -0.000035086316, -0.013787194, 0.018333768, -0.025808856, 0.01197731, -0.014240641, -0.021569442, -0.07235577, 0.015790833, 0.07051888, -0.09803995, 0.020495994, 0.0007384825, -0.0027266983, 0.03956342, -0.061942317, -0.006841304, -0.054652456, 0.022062853, 0.0050070975, 0.014867274, -0.042505514, 0.03899678, -0.002742749, -0.03388238, 0.0006341626, 0.0043469467, -0.026006838, -0.05382788, -0.025274688, 0.00046234217, 0.036418866, 0.0022927113, 0.03060308, 0.024053773, 0.012819702, -0.056986537, -0.045696, -0.021085966, -0.017024172, -0.035328474, 0.02319034, 0.011516923, 0.0072190645, 0.00079755974, 0.009485171, -0.021936087, -0.004135782, -0.062918, -0.02185894, 0.008199985, 0.04685388, 0.00572528, -0.042947467, 0.023429219, -0.015358843, -0.010224128, -0.019305708, -0.04232563, 0.033555873, -0.017649941, 0.018863538, -0.0082277525, 0.039818734, 0.011953176, -0.0015674818, 0.026395414, -0.021302143, 0.052965406, -0.0889442, 0.015120549, -0.03595036, 0.00885053, -0.0316501, 0.008767527, -0.03910736, 0.0035782075, -0.011844713, 0.0073405085, 0.013546157, -0.001394585, -0.0072244755, 0.047209453, -0.0056066504, 0.0115044685, 0.036355887, 0.015738497, 0.041819654, -0.036501583, 0.0058697597, 0.026937716, -0.0042040288, 0.06994945, -0.0016084182, -0.056668576, 0.006376454, -0.018162044, -0.029766992, -0.0070448727, 0.03030743, 0.022082722, -0.012325634, 0.047960114, -0.018438129, 0.026550142, -0.08271825, 0.069413364, -0.040486116, 0.034727506, 0.030108988, -0.04133625, -0.029786991, 0.029141676, 0.018547248, 0.027667733, 0.008559842, -0.024279347, -0.008305973, -0.008182398, 0.021805892, -0.007978974, -0.01067629, 0.02965719, -0.022380637, -0.04717387, -0.040280595, -0.0035846618, -0.035994705, 0.014099024, 0.03432559, 0.008259968, 0.04926102, 0.00008602467, 0.013023371, -0.035448007, -0.06805746, -0.05215253, 0.021759389, 0.02667032, 0.025592446, -0.013707798, 0.010434229, -0.00016320526, -0.03964802, -0.066464365, 0.008149845} + testVectorSimilarGemma300m := []float32{ // task: sentence similarity | query: I love sports + -0.12093516, 0.020611502, -0.008910375, -0.040245544, -0.059986454, -0.025952723, -0.002080849, 0.10856603, 0.014315221, -0.09833044, -0.023855248, -0.029891195, 0.046237975, -0.022247097, 0.028401012, 0.06893259, -0.024836207, -0.010965274, -0.05950975, -0.0469831, 0.006604364, 0.028864492, -0.03337461, -0.013447034, 0.027711349, 0.015716122, 0.0030353507, 0.019953938, 0.04457284, -0.0020691908, 0.012331259, -0.015210176, -0.007976071, -0.005732741, 0.015023559, -0.02808257, 0.013296977, -0.030053237, -0.0027864228, 0.0048620556, -0.120145306, 0.099067025, 0.014721063, 0.02061065, -0.020422328, 0.0067061586, -0.01781667, -0.025231302, -0.019336974, -0.002130203, -0.0006079967, 0.0095950095, -0.066419825, -0.0037761212, -0.054431796, -0.012971789, -0.012736796, 0.051682115, 0.018364238, 0.031189702, -0.06843367, -0.0045142234, 0.00085250946, 0.0035914048, 0.021059195, -0.028496068, 0.06355422, -0.01687848, 0.06509813, 0.14370309, 0.0016759251, 0.022435976, 0.00047672645, -0.016653359, 0.115529075, 0.01360904, 0.004941204, -0.030297674, -0.008789805, 0.01397979, 0.051636558, -0.005012114, -0.037343733, -0.047853902, 0.11073606, -0.084064074, 0.052548707, -0.0520407, 0.014269445, -0.033926487, -0.008524317, 0.033744846, -0.029268231, 0.0056701177, 0.032077253, -0.041169014, -0.014300659, 0.06423583, 0.010563547, -0.028144028, -0.039804507, 0.037515283, 0.04934839, 0.09449742, -0.021899939, -0.023938278, -0.006887751, 0.049292684, 0.02073799, 0.008462621, 0.036157403, -0.07624548, 0.025872508, -0.057896953, -0.01414715, 0.017976722, -0.025459098, 0.020017276, -0.007268415, -0.003648494, -0.00079534267, 0.00058695546, -0.035642993, 0.009572605, 0.0016801606, 0.057227295, -0.030347968, -0.030702807, 0.03358033, 0.00994441, -0.007324845, 0.014087407, 0.05990585, 0.03134507, -0.037075397, 0.022567568, -0.02786478, 0.038499955, 0.03147155, 0.039133385, -0.047081392, -0.08869925, -0.08950062, -0.06822118, 0.033547968, 0.037909366, 0.037242252, 0.005774596, 0.01493051, 0.0014363725, 0.010530365, 0.06160555, 0.06234572, 0.005520962, -0.054033842, 0.00843325, -0.0015555184, 0.020654468, -0.04562308, -0.031766094, 0.013994946, 0.0495752, -0.011859846, 0.11232993, 0.038654804, -0.055293735, -0.0070923762, 0.063161306, 0.035427958, 0.032231122, -0.027985664, -0.053004578, 0.0053710183, -0.017613428, -0.006357463, -0.0050828317, -0.014690271, 0.065785326, 0.013647075, 0.020660594, -0.015580897, -0.025672581, -0.01744955, 0.05950605, -0.058893383, 0.006920672, 0.00094713556, 0.04646492, 0.021234158, -0.04622453, 0.0020552215, 0.008390582, 0.036585737, 0.008928627, 0.030316306, -0.0028939229, 0.07428975, -0.011881972, 0.021745382, -0.006310547, 0.018207308, -0.04841237, -0.0067020026, 0.026207998, -0.01533991, -0.03444576, -0.031102652, -0.013674477, -0.028354386, -0.0060756416, 0.0414354, -0.0013959741, -0.02041033, -0.005466393, 0.047139637, 0.0031896567, -0.016207084, 0.003453848, 0.03323289, 0.015824148, -0.008917649, 0.028954046, -0.040323745, -0.031683564, -0.07087632, 0.007851276, -0.021268945, -0.030444708, -0.013992206, -0.05300477, 0.010955706, 0.032757737, 0.051291436, -0.027210433, -0.011823065, -0.010752185, 0.00061423174, 0.06546056, -0.03221141, 0.079450436, -0.09526929, 0.06514671, -0.010331335, -0.018138617, -0.021547752, -0.0520411, 0.0143707795, -0.063532084, 0.0020811746, -0.08046814, 0.013607465, 0.015301091, 0.024845928, -0.053565703, -0.026712855, -0.0207584, 0.03337548, 0.013841997, -0.029905228, -0.002294361, 0.018276034, 0.008458818, -0.050662283, -0.0634163, 0.009141184, 0.004138546, -0.011172376, 0.022109954, -0.019690303, 0.032628126, 0.028965002, -0.017625837, -0.05909601, -0.01099851, -0.008373381, 0.009897048, -0.005982712, -0.026032105, -0.023465456, 0.009602586, 0.031038921, 0.0049723056, 0.025946032, 0.0579722, 0.009515565, 0.02810677, 0.037245158, 0.03231413, -0.014389683, -0.047064178, 0.008645607, -0.03386236, 0.025474362, -0.0027332422, 0.04355698, -0.012834391, -0.015077516, -0.07465946, -0.008824354, 0.021723656, -0.10536976, 0.027371578, -0.0061267177, -0.057402376, -0.016892958, 0.05463014, -0.034877088, 0.035036497, -0.051474884, -0.0117170205, -0.03684217, -0.012235515, -0.046839528, 0.01815173, 0.033943597, 0.03588609, 0.041779686, 0.015047916, 0.09113202, 0.010342291, -0.000217042, 0.025122525, 0.01914211, 0.022923606, -0.015960813, 0.011607169, 0.007672376, -0.04069214, -0.022554014, 0.040715657, -0.046215154, 0.017459909, 0.041611668, 0.04186466, 0.048546396, 0.070445254, -0.04907234, -0.018840812, -0.007940024, -0.037182562, 0.011019142, -0.026205545, 0.012075213, -0.024166368, 0.06166631, -0.03057955, -0.032431286, -0.005570214, 0.01577734, 0.028152578, 0.038514107, 0.034939688, -0.035715632, 0.06456991, -0.056508653, 0.06588127, -0.00509685, -0.014286179, 0.012863249, 0.0133177405, -0.04542378, 0.04738746, 0.036716007, -0.009201173, -0.07236751, -0.03568247, -0.05744905, -0.0040989043, 0.065951645, 0.06781417, -0.02379004, -0.04633536, -0.0154118035, -0.03867407, -0.042976495, 0.014570381, 0.022328109, 0.039635953, 0.012322261, 0.0035027252, 0.019434324, 0.0014310282, -0.00048495538, -0.020233585, -0.036483005, -0.022304807, -0.003983367, 0.047288608, 0.012361091, 0.099381104, -0.021652738, 0.0043193894, 0.05916932, 0.0026121803, 0.018273495, -0.017659029, -0.021731468, 0.006376373, 0.022413818, -0.046105225, 0.01026245, 0.014936892, 0.03336531, -0.0042252345, 0.03968516, -0.030616414, 0.014258155, 0.017743789, -0.001629017, 0.02279657, 0.02895259, 0.0018774783, -0.030175555, 0.033012524, -0.00047780894, -0.034371816, 0.010158613, 0.009746794, -0.047972564, -0.008128641, -0.059709273, -0.0012170484, 0.039204497, 0.060498253, 0.04275535, 0.08358398, 0.014019027, 0.04653778, 0.01719581, 0.015080213, 0.015221243, 0.015450773, 0.013407785, -0.01721452, 0.025952034, -0.036236726, 0.0049564103, 0.02973437, 0.021373525, 0.070225604, -0.03525609, -0.07123085, 0.009256419, 0.00031672846, -0.014771204, 0.023140691, -0.019313246, 0.023433223, -0.008301419, 0.021429082, 0.020874094, -0.044997394, 0.0629941, -0.008743041, 0.021177683, -0.012568246, -0.020473924, 0.06439173, 0.018100806, 0.0028072763, 0.0043916805, 0.0207422, 0.00394057, 0.0012306252, -0.05664472, 0.017976053, -0.032977957, -0.007007895, -0.05340276, 0.030489808, -0.01943697, -0.014338036, 0.04065199, -0.013697228, 0.023744242, -0.018449724, 0.010409184, 0.024293711, 0.008373324, 0.018142382, 0.026173016, -0.06754546, -0.012745807, 0.0093034925, 0.0074641323, -0.008144507, 0.02494148, 0.018926967, 0.034302726, -0.00033791867, -0.09027262, -0.012825458, 0.0024748768, -0.020475814, 0.056220286, 0.033924717, -0.021101844, -0.0009108224, -0.16524875, -0.053439587, -0.018944656, -0.0055759964, -0.013928266, 0.026028298, 0.03426948, -0.011940294, 0.009683816, -0.039616153, -0.013615597, 0.004027068, 0.024830837, 0.035919204, 0.03237462, 0.021004073, -0.03552714, -0.000560216, -0.028964777, 0.026448, 0.028636852, 0.0756387, 0.020007858, -0.01377832, -0.0366827, 0.0003926886, -0.017364109, -0.003340915, -0.015707148, -0.035216868, -0.038999107, -0.0040652934, -0.038285878, 0.004660833, -0.034794, 0.020163676, 0.01912582, -0.014481669, 0.020746116, 0.0028038125, -0.048530567, -0.05419415, 0.017702196, -0.035930503, -0.0051735532, -0.008288992, 0.0027074115, -0.02029673, 0.026149925, -0.0022992827, -0.018746978, -0.0015564192, 0.019480761, 0.016744304, -0.017822778, 0.01381326, 0.026279353, 0.0047759893, -0.017716758, -0.028837685, 0.03469464, 0.031002525, -0.0037651344, -0.030923575, 0.016952207, -0.0090538, -0.008810055, 0.066860095, -0.041087292, -0.039752748, -0.000098491866, 0.017583186, 0.008133089, 0.07257567, 0.025880326, -0.01480255, -0.007923338, -0.006952911, -0.012049002, -0.05708306, -0.01677453, 0.018310674, 0.014435502, 0.042109467, -0.036192324, 0.022005899, 0.014573509, 0.032290503, -0.0038509339, -0.043771483, -0.02470985, 0.0010982379, 0.064717434, -0.03259728, 0.024640672, 0.03077456, 0.0079402, -0.02055057, -0.021976018, -0.018206745, -0.008835613, -0.01933013, -0.036507756, 0.0077984473, 0.014035703, 0.01102294, 0.015074801, 0.025761256, -0.011909488, 0.02831095, -0.036591504, 0.012883212, -0.020112932, -0.030147692, 0.00969793, -0.03633476, 0.011417734, -0.0026998064, 0.0202021, 0.03074099, -0.02620934, -0.0042247972, 0.015047929, 0.031132108, 0.024823781, -0.023235321, 0.019149283, -0.009282112, 0.02559514, -0.018280253, -0.004956729, -0.074955806, 0.030954149, 0.06877509, -0.084637076, 0.017542837, -0.004280242, -0.00021268544, 0.017178413, -0.05687185, -0.034910038, -0.04442082, 0.0243306, -0.014917598, 0.058736514, -0.05364744, 0.0062897257, 0.0055779796, -0.022743607, 0.01642166, 0.024381971, -0.01984619, -0.016100237, 0.00030256563, -0.01340241, 0.060572013, -0.0043026856, 0.04169896, -0.01154707, 0.016755912, -0.0724847, -0.08045217, -0.025079053, -0.03765303, -0.03722819, 0.05289811, 0.024918923, -0.009980753, -0.029925726, 0.009984141, -0.01912415, 0.00049060804, -0.06848064, -0.015760465, 0.03413404, 0.049184043, -0.00975605, -0.030466484, -0.010264119, -0.0373076, -0.023015931, -0.047553197, 0.009903521, -0.0058034877, 0.012413765, 0.049728055, -0.009594009, 0.011971146, -0.0042625107, 0.023919968, 0.01605025, -0.0150553305, 0.022165168, -0.12511821, 0.033200376, -0.03162343, 0.05615519, -0.037898626, -0.014045751, -0.013009668, 0.011196862, 0.01854401, -0.006560372, 0.015376596, -0.008030198, -0.024911027, 0.02711686, -0.014674427, 0.019948607, 0.076200984, 0.022003805, 0.043584667, -0.022772027, -0.01095706, 0.021133784, 0.012501992, 0.049770206, 0.0070304708, -0.073494695, -0.016590085, -0.016660782, -0.040903624, -0.040648215, 0.014360744, 0.011617368, 0.0032610826, 0.005328433, -0.023914797, 0.018197423, -0.032538015, 0.09887836, -0.049765594, 0.07105858, 0.02056067, -0.032071345, -0.015831899, 0.04767516, 0.02077486, 0.0040670233, 0.030935472, -0.015737839, -0.050856307, -0.0046306634, 0.037393037, 0.011872599, -0.013058071, 0.011674667, 0.0016889643, -0.030171916, -0.030024856, -0.023465512, -0.023812467, 0.016806535, 0.031105634, 0.02376353, 0.041907493, 0.029373415, 0.0130377645, -0.046747137, -0.073259935, -0.05158967, 0.039449807, 0.02116494, 0.013297292, 0.0010564493, 0.009362349, 0.01359117, -0.026652597, -0.025040226, 0.022119204} + testVectorDifferentGemma300m := []float32{ // task: sentence similarity | query: I like painting + -0.1443442, -0.028417166, 0.026512455, 0.02469296, -0.077621624, 0.03324541, -0.017255029, 0.02957162, 0.025552273, -0.051966775, -0.011004275, 0.0063372403, -0.0017320421, 0.009218, -0.0046874583, 0.05777593, -0.0066171484, -0.015365088, -0.0007434527, -0.070197836, 0.01056687, 0.026956744, -0.0019329813, -0.025510522, 0.008876716, 0.055220094, 0.02222488, 0.017477898, 0.021095745, -0.0069521405, 0.03249371, 0.030433813, 0.053857606, -0.0025883042, -0.013010067, 0.0050371825, -0.0020586236, -0.015826192, 0.012195557, -0.04371786, -0.04329228, 0.029894503, 0.019512994, -0.0027994274, 0.042172685, -0.006894716, 0.02104146, -0.07526135, 0.004647484, 0.018697163, 0.00238482, -0.03877407, -0.06573979, -0.017682832, -0.055164915, 0.0209118, 0.0072387136, 0.04374714, 0.004697197, 0.008147401, -0.0497929, -0.005160008, 0.05281443, 0.024757981, -0.0070049227, -0.012380584, 0.04517082, 0.049620144, 0.06837677, 0.15019263, -0.014992098, 0.010042911, -0.022948045, -0.061651878, 0.14630201, 0.045178488, 0.062015854, -0.021479113, -0.016256297, 0.009133047, 0.011121708, 0.021634925, -0.03087203, -0.027060991, 0.10475591, -0.017273618, 0.06501571, -0.0169293, 0.0021167838, -0.017742388, -0.019575639, 0.035203632, -0.028989624, -0.02494871, 0.026177177, -0.015540892, 0.008646821, 0.004100635, -0.0028663692, -0.016234558, 0.0063134693, 0.03371466, 0.021190902, 0.055219818, -0.024649503, 0.0175192, 0.017893119, 0.028698562, -0.03099476, -0.0057745585, -0.0413978, -0.06005816, -0.014233842, 0.024748776, -0.008643694, 0.016339593, -0.020552939, 0.03292996, -0.009537976, 0.014018632, 0.01763642, 0.05975775, 0.0029334824, -0.0047016647, -0.0083535295, 0.054072026, -0.08405563, 0.0058518145, 0.042427924, 0.00020061077, -0.0029458646, -0.02721436, 0.048089642, -0.0054426426, -0.017666593, 0.03306825, 0.0033244742, 0.060268924, 0.04411927, -0.003085405, -0.04288468, -0.107294254, -0.066909224, 0.022994127, 0.070718184, -0.005822848, 0.0006652277, -0.030435862, 0.061890278, 0.0050106607, -0.016857354, 0.043030553, 0.014467871, -0.0223694, -0.0054267556, 0.021732625, -0.031562857, 0.05166368, -0.038568582, -0.05154218, 0.064419515, 0.047719017, -0.005110732, 0.10690987, 0.025769142, -0.036118083, -0.013821938, 0.047403567, -0.0012418113, -0.005764147, -0.0036736932, -0.05797412, 0.049824025, -0.032758128, -0.02022844, -0.014086535, -0.016156623, 0.012373451, -0.044458818, 0.03588304, -0.047521953, 0.043179963, -0.042197183, 0.02130946, -0.057825252, 0.011781713, -0.010509205, 0.061429467, 0.0066201105, -0.05211102, 0.00062470714, -0.00972616, 0.021019686, 0.0041651092, 0.025401795, -0.01995199, 0.098284386, -0.05506446, -0.030382188, 0.020427698, -0.0085525075, -0.02853404, 0.014912605, -0.0034706064, 0.013816123, -0.032658678, -0.056357507, -0.025338568, -0.04441238, -0.00828884, 0.053387288, -0.0024037235, -0.028416682, 0.007530226, 0.05361386, 0.05015508, -0.028458595, 0.005316001, -0.0044929893, 0.00036122795, -0.0036939376, 0.018042322, -0.020929378, -0.0074131996, -0.033437256, 0.032637134, -0.070041314, -0.022317225, -0.02509162, -0.018351689, 0.02460978, 0.059651405, 0.056030903, 0.022200828, -0.021088794, -0.030373443, -0.025861306, 0.011688258, 0.0026238102, 0.09931467, -0.12504272, 0.02324408, 0.0012753146, -0.034497254, -0.012026403, -0.03356252, 0.0029964026, -0.056759935, 0.018320821, -0.09724704, -0.00069841446, 0.026699992, 0.01324438, -0.06046133, -0.01269644, 0.014076506, 0.035928342, 0.013973397, -0.012125014, -0.0020985417, 0.022592248, -0.022421028, -0.018405525, -0.012260072, -0.00075334887, -0.009531461, 0.012784324, 0.059497576, -0.06133455, 0.031706527, 0.03608758, -0.02981622, -0.067895964, 0.026524385, -0.038300373, -0.02944638, 0.012435877, 0.014434745, 0.015637642, -0.0040794145, 0.058997385, -0.02453778, 0.03116346, 0.051353246, -0.011478251, 0.070798084, 0.043342296, 0.06566616, -0.0020077839, -0.023084713, -0.035535786, -0.054860085, 0.022259919, -0.013491835, 0.0015351815, -0.026780557, -0.0016713794, -0.023171742, -0.0011553752, 0.0054534213, -0.08509324, -0.02608315, 0.03013138, -0.0732478, -0.018833764, 0.06322526, -0.004825427, -0.004344794, -0.019292658, 0.020502808, 0.007627092, 0.014571389, -0.03887241, 0.018845832, 0.028440136, 0.012875417, 0.031035278, -0.04739163, 0.037676234, 0.04612362, 0.013890085, -0.017010426, 0.017662415, -0.009759672, 0.00813229, -0.020016199, 0.014598222, -0.031462032, -0.041590422, 0.025538549, -0.029204553, -0.054958463, 0.048067145, 0.034460012, 0.027083538, 0.034184, 0.009389896, -0.038984694, -0.038571905, -0.07981689, -0.04208416, -0.010777195, -0.0007199522, -0.035441108, 0.047764268, 0.008691572, -0.037907172, 0.030100692, -0.012952405, 0.0174126, 0.0491028, -0.02000082, -0.047353476, 0.043474067, -0.032400623, 0.053951513, -0.03894972, 0.010698396, -0.0018346183, -0.00686564, 0.0065277372, 0.024347533, 0.050822496, -0.0077478825, -0.05985167, 0.019185618, -0.022875741, -0.05947086, 0.015800577, 0.013403731, -0.031099647, 0.0040469468, -0.02774163, -0.012714507, -0.02128729, -0.032620564, 0.008364622, 0.017776186, 0.005085895, 0.043284558, 0.039082743, 0.04219015, 0.010108826, 0.020092957, -0.072503634, -0.012161342, -0.033461343, 0.018623758, 0.010599123, 0.101111166, -0.01503623, 0.0070683435, 0.05691609, 0.006071612, -0.0038354946, 0.0009494499, 0.007201938, -0.016995758, -0.019628914, -0.032996576, -0.033168867, -0.011283413, 0.02229706, 0.01641914, 0.025900288, 0.012391982, 0.010614977, -0.012162092, -0.01698449, 0.0026643646, 0.070999704, 0.012748991, -0.04945057, -0.06466018, 0.00623248, 0.005958388, -0.01100925, 0.020905152, -0.060147718, -0.057927724, -0.08219422, 0.0057179346, 0.0066705225, 0.075442925, 0.050991286, -0.0107641155, 0.03913386, 0.07935299, -0.02531578, 0.024278138, 0.025572544, -0.028181585, 0.017810883, -0.031566516, 0.021992762, -0.08051712, -0.0062552597, 0.0024508496, 0.024727186, 0.033429384, -0.068435304, -0.07934678, 0.017192548, -0.008349861, -0.008765958, 0.036776952, -0.013890716, 0.035523847, -0.036987375, -0.018866254, 0.007513346, 0.011781799, 0.06126507, 0.0027281318, 0.0066014244, -0.041880976, -0.03575284, 0.07037295, 0.024755038, -0.04612966, 0.015746012, 0.01716983, 0.023604529, 0.006626573, -0.07414435, 0.0058888965, -0.013919645, 0.018376432, -0.037578925, 0.04096414, -0.03968656, -0.0013067513, 0.056138035, -0.026655704, 0.013928886, -0.030145934, 0.004191454, 0.019197201, 0.007136742, 0.026576793, -0.0011043522, 0.0043507568, -0.032184187, -0.049869895, 0.00605855, 0.0100076385, -0.0035252376, -0.0006575648, 0.014964906, 0.011121768, -0.09418935, -0.02173339, 0.03699047, 0.013662403, 0.023690486, 0.001209606, -0.04165136, -0.0151243275, -0.09361919, -0.03941253, -0.013746467, -0.04788991, 0.03444871, 0.020477198, 0.045509163, -0.046539843, -0.025255777, -0.0015786982, -0.046270832, -0.017472308, -0.0130196195, -0.032322172, 0.045951143, 0.06135397, -0.03121471, 0.01449136, -0.020335117, -0.02914758, -0.0153473485, 0.04127623, 0.07522918, -0.008998184, -0.034172263, 0.05808758, 0.041424237, -0.0018481913, -0.030665271, -0.0004568934, -0.0016977047, -0.031964425, -0.06840241, 0.021449873, 0.0008009028, 0.027556466, 0.04777661, -0.02481272, 0.03489042, -0.006775982, -0.05543, -0.0024584034, 0.027125172, -0.025146892, 0.02891035, -0.017902246, -0.0015315852, -0.03000109, 0.029612038, 0.041500784, 0.026526289, 0.008772361, 0.0321838, -0.002989166, -0.06957945, 0.0017932389, 0.047595758, 0.0006322455, -0.045255054, 0.02382494, 0.017563777, 0.046147116, 0.0330123, 0.01738024, 0.0027083878, 0.0040363804, 0.002795955, 0.023532744, -0.0047257445, -0.03896923, 0.009069773, 0.029678797, 0.010273503, 0.08298784, -0.006389739, 0.03416527, 0.026249547, -0.026896635, -0.027191067, -0.028044846, -0.04965675, -0.010646439, 0.030894797, -0.007450026, 0.0074864407, 0.0036938882, 0.03660267, 0.0003209326, -0.009704369, 0.010476034, 0.028249344, -0.014981011, -0.030014703, -0.029595135, 0.038444005, -0.015293533, 0.0108686695, -0.00678427, 0.00092730584, -0.047056876, 0.05737575, 0.033338692, -0.012663544, 0.036851086, 0.014055511, 0.026467778, -0.015435891, 0.0075418944, 0.013235594, 0.013859267, -0.0005560112, 0.03190303, 0.045441065, 0.029871214, 0.019988945, 0.01572087, 0.004945724, -0.013369053, 0.050816264, -0.03135353, -0.016080001, 0.0008996468, -0.034318876, 0.0045453254, 0.047182124, -0.015273028, 0.027252637, -0.028339466, -0.0019375045, 0.009719526, 0.0458279, -0.06574584, 0.013305029, 0.05056652, -0.12162757, -0.014187217, 0.0063218586, 0.041059338, 0.048660208, -0.021941453, -0.053895853, -0.04299649, -0.009300134, 0.002054371, 0.024348678, -0.047116514, 0.028571201, -0.008469832, -0.022030527, 0.036953792, -0.033375565, 0.008975986, -0.037881378, 0.01043165, 0.0006747108, 0.06652062, 0.018278087, -0.024426067, -0.0409778, -0.011088251, -0.04931821, -0.0517674, -0.044955887, 0.0011458874, -0.017981151, 0.07482439, -0.024284545, -0.026437232, -0.0049353903, 0.010894298, -0.030790394, 0.01541707, -0.06582309, -0.06140171, 0.032470774, 0.036834206, -0.037138645, -0.026651448, -0.027371597, -0.0032075131, -0.015259351, 0.035292186, -0.02894929, 0.032220773, 0.00969043, 0.02151399, -0.036721025, 0.020088056, 0.0015942345, -0.015775269, 0.012056114, 0.020293562, 0.07773489, -0.08842956, -0.010242703, -0.01625802, -0.017070046, -0.056630902, -0.0010308587, -0.041243482, 0.012084447, -0.0066000614, 0.0031557267, 0.016582344, -0.0026721174, -0.03592635, 0.01187199, 0.009072085, 0.039055027, 0.029770957, -0.016580446, 0.043791745, -0.03143822, -0.008052862, 0.023053484, -0.020099731, 0.05578338, 0.017673023, -0.02671604, 0.010317831, -0.03082318, -0.006894204, -0.0018696925, 0.01863518, -0.0076981355, 0.017189367, 0.048528466, 0.0047294097, -0.0063338852, -0.11867066, 0.030825129, -0.06183182, 0.015833512, 0.018354883, -0.048288055, 0.005880975, 0.022758886, -0.01155591, 0.0038547632, 0.019663012, 0.013731036, 0.0047337995, -0.042200748, 0.015787793, 0.048906695, 0.0134694455, -0.01827025, -0.034975924, -0.045810193, 0.018111935, 0.03886665, -0.013230341, -0.030711075, -0.0063968184, 0.045388747, 0.0487291, 0.055466093, -0.009714558, -0.041657798, -0.054220382, -0.032607798, 0.054758232, 0.039203733, 0.009653996, 0.025278272, 0.015724013, -0.028571654, -0.014372956, -0.0017647977, 0.02995611} + + + inputVectorGemma300m := []float32{ // I like soccer + -0.1692533, 0.0012864728, 0.03160527, -0.008473783, -0.004158038, -0.0067565744, -0.03355115, 0.08678089, 0.043444235, -0.064195186, -0.035696406, -0.019155798, 0.046693508, -0.009722655, 0.117483765, -0.004985726, -0.010948276, -0.040211644, -0.035650425, -0.047489632, 0.007980266, -0.01608557, -0.035878863, -0.0042388923, 0.017376821, 0.014855003, 0.008616117, 0.046172775, 0.030794995, -0.01824699, -0.006988346, -0.033073254, 0.0041492376, -0.0023215867, -0.020426307, 0.029400608, -0.005926205, -0.0723277, -0.024655117, 0.0037941467, -0.07928841, 0.099191576, -0.012616762, 0.04717856, -0.018504664, -0.021856124, -0.042123944, -0.012603044, 0.003697192, 0.017003154, -0.011165061, 0.008994696, -0.047951225, 0.046651088, -0.049044006, -0.004639917, -0.054477103, -0.0012181259, -0.009928201, 0.038266897, -0.046699554, -0.0017570099, -0.010336599, -0.030371059, 0.05563174, 0.015699878, 0.00753512, -0.019252347, 0.038162492, 0.17045872, -0.02470167, -0.0036419698, -0.015861133, -0.0029464292, 0.1651795, 0.039554324, -0.026409907, -0.036681112, -0.026388783, 0.0090432605, 0.035855643, 0.042130668, -0.020753551, -0.034951247, 0.08628278, -0.036217976, -0.00865261, -0.021584768, -0.003251138, -0.039718226, -0.008962911, -0.0022774576, -0.013862549, -0.005562367, 0.024436736, -0.013921673, -0.048316874, 0.019940387, 0.0014775979, -0.032964326, -0.06417467, 0.025672233, 0.06302021, 0.1145756, -0.032012377, -0.023344671, -0.025099384, -0.012263204, -0.0051600053, 0.02490048, -0.010529879, -0.06603198, 0.044164587, -0.08339577, 0.04196999, 0.027768835, -0.010883158, -0.021746315, 0.018004056, 0.03370079, 0.014477837, -0.0092791645, -0.006702051, -0.012394792, 0.0040597273, 0.041374322, -0.028010318, -0.024939025, 0.03740956, 0.008745224, -0.0014981267, 0.03216363, 0.052546848, 0.05244848, -0.014790123, 0.036461655, -0.051543072, 0.010518737, 0.04951628, 0.04067535, -0.040843856, -0.107296534, -0.013089836, -0.08491402, 0.06157821, 0.032367397, 0.03348762, 0.042206883, -0.00594338, -0.025232678, 0.013511566, 0.0378708, 0.03848343, 0.022714498, -0.024680793, -0.008462757, -0.014574636, 0.023904387, -0.007448843, 0.0077687018, -0.04321823, 0.022320602, 0.014373943, 0.08577145, 0.028573446, 0.03162947, 0.079125546, 0.07723836, 0.047792584, 0.013243279, 0.0253167, -0.03028161, -0.012152348, 0.0007505146, -0.038605034, 0.032488566, -0.02394028, 0.033529513, 0.09537326, 0.039628506, -0.008737912, 0.010879006, -0.039960258, 0.025126265, -0.060327616, 0.038962826, 0.019177048, 0.063604854, -0.0017345352, -0.008455257, 0.008800663, -0.023823045, 0.0010294339, 0.01460611, 0.008159556, 0.032981064, 0.08910335, -0.023378681, -0.002700438, -0.0084776515, -0.014194496, -0.05032008, -0.010500548, 0.03797832, -0.052485213, -0.016859492, -0.021329574, -0.036764633, -0.03391469, -0.037864137, 0.030099422, 0.06109096, -0.05197362, -0.013461265, 0.009364578, -0.0028626693, 0.020795465, 0.0016758789, 0.013723497, -0.008489808, -0.046119478, 0.025888853, 0.0045335335, 0.0044997353, -0.035563577, -0.00097277435, -0.007764648, 0.0099971965, 0.017420646, -0.03733101, 0.0023177601, 0.04998836, 0.020596368, -0.008602767, -0.06261901, -0.053756237, -0.006233781, 0.012853957, 0.029696133, 0.06489569, -0.080548316, 0.0011954382, 0.04386933, -0.022549842, -0.025756229, -0.025153007, 0.016473636, -0.061775997, -0.0036062591, -0.052893996, -0.0033645956, 0.0025911739, -0.0354912, -0.021296786, -0.0049578496, -0.009723892, 0.019947931, 0.015654868, -0.011591985, 0.037010554, -0.017878547, -0.00582528, 0.010578879, -0.016738638, 0.04027684, 0.030578919, 0.027637789, 0.01421785, -0.04417791, -0.0060184384, 0.03687135, -0.046730746, -0.04642275, -0.027020026, -0.024663454, 0.013994005, 0.028972233, -0.013539849, -0.010923012, 0.04937362, 0.058093403, -0.033093557, -0.004446155, 0.05781441, 0.014585675, -0.0101948045, -0.064537466, 0.0075767366, -0.019671215, -0.06659401, -0.0026016736, 0.015380555, 0.02411438, -0.034348246, 0.014561086, -0.03902613, -0.0139906965, -0.03168664, -0.034330413, 0.024511952, -0.112817965, 0.032417696, -0.018719798, -0.05043958, -0.0103687635, 0.03465242, 0.060490068, 0.020124033, -0.078843564, 0.001308025, -0.053245865, -0.022763545, -0.02139559, 0.03735931, 0.032511637, 0.012342246, 0.019464385, 0.0076219207, 0.028626412, 0.014582096, -0.012357209, -0.0064882142, 0.047774695, -0.032290578, 0.02484459, 0.056304857, 0.042326268, -0.039436046, -0.08119261, 0.044682946, -0.0021563896, 0.040265467, 0.036399405, 0.0410946, 0.036371965, 0.049223393, -0.05188147, 0.0129806325, -0.015184557, -0.041273043, -0.010013618, -0.02143192, 0.015909793, -0.006945864, 0.018761728, -0.009590044, -0.035413098, 0.012083478, -0.0050785867, 0.016649907, 0.03491653, 0.016336259, -0.008774716, 0.024679732, -0.031390473, -0.004790959, 0.032249358, 0.013988697, 0.05729381, 0.046889875, 0.011847253, -0.080560975, 0.050727826, 0.019729907, -0.040627114, -0.005202822, -0.055606462, -0.034668762, 0.02923332, 0.041719038, 0.007770019, -0.020708721, -0.02907915, -0.032801833, -0.05256542, -0.051490523, 0.02067707, 0.0326873, 0.006894497, 0.025088808, -0.027237853, -0.011512894, -0.021273285, -0.009388583, -0.011209785, -0.0017131915, -0.0021190296, 0.07216364, -0.0013129538, 0.069061734, -0.06474395, 0.025966913, 0.019519554, -0.03838686, 0.007905083, -0.01988511, -0.038216844, -0.005841453, 0.027462821, -0.07278075, -0.022328962, 0.02734593, 0.052941352, -0.0009798848, 0.009299564, -0.014813635, 0.013982492, -0.008348619, -0.043679696, 0.010329697, -0.017227199, 0.010499658, -0.025637288, 0.055605978, 0.03039472, -0.07217021, 0.01579936, -0.026567502, -0.062674165, -0.0014790847, -0.018996824, -0.027211595, 0.03482409, 0.009223147, 0.019511411, 0.048895393, 0.030188454, 0.0049228845, 0.0532175, 0.03694532, 0.04362496, -0.020647503, 0.0043141195, 0.0088069625, 0.023237102, -0.023833005, -0.050741483, 0.01909566, 0.047327474, 0.070040174, -0.029358214, -0.06061147, 0.0067298324, 0.0141942715, -0.010386815, 0.017125133, -0.023783553, 0.015236323, -0.01740756, -0.0056256056, 0.00357327, -0.04910201, 0.014712581, -0.027846444, 0.028824378, 0.0058737616, -0.0064475797, 0.063496575, 0.025007023, 0.009075108, -0.014722366, -0.017214185, -0.004045941, -0.009011644, -0.017264782, 0.020096825, 0.009501295, -0.03275829, -0.038627304, 0.050830785, 0.0237389, 0.0067473655, 0.044562086, -0.0072219917, 0.039214004, 0.016432792, 0.002697602, 0.01936228, 0.021932079, 0.009904801, 0.0018933407, 0.018377187, -0.031552054, -0.034091122, 0.043246187, -0.010147959, 0.018935807, 0.018248482, 0.04796861, -0.02726615, -0.060525652, -0.024159484, -0.049189217, 0.065707915, 0.027796863, 0.009987276, -0.018714612, 0.015525749, -0.094883256, -0.028654775, -0.03268434, 0.02326122, 0.0047541303, 0.005424112, 0.05408284, -0.020326713, 0.0029418543, -0.061265014, 0.0023763443, -0.011163637, 0.00027694015, 0.008298283, 0.037613552, 0.026374046, -0.015731283, 0.012003395, -0.0018223721, 0.01733744, -0.013671909, 0.053295672, -0.0085054645, -0.013553342, -0.014815456, 0.0010086424, 0.02932319, -0.028405262, 0.02648875, -0.0019984501, -0.030139918, 0.0066717565, -0.074200384, -0.00082579546, -0.03788137, 0.016546283, 0.014469636, -0.041573565, 0.014768847, 0.048441906, -0.0229561, -0.07434068, 0.02624827, -0.055282746, -0.011444125, 0.030702641, -0.0040901247, -0.027663369, 0.043003645, -0.027503515, -0.030775892, 0.054818925, 0.0076672235, 0.0058906386, -0.06256432, 0.03803501, 0.05782031, -0.0074083596, -0.02732593, -0.014384261, 0.014559447, 0.020135818, 0.008831414, -0.036149014, -0.008842212, -0.031503562, -0.0035714924, 0.008464324, -0.041444063, -0.049377233, 0.023688035, 0.024294782, -0.015798554, 0.068773486, 0.041664105, -0.027836312, -0.05769646, 0.10535017, -0.018578747, -0.048650634, -0.033943735, 0.012195666, 0.013794228, 0.007375559, 0.006558913, 0.03028655, 0.031346913, 0.032102544, 0.015212544, -0.047056682, -0.03895633, 0.05020355, 0.008692916, 0.025327034, 0.020505724, -0.0031781953, 0.028987793, 0.013090643, -0.04100184, -0.01330123, 0.065087736, -0.007792441, 0.011275172, 0.011950567, 0.039780706, -0.010193511, 0.028074007, 0.032920916, -0.009486311, 0.053533226, -0.0143349925, 0.05587921, -0.0049194973, 0.00895221, 0.009317551, -0.06350355, -0.019005748, 0.00017087512, -0.0027812044, 0.030467179, 0.012287833, -0.0048773326, -0.016345395, 0.014010801, -0.009650713, -0.033069227, 0.05199222, -0.022508832, 0.026915751, -0.0059237746, -0.06449991, -0.059447147, 0.020208929, 0.04902009, 0.044918224, 0.043346744, -0.013743072, -0.00265391, 0.028749233, 0.025065886, 0.0077623595, -0.034190893, 0.0070366585, 0.012191028, -0.0149351265, 0.01937088, 0.0268846, -0.027867192, -0.042951882, -0.015446939, -0.00560963, -0.015101582, -0.053330254, -0.026021305, 0.028395927, 0.013004832, -0.034744825, 0.045878466, 0.004479989, 0.026543245, -0.05334198, -0.009087808, 0.032807223, -0.043319605, -0.03260215, 0.038798064, 0.031704895, -0.029170511, 0.02212051, 0.03159859, 0.008861513, -0.010982833, -0.015952101, -0.0061396365, 0.006693252, 0.03685477, 0.0032489963, -0.022487951, 0.042037737, -0.018716915, -0.025958333, 0.045221284, -0.015513015, 0.063151754, -0.030507054, 0.017254626, 0.005142329, 0.023579448, 0.0038528352, -0.027187478, 0.03638105, -0.0060239383, -0.011738006, -0.17054205, 0.017579285, -0.03394865, 0.0005166473, -0.02053941, 0.02616181, -0.000803231, -0.012759502, 0.04088744, -0.05062309, 0.013646933, -0.0012324008, 0.008791449, 0.029120648, -0.02099542, -0.04747789, 0.01503862, -0.0069903857, 0.05318538, -0.030782493, -0.011116818, 0.02928989, -0.027050706, 0.04481006, 0.017402014, -0.023222549, 0.006652645, -0.020090295, -0.024252808, -0.0073200935, -0.0022071472, 0.055031642, -0.00740071, 0.0070644002, 0.00975761, 0.021836216, -0.060212888, 0.008957903, -0.029012976, 0.031938586, 0.022586484, -0.051117484, -0.019314123, 0.02212107, 0.022286385, 0.052936107, 0.015520078, 0.019945484, -0.017580401, -0.025035389, 0.03229371, -0.058282852, -0.030969277, 0.011856393, -0.073136255, -0.030528687, -0.027031949, -0.0066383206, -0.05499542, -0.019549126, 0.022830036, -0.016121166, 0.02137481, -0.042628683, -0.0020061648, -0.030976573, -0.034418184, -0.04401557, 0.021999035, 0.041957136, 0.04944364, -0.002846621, -0.0063519115, -0.0065475013, -0.049452588, -0.035580825, 0.009701199} + testVectorSimilarGemma300m := []float32{ // I love sports + -0.19343194, 0.004856487, -0.0014202092, -0.034982327, -0.042895004, -0.03478415, -0.021518571, 0.076288305, 0.021327905, -0.08115373, -0.018364046, -0.046180096, 0.05580578, -0.013486628, 0.12047895, 0.028545877, -0.0010285492, -0.05813281, -0.052106522, -0.028816907, -0.006889241, 0.017485179, -0.03893138, 0.0058361324, 0.039129637, 0.022666205, -0.008147375, 0.0042858617, 0.037214737, -0.013728564, 0.022394637, -0.031806376, -0.014492319, -0.0060505946, 0.0046134074, 0.023479525, 0.009634952, -0.03694791, 0.020696364, -0.010995064, -0.12716065, 0.12423946, 0.0045062704, 0.0063163047, -0.004659048, -0.015114171, -0.027970241, -0.023592342, -0.01493156, -0.004219005, 0.030313857, -0.0014878536, -0.05279661, -0.00031150063, -0.057897333, -0.017468948, -0.026589166, 0.028147256, 0.01997812, 0.054668825, -0.044135034, 0.0017772043, 0.0038535267, -0.01680548, 0.03619409, -0.007872826, 0.02353458, 0.011765093, 0.049635943, 0.17513552, -0.006134462, 0.0024269389, -0.004895456, 0.01940363, 0.14693989, 0.014233306, -0.024014022, -0.025952553, -0.022265725, 0.019483108, 0.05030047, -0.004645346, -0.019672848, -0.05309829, 0.07238553, -0.083060116, -0.007971565, -0.047031052, -0.008420404, -0.03436547, -0.0089469515, 0.031068087, -0.020502402, -0.008010455, 0.03185636, -0.06224184, -0.006626665, 0.038336184, -0.015114512, -0.03026035, -0.04493791, 0.024951058, 0.052914113, 0.15853235, -0.016719542, -0.021242416, -0.025095126, -0.00809016, 0.015759498, 0.024955342, 0.023662813, -0.05580142, 0.06161405, -0.09621664, -0.003219709, 0.034229737, -0.021975461, -0.0053198445, -0.0034411338, 0.045536622, 0.011735175, 0.0035118712, -0.008963017, 0.015073859, 0.019780207, 0.032661304, -0.021795616, 0.0017255745, 0.024568968, 0.019012863, -0.008230681, 0.037154622, 0.041309826, 0.067678444, -0.026305677, 0.021535365, -0.016851775, -0.0152222505, 0.045788057, 0.057052888, -0.038303558, -0.106455006, -0.021198286, -0.07732979, 0.028994994, 0.017056191, 0.025309308, 0.023671629, 0.015379445, -0.005398609, 0.023989787, 0.044273213, 0.05828306, 0.011676626, -0.03484274, -0.035037562, -0.017991228, 0.03283462, -0.02003847, 0.0016254452, -0.016071863, 0.020303126, -0.023787, 0.094560616, 0.0025399597, -0.004472838, 0.04458538, 0.06905149, 0.027215045, 0.0066002845, -0.009430449, -0.030544672, -0.00013865117, -0.0027440996, -0.031789605, 0.016110262, -0.0030023127, 0.029577088, 0.08732629, 0.015798338, 0.0032837058, -0.020259963, -0.044099357, 0.046423882, -0.06579622, 0.004351193, 0.024269303, 0.042842742, -0.0011130862, -0.024093682, -0.0039447034, -0.022359395, 0.024692332, 0.012061032, 0.020809356, 0.015261888, 0.083863884, -0.012388093, 0.036941, -0.0051759467, 0.010397039, -0.05126886, -0.013456974, 0.051382013, -0.02835908, -0.022303138, -0.009831453, -0.03233279, 0.013973325, -0.0075743273, 0.036757275, 0.04820738, -0.06389594, -0.002037313, 0.013740722, 0.00043125064, 0.000553012, -0.004455251, 0.01583632, 0.015635338, -0.024908446, 0.015284511, -0.023797033, -0.012129262, -0.06640167, 0.006065103, -0.013993358, 0.007050734, 0.0017448603, -0.04772027, -0.010784726, 0.036574334, 0.028343702, -0.039030306, -0.027407106, -0.0014757065, 0.00625449, 0.028657254, 0.014299372, 0.03674544, -0.0783059, 0.02149714, 0.014071517, -0.02553595, -0.030477399, -0.037762143, 0.031486116, -0.041232266, 0.014259505, -0.057757962, 0.017944107, 0.0051843254, 0.006666434, -0.037803575, -0.0296561, 0.0019770458, 0.027963303, 0.017541502, -0.012126993, 0.009756087, 0.004347292, 0.0042871768, -0.021956902, -0.054240514, 0.02735245, 0.025096968, 0.005636328, 0.017072977, -0.03227497, 0.0013510277, 0.034683112, -0.042369954, -0.050987575, -0.017900253, -0.01868916, 0.027188914, -0.0025845054, -0.031895738, -0.031405177, 0.02554241, 0.046531547, -0.01761107, 0.013156642, 0.072531156, 0.02980141, -0.00087580323, 0.010031103, 0.005241903, 0.0013325217, -0.053635355, 0.017545294, -0.0087774955, 0.022754954, 0.0058755176, 0.04333989, -0.021393511, -0.011524931, -0.036289275, -0.026273437, 0.0033234956, -0.110892735, 0.044619057, -0.012643462, -0.03960745, -0.02331842, 0.017553236, 0.0645598, 0.027876887, -0.076686434, -0.04675943, -0.03927277, -0.0154067995, -0.032418218, 0.038411193, 0.012076592, -0.0016886882, 0.029466614, -0.0022331316, 0.035377674, 0.013796891, -0.005605323, 0.0011327396, 0.01822024, -0.008982852, -0.009792097, 0.025394138, 0.03587774, -0.040699065, -0.07788869, 0.046736926, -0.0306519, 0.04964376, 0.048529863, 0.070256196, 0.04131678, 0.04408903, -0.078293994, 0.00293659, -0.011773447, -0.015023819, 0.0055670207, -0.013250244, 0.015544129, -0.017794935, 0.058797315, -0.01772969, -0.019127075, 0.006285778, 0.011321112, 0.017023705, 0.02131865, 0.020815566, -0.01609575, 0.030654175, -0.053676542, 0.04695926, 0.04109149, -0.016295958, 0.04661722, 0.028767712, -0.0052141896, -0.07589216, 0.039310485, 0.007456763, -0.02754816, -0.042905826, -0.024375219, -0.018164845, 0.041079573, 0.054144345, -0.013629608, -0.03297385, -0.025355512, -0.061367005, -0.059703227, -0.028251009, 0.042787436, 0.042598177, 0.018997505, 0.019697476, -0.009417489, -0.04065503, -0.029777762, -0.04377524, 0.009216185, -0.04538888, 0.011838789, 0.051156387, 0.0023259886, 0.050309237, -0.048539735, 0.026938692, 0.03522926, -0.014742535, -0.00071712927, -0.023006592, -0.037686184, 0.01210538, 0.006555123, -0.07747232, -0.022557113, 0.018786257, 0.068009704, 0.008337162, 0.033578936, -0.029785099, 0.016466253, 0.007817518, -0.030884616, 0.012660635, -0.024243841, -0.0049353605, -0.028811675, 0.058977775, -0.000027494478, -0.055797398, -0.011272795, -0.004514563, -0.059993695, -0.004707711, -0.034156643, -0.007128923, 0.05746211, 0.033852138, 0.025453582, 0.080071315, 0.024052141, 0.023689957, 0.048191514, 0.008009531, 0.028455708, 0.023041163, -0.022618847, 0.0031471949, 0.016035846, -0.014400845, -0.012957847, 0.02400062, 0.038235173, 0.094114095, -0.04917773, -0.07413769, 0.011682055, 0.008769331, -0.021496663, 0.008728028, -0.030536607, 0.0006690131, -0.017194578, 0.011936575, 0.010094599, -0.042898107, 0.018950386, -0.02937901, 0.014039939, -0.01698565, -0.02066156, 0.0373574, 0.009998972, 0.001802131, -0.008044042, 0.003914921, 0.008116994, 0.0014278024, -0.02694805, 0.03151597, -0.013771035, -0.019928807, -0.04330614, 0.043885667, -0.012527679, 0.0026973744, 0.04534283, 0.006216817, 0.024060806, -0.0022186572, 0.0065589943, 0.012639574, 0.023651889, 0.013968561, 0.013339394, -0.050039265, 0.01702512, 0.008877593, 0.026182089, -0.02889301, 0.007269859, 0.035490654, 0.02787452, -0.015825234, -0.0623267, -0.027962267, -0.035724636, 0.016807783, 0.03238781, -0.0031108335, -0.0031055498, -0.013057319, -0.1371084, -0.028632041, -0.013401502, 0.0322828, -0.004034702, 0.0139085, 0.05223514, -0.023679541, -0.018288296, -0.059301626, -0.022060638, -0.0052208686, 0.015364536, 0.024212083, 0.04524196, 0.011842322, -0.048383873, 0.01405416, -0.011845293, 0.02627249, 0.036867995, 0.05105408, -0.0077107023, -0.004397384, -0.035618074, -0.013664328, -0.033662252, -0.0016344389, -0.0042840918, -0.03220223, -0.0150563745, -0.017037738, -0.018130936, 0.008116968, -0.019788342, -0.008171407, -0.016482305, -0.02400652, 0.005815669, 0.020557173, -0.039122712, -0.053071667, 0.028574789, -0.039765667, 0.0059386767, 0.016884543, -0.0035789448, -0.047877137, 0.029186647, -0.01890302, -0.02593091, 0.033793293, 0.012029369, 0.009233877, -0.008896448, 0.020889392, 0.028819507, 0.0027430528, -0.014662365, -0.037347604, 0.013889639, 0.059220634, 0.0037976506, -0.047773235, 0.017435005, -0.004968436, -0.004422793, 0.05192342, -0.033757728, -0.042297803, 0.013030133, 0.031305652, -0.0019159089, 0.06606626, 0.034871552, -0.044141795, -0.035944957, 0.090333775, -0.03483784, -0.073589124, -0.0012299955, 0.033732604, -0.0030162495, 0.024373392, 0.015051603, 0.013084365, 0.010935514, 0.040480785, 0.014048396, -0.06276354, -0.035487622, 0.01729997, 0.042335246, -0.0036487868, -0.00082442485, 0.012274049, 0.010781612, 0.018644968, -0.021897381, -0.005642314, 0.0046939226, -0.03440462, -0.022529304, -0.008515985, 0.025201347, -0.0111573255, 0.024483006, 0.017403616, -0.03268361, 0.051266085, -0.012837571, 0.043969687, -0.0034147927, -0.026267864, 0.003265275, -0.05500881, -0.01661759, 0.0018414019, 0.0015231966, 0.05702346, 0.019364413, -0.02586545, 0.021653125, 0.029588293, 0.0077850185, -0.0430362, 0.054687977, -0.015070888, 0.036050543, -0.009546, -0.056619074, -0.059146564, 0.036447756, 0.05653066, 0.033108942, 0.039168853, -0.032555796, -0.007815384, 0.007881921, 0.010634688, -0.025651999, -0.017045695, 0.0076846895, -0.008805362, 0.037417904, 0.008402284, -0.015121615, -0.010847883, -0.03451446, -0.0046214983, 0.008966725, -0.02883603, -0.012423276, -0.0033418261, 0.004659408, 0.026335558, -0.02825116, 0.05429816, -0.022009708, 0.021676816, -0.056272957, -0.047923394, 0.024098612, -0.05631526, -0.019464215, 0.06373419, 0.04063563, -0.03038542, -0.008974301, 0.031870365, 0.019730875, -0.010265835, -0.041838143, -0.008599757, 0.017530069, 0.040771917, 0.008802026, -0.008744444, 0.008521846, -0.040848725, -0.034978814, -0.0030485387, 0.032500513, 0.012951092, -0.0072883945, 0.053486913, -0.0020134859, 0.0051054927, -0.0073940502, 0.017484246, 0.003429821, -0.002419117, -0.046042033, -0.17463663, 0.03856373, -0.015983624, 0.050707515, -0.026031772, -0.010148257, 0.011725887, 0.0016998255, 0.036961447, -0.07102596, 0.00970325, 0.0015294887, -0.027086413, 0.037476, -0.029566403, -0.028614618, 0.050893623, 0.008336737, 0.044098064, -0.03156252, -0.02855941, 0.022602454, -0.011040451, 0.023424903, 0.0142049175, -0.047183294, -0.0064510517, -0.019843563, -0.041868143, -0.04886951, -0.008193578, 0.044128478, -0.0011999609, -0.028832013, 0.016546667, 0.011064687, -0.044159926, 0.05557552, -0.027433444, 0.06320927, 0.023127085, -0.04544467, 0.0009640562, 0.03069771, 0.033332612, 0.030118337, 0.036347955, 0.016979406, -0.05468095, -0.029338872, 0.040686868, -0.030930685, -0.03105952, -0.012754343, -0.03453242, -0.022945372, -0.022502676, -0.016581576, -0.03626855, -0.004950012, 0.027985564, -0.009342826, 0.022715006, -0.01765263, -0.0097409375, -0.050807662, -0.041757226, -0.033696514, 0.023245256, 0.029898118, 0.036003124, 0.007729071, -0.003491109, 0.001163647, -0.02175583, -0.015647098, 0.015489585} + testVectorDifferentGemma300m := []float32{ // I like painting + -0.1926697, -0.035642363, 0.031817686, 0.029955518, -0.059222676, 0.020839859, -0.03217778, 0.023262957, 0.04454794, -0.038457688, -0.013782364, -0.029629003, -0.0015514725, -0.0025953925, 0.08594901, 0.012543971, 0.022877285, -0.0539877, 0.0071293185, -0.06114372, -0.02399481, 0.017347233, -0.013431345, -0.020632474, 0.0084933415, 0.05489676, 0.020937936, 0.020416124, 0.030836469, -0.01780731, 0.042039126, 0.012047507, 0.05086373, 0.012042791, -0.023956, 0.053949486, -0.012623748, -0.034927554, 0.025583144, -0.04811404, -0.054478996, 0.05731774, 0.03316636, -0.00050957035, 0.035738435, -0.023439184, -0.006801935, -0.082153425, 0.014538397, 0.018407045, 0.0075377375, -0.027926631, -0.070389025, -0.008094685, -0.06977042, 0.009677597, -0.015956992, 0.027900478, 0.0020678404, 0.048942853, -0.041446097, 0.022576027, 0.051491406, -0.0014917557, 0.008625844, -0.0043387655, -0.0047612684, 0.07140532, 0.042618718, 0.17041971, -0.028972447, -0.0047441893, -0.025096409, -0.030032141, 0.16886425, 0.058096644, 0.037717335, -0.030105436, -0.03797992, 0.009790895, 0.008489862, 0.022140607, -0.024602605, -0.0381059, 0.070894934, -0.014602448, -0.006761332, -0.021940302, -0.013133459, -0.009903259, -0.023793485, 0.022082023, -0.026805475, -0.043964736, 0.027707351, -0.028255658, -0.0001976983, -0.03216062, -0.031867422, -0.0058394535, -0.0033501335, 0.03038103, 0.026945448, 0.09971729, -0.015736403, 0.001220989, 0.005006808, -0.018091204, -0.03810166, 0.026438005, -0.048983816, -0.051132087, 0.028618334, -0.020311527, 0.017388672, 0.050980616, -0.024517052, 0.017962899, -0.00392045, 0.04583897, 0.022590727, 0.0676565, 0.021320425, -0.004317383, 0.021294191, 0.02129455, -0.07311426, 0.034948383, 0.033541717, 0.0098310625, -0.0060448977, 0.007439606, 0.034033414, 0.022343596, 0.0027079964, 0.022199474, -0.0070422986, 0.016271569, 0.05499021, 0.009954112, -0.0056403982, -0.12420096, -0.02735069, 0.0069549717, 0.060775198, -0.017042397, 0.0058350833, -0.0062244474, 0.05905691, -0.0031366863, -0.007791399, 0.031078883, 0.010323441, -0.0014730418, 0.0053147296, -0.013072945, -0.055407453, 0.047932595, -0.018930543, -0.044017296, 0.037597846, 0.039863963, -0.018900512, 0.0845186, -0.010417474, 0.023395626, 0.016810363, 0.056994416, 0.012837578, -0.021924224, 0.033831127, -0.02382465, 0.04877439, -0.011518865, -0.04152086, -0.0044305236, -0.008469993, -0.02028049, 0.027489573, 0.016489038, -0.033911534, 0.045574833, -0.059043277, 0.018683476, -0.06130138, 0.02324724, 0.007346426, 0.05769748, 0.0049307104, -0.040081672, -0.0003312317, -0.028875068, -0.01370211, 0.003007982, -0.0016287845, 0.010707911, 0.10469356, -0.049904805, -0.022174763, 0.006582559, -0.01154778, -0.014841459, -0.004346105, 0.0018666037, -0.009765023, -0.010002347, -0.04491407, -0.04703042, -0.010302926, -0.01265582, 0.03952893, 0.027985346, -0.06924281, 0.0065133274, 0.033925395, 0.024410013, -0.008306433, -0.0078027085, -0.025789358, -0.0068259803, -0.010741692, 0.012609999, 0.010119814, 0.0077549014, -0.021466093, 0.017929932, -0.040093187, 0.012494919, -0.013709428, -0.02208369, 0.006872704, 0.04263633, 0.037000258, 0.007948924, -0.03941538, -0.045898322, -0.02609465, -0.024663381, 0.05756676, 0.067238495, -0.119688906, -0.019630732, 0.02991954, -0.03472285, -0.017943986, -0.033447362, 0.023779642, -0.06577707, 0.0067370557, -0.0687303, 0.0065962053, 0.0061039454, -0.016629, -0.04510535, -0.015211621, 0.029251361, 0.023134531, 0.011547867, -0.0040699355, 0.006923512, 0.0037884016, -0.028313199, 0.008462343, -0.009778717, 0.006920203, 0.0063349097, 0.02374252, 0.045654148, -0.08480115, -0.0011817855, 0.041155607, -0.055951916, -0.05476447, 0.029237539, -0.045470353, -0.014420287, 0.02055561, -0.010190379, -0.005286857, 0.025653085, 0.06773997, -0.046298243, 0.013415093, 0.050633736, -0.0024975694, 0.05613517, 0.0035553428, 0.02440465, 0.008124254, -0.026994761, -0.030983595, -0.021859322, 0.009971498, -0.017806763, -0.008170644, -0.016167859, 0.0068967952, 0.017938545, -0.020570483, -0.00037582143, -0.090453446, -0.010543305, 0.019172495, -0.06319688, -0.01383554, 0.04169752, 0.065818444, -0.009210325, -0.04513292, -0.0010278624, -0.0074894107, 0.013387586, -0.0396858, 0.01710703, 0.030226326, -0.015481609, 0.019845698, -0.04502317, -0.008714529, 0.039976515, 0.012067466, -0.03146507, 0.019926514, -0.05819628, 0.008721303, 0.00021206481, 0.043529697, -0.032725442, -0.105369724, 0.02372145, -0.014628045, -0.008877653, 0.04057033, 0.052441, -0.004081206, 0.009773011, -0.007339222, -0.021657936, -0.056223307, -0.06412481, -0.054160483, -0.0068936185, -0.015623868, -0.031800307, 0.041017875, 0.00816558, -0.020093054, 0.038469817, -0.012630343, 0.014701592, 0.055558413, -0.032229923, -0.028807394, 0.030902434, -0.029458439, 0.035793405, 0.01324962, 0.000034907163, 0.0518265, 0.026117068, 0.039631568, -0.10404563, 0.050610237, 0.013882799, -0.03076369, -0.0023800754, -0.029208858, -0.084563866, 0.012986312, -0.008206684, -0.018768065, -0.00042428653, -0.03791216, -0.033074964, -0.05044729, -0.073135406, 0.002787554, 0.032591213, 0.01970521, 0.05624398, 0.0096798325, 0.014982218, -0.014708999, 0.008047497, -0.02731766, -0.027414491, -0.012358737, 0.03259815, 0.0047817836, 0.05117304, -0.047619436, 0.033420604, 0.03471258, -0.018328367, -0.01867124, -0.019547088, -0.011595926, 0.0032670524, -0.021058448, -0.039692648, -0.04287113, -0.000016755945, 0.04851344, 0.031761795, 0.024851546, -0.017080046, 0.019438002, -0.018958092, -0.038033366, 0.0017133867, 0.021565616, 0.0028715723, -0.04642468, -0.020290326, 0.007632987, -0.02535685, -0.012915251, -0.011563421, -0.06687321, -0.045728583, -0.052118458, -0.010069207, 0.046504967, 0.04738205, 0.025347108, -0.01327845, 0.044743847, 0.05591417, 0.013724283, 0.017814584, 0.052208632, -0.0144258775, -0.02661419, -0.025696425, 0.013761021, -0.074994445, 0.0033880514, -0.008660243, 0.028923746, 0.055158827, -0.06914018, -0.10797683, -0.0011902245, 0.014914085, -0.017377062, 0.026699718, -0.046571605, 0.01658393, -0.020768005, -0.025582785, 0.0034914429, 0.012487686, 0.0037895092, -0.016979583, 0.0069164103, -0.037476525, -0.033620365, 0.069075905, 0.0029506728, -0.047139205, 0.009646034, 0.0013344728, 0.027345218, 0.012153301, -0.06948698, 0.011463745, -0.01006065, 0.010475116, -0.05344952, 0.05855788, -0.031346325, 0.002176756, 0.065583065, -0.004646086, 0.020427255, -0.010421923, -0.019812837, 0.014603378, 0.018015128, 0.026288904, -0.00062973774, 0.024565792, -0.0023929349, -0.06082387, 0.023625651, -0.015958495, -0.003404223, 0.012712491, 0.019922081, -0.0048192297, -0.07975315, -0.039972965, -0.0057269167, 0.048303705, 0.017163005, -0.024014894, -0.008115678, -0.016941702, -0.09436062, -0.037066765, -0.01934297, -0.0067856866, 0.053709816, 0.020500178, 0.065878935, -0.06292582, -0.05333875, -0.027347548, -0.050865293, -0.004804837, -0.02955358, -0.035872925, 0.056049988, 0.05063509, -0.03033497, 0.018578852, -0.019671759, -0.032566875, -0.009953692, 0.009492768, 0.04184148, -0.0007386845, -0.044894844, 0.05024644, 0.027048644, 0.00034926314, -0.009528811, -0.0073785293, 0.0072666765, -0.04707812, -0.053439926, 0.014036265, 0.000112570124, 0.008990334, 0.017077168, -0.029458689, 0.01938296, 0.026302593, -0.0400524, -0.016178224, 0.024995584, -0.009259373, 0.013555074, 0.018205203, -0.015784208, -0.04355028, 0.038281478, 0.0068848, 0.021030333, 0.054626204, 0.016500693, -0.009381844, -0.08090096, 0.031484563, 0.06740883, 0.013055411, -0.036041055, 0.0052926484, -0.0012102873, 0.066706836, 0.03997172, -0.0018622348, -0.011409159, -0.0075420733, 0.008227084, 0.007555531, 0.0018359828, -0.034758817, 0.024863677, 0.046809863, 0.011756619, 0.089938425, 0.013297065, 0.005511697, 0.00795666, 0.06314317, -0.04556482, -0.04575526, -0.012017404, 0.017150609, 0.009820648, -0.014716402, 0.0425081, -0.003504213, 0.030636765, 0.010859593, -0.011091928, -0.017676145, 0.00696458, -0.0013607097, -0.051944118, 0.008481624, 0.011871197, -0.043759435, 0.019282268, 0.007915233, 0.00066617347, -0.039848946, 0.08665708, 0.014774085, 0.005046155, 0.02529792, 0.03290366, 0.01441463, 0.0025122142, 0.018255001, -0.017934302, 0.029905617, 0.011688054, 0.062003147, 0.045690123, 0.020135969, 0.020229999, -0.018070715, -0.013267489, -0.0041150907, 0.013052246, -0.021474108, 0.03837032, -0.008268676, -0.022869095, 0.010455993, 0.024507781, -0.036975656, 0.052558165, -0.018306263, 0.011121863, 0.028138937, -0.007675367, -0.057565354, 0.007660535, 0.038151264, 0.0040204735, -0.0065488075, -0.018040214, 0.020441415, 0.036678903, 0.060007375, -0.04068517, -0.019969257, -0.026102042, -0.0060978252, -0.009270404, 0.02319634, 0.026101105, -0.024973158, -0.034084387, 0.013564288, -0.04005809, 0.005125816, -0.032981362, 0.015466502, 0.015119961, 0.034474876, -0.012335972, -0.0034615838, -0.0497132, 0.00054067635, -0.04277186, -0.01846649, 0.006053916, -0.025857633, -0.0068883966, 0.091375045, -0.002938799, -0.03467201, 0.02381599, 0.01958393, -0.0024987685, 0.0020010495, -0.013350033, -0.05080437, 0.03227389, 0.034332823, -0.02435927, -0.01578944, -0.012705026, 0.0012791384, -0.027679777, 0.069177106, -0.0077255378, 0.063896894, 0.000017430384, 0.03139062, -0.02148748, 0.0052214894, 0.013935673, -0.033864684, 0.011008822, 0.015615474, 0.010834747, -0.15180886, 0.012295562, -0.013116554, -0.024834225, -0.03265783, 0.0020440016, -0.013084595, 0.01855664, 0.03853777, -0.03912196, 0.018408498, 0.0004921446, -0.034448866, 0.0028652, 0.008604641, -0.015923971, 0.027590593, -0.030997323, 0.047571156, -0.026510201, -0.005421846, 0.035658535, -0.034196235, 0.041060053, 0.03375925, -0.0068177977, 0.0011750234, -0.03352342, -0.015317193, -0.0040746816, 0.0046356325, -0.001307895, 0.00474877, 0.003052953, 0.036096707, -0.012817789, -0.106328934, -0.0006774031, -0.052006587, 0.016179409, 0.0039911037, -0.05843669, 0.010192754, 0.03271481, -0.0053806966, 0.038259655, 0.030500714, 0.040700536, -0.016013231, -0.0622162, 0.012646027, -0.0041393293, -0.005526243, -0.037090223, -0.066543445, -0.031290423, 0.020217495, 0.030126689, -0.035782844, -0.045914304, -0.016469074, 0.015082676, 0.02176836, 0.015028209, -0.012254033, -0.036303442, -0.032566965, -0.0468213, 0.042742345, 0.054322172, 0.029375346, 0.044665392, -0.0027555292, -0.046337903, -0.023748377, 0.020291269, 0.041817993} + */ + quantize := func(input []float32) []int8 { + // Find max absolute value + var maxAbs float32 + for _, v := range input { + abs := v + if abs < 0 { + abs = -abs + } + if abs > maxAbs { + maxAbs = abs + } + } + + // Quantize with scaling factor + quantized := make([]int8, len(input)) + scale := float32(127.0) + if maxAbs > 0 { + scale = 127.0 / maxAbs + } + + for i, v := range input { + scaled := v * scale + scaled = float32(math.Round(float64(scaled))) + if scaled > 127 { + quantized[i] = 127 + } else if scaled < -128 { + quantized[i] = -128 + } else { + quantized[i] = int8(scaled) + } + } + return quantized + } + + // Convert []int8 to []byte for SQLite + quantizedInput := quantize(inputVectorAllMiniLm) + const metadataValue = 342 + err = sqlitex.ExecScript(conn, `CREATE VIRTUAL TABLE embeddings USING vec0( + allMiniLm int8[384] distance_metric=cosine, + meta int);`) + require.NoError(t, err) + err = sqlitex.Exec(conn, `INSERT INTO embeddings(rowid, allMiniLm, meta) VALUES (?, vec_int8(?), ?)`, + func(*sqlite.Stmt) error { return nil }, 1, quantizedInput, metadataValue) + require.NoError(t, err) + + quantizedInputBGEM3 := quantize(inputVectorMiniLML12v2) + err = sqlitex.ExecScript(conn, `CREATE VIRTUAL TABLE vec_BGEM3 USING vec0(embedding int8[384] distance_metric=cosine, meta int);`) + require.NoError(t, err) + err = sqlitex.Exec(conn, `INSERT INTO vec_BGEM3(rowid, embedding, meta) VALUES (?, vec_int8(?), ?)`, + func(*sqlite.Stmt) error { return nil }, 1, quantizedInputBGEM3, metadataValue) + require.NoError(t, err) + + queryAllMini := ` + SELECT + rowid, + distance, + meta + FROM embeddings + WHERE allMiniLm MATCH vec_int8(?) + AND k=1 + ORDER BY distance + ` + + queryBGEM3 := ` + SELECT + rowid, + distance, + meta + FROM vec_BGEM3 + WHERE embedding MATCH vec_int8(?) + AND k=1 + ORDER BY distance + ` + + var similarity float32 + var rowid int64 + var meta int64 + testVector := quantize(testVectorSimilarAllMiniLm) + err = sqlitex.Exec(conn, queryAllMini, + func(stmt *sqlite.Stmt) error { + rowid = stmt.ColumnInt64(0) + distance := float32(stmt.ColumnFloat(1)) + similarity = max(0, 1-distance) + meta = stmt.ColumnInt64(2) + return nil + }, testVector) + require.NoError(t, err) + require.Equal(t, int64(1), rowid) + require.InDelta(t, .8, similarity, 0.1) + require.Equal(t, int64(metadataValue), meta) + + testVector = quantize(testVectorDifferentAllMiniLm) + err = sqlitex.Exec(conn, queryAllMini, + func(stmt *sqlite.Stmt) error { + rowid = stmt.ColumnInt64(0) + distance := float32(stmt.ColumnFloat(1)) + similarity = max(0, 1-distance) + meta = stmt.ColumnInt64(2) + return nil + }, testVector) + require.NoError(t, err) + require.Equal(t, int64(1), rowid) + //require.InDelta(t, 0.5, similarity, 0.1) + require.Equal(t, int64(metadataValue), meta) + + testVector = quantize(testVectorSimilarMiniLML12v2) + err = sqlitex.Exec(conn, queryBGEM3, + + func(stmt *sqlite.Stmt) error { + rowid = stmt.ColumnInt64(0) + distance := float32(stmt.ColumnFloat(1)) + similarity = max(0, 1-distance) + meta = stmt.ColumnInt64(2) + return nil + }, testVector) + + require.NoError(t, err) + require.Equal(t, int64(1), rowid) + //require.InDelta(t, .9, similarity, 0.1) + require.Equal(t, int64(metadataValue), meta) + + testVector = quantize(testVectorDifferentMiniLML12v2) + err = sqlitex.Exec(conn, queryBGEM3, + + func(stmt *sqlite.Stmt) error { + rowid = stmt.ColumnInt64(0) + distance := float32(stmt.ColumnFloat(1)) + similarity = max(0, 1-distance) + meta = stmt.ColumnInt64(2) + return nil + }, testVector) + + require.NoError(t, err) + require.Equal(t, int64(1), rowid) + //require.InDelta(t, 0.4, similarity, 0.1) + require.Equal(t, int64(metadataValue), meta) + + testVector = quantize(testVectorVeryDifferentMiniLML12v2) + err = sqlitex.Exec(conn, queryBGEM3, + + func(stmt *sqlite.Stmt) error { + rowid = stmt.ColumnInt64(0) + distance := float32(stmt.ColumnFloat(1)) + similarity = max(0, 1-distance) + meta = stmt.ColumnInt64(2) + return nil + }, testVector) + + require.NoError(t, err) + require.Equal(t, int64(1), rowid) + //require.InDelta(t, 0.4, similarity, 0.1) + require.Equal(t, int64(metadataValue), meta) + +} + func TestBase58BTC(t *testing.T) { pool, err := OpenSQLite("file::memory:?mode=memory&cache=shared", 0, 1) require.NoError(t, err) diff --git a/backend/storage/storage_migrations.go b/backend/storage/storage_migrations.go index 6ca6a5763..7f8fe59c6 100644 --- a/backend/storage/storage_migrations.go +++ b/backend/storage/storage_migrations.go @@ -63,6 +63,19 @@ type migration struct { // // In case of even the most minor doubts, consult with the team before adding a new migration, and submit the code to review if needed. var migrations = []migration{ + // delete content of embeddings table before reindexing with new schema + {Version: "2026-01-24.1", Run: func(_ *Store, conn *sqlite.Conn) error { + // Drop first to make idempotent (vec0 doesn't support IF NOT EXISTS). + if err := sqlitex.ExecScript(conn, "DROP TABLE IF EXISTS embeddings;"); err != nil { + return err + } + return sqlitex.ExecScript(conn, sqlfmt(` + CREATE VIRTUAL TABLE embeddings USING vec0( + multilingual_minilm_l12_v2 int8[384] distance_metric=cosine, + fts_id int + ); + `)) + }}, {Version: "2025-12-30.173837", Run: func(_ *Store, conn *sqlite.Conn) error { return sqlitex.ExecScript(conn, sqlfmt(` DROP VIEW public_blobs; diff --git a/backend/testutil/testutil.go b/backend/testutil/testutil.go index 9f90ab316..e867ce364 100644 --- a/backend/testutil/testutil.go +++ b/backend/testutil/testutil.go @@ -3,8 +3,12 @@ package testutil import ( "context" + "encoding/json" + "net/http" + "net/http/httptest" "os" "strings" + regular_sync "sync" "testing" "unicode" "unicode/utf8" @@ -214,3 +218,135 @@ func Manual(t *testing.T) { t.Skip("manual test is skipped") } + +type mockEmbedRequest struct { + Model string `json:"model"` + Input []string `json:"input"` +} + +type mockPullRequest struct { + Model string `json:"model"` + Stream *bool `json:"stream"` +} + +// MockOllamaServer is a test double for an Ollama HTTP server. +type MockOllamaServer struct { + Server *httptest.Server + + Mu regular_sync.Mutex + + BatchSizes []int + LoadedModels []string + SeenEmbeddings int + ShowRequests int + EmbedRequests int + embeddingDims int + contextSize int + + FirstEmbedOnce regular_sync.Once + FirstEmbedDone chan struct{} +} + +// MockOllamaServerOption configures MockOllamaServer. +type MockOllamaServerOption func(*MockOllamaServer) + +// WithMockOllamaEmbeddingDims sets the embedding dimensions for the mock server. +func WithMockOllamaEmbeddingDims(dims int) MockOllamaServerOption { + return func(s *MockOllamaServer) { + if dims > 0 { + s.embeddingDims = dims + } + } +} + +// WithMockOllamaContextSize sets the context size for the mock server. +func WithMockOllamaContextSize(size int) MockOllamaServerOption { + return func(s *MockOllamaServer) { + if size > 0 { + s.contextSize = size + } + } +} + +// NewMockOllamaServer creates a new mock Ollama HTTP server for testing. +func NewMockOllamaServer(t *testing.T, opts ...MockOllamaServerOption) *MockOllamaServer { + t.Helper() + + s := &MockOllamaServer{ + embeddingDims: 384, + contextSize: 2048, + FirstEmbedDone: make(chan struct{}), + } + for _, opt := range opts { + opt(s) + } + + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/pull": + var request mockPullRequest + require.NoError(t, json.NewDecoder(r.Body).Decode(&request)) + require.NotEmpty(t, request.Model) + require.NotNil(t, request.Stream) + require.False(t, *request.Stream) + + s.Mu.Lock() + s.LoadedModels = append(s.LoadedModels, request.Model) + s.Mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(map[string]string{"status": "success"})) + case "/api/show": + var request mockPullRequest + require.NoError(t, json.NewDecoder(r.Body).Decode(&request)) + require.NotEmpty(t, request.Model) + + s.Mu.Lock() + s.ShowRequests++ + embeddingDims := s.embeddingDims + contextSize := s.contextSize + s.Mu.Unlock() + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{ + "model_info": map[string]any{ + "gemma3.embedding_length": embeddingDims, + "gemma3.context_length": contextSize, + }, + "capabilities": []string{"embedding"}, + })) + case "/api/embed": + var request mockEmbedRequest + require.NoError(t, json.NewDecoder(r.Body).Decode(&request)) + require.NotEmpty(t, request.Model) + + s.Mu.Lock() + s.EmbedRequests++ + s.BatchSizes = append(s.BatchSizes, len(request.Input)) + embeddingDims := s.embeddingDims + s.Mu.Unlock() + response := make([][]float32, 0, len(request.Input)) + for _, input := range request.Input { + vec := make([]float32, embeddingDims) + if embeddingDims > 0 { + vec[0] = float32(len(input)) + } + response = append(response, vec) + } + + s.Mu.Lock() + s.SeenEmbeddings += len(response) + s.Mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"embeddings": response})) + + s.FirstEmbedOnce.Do(func() { + close(s.FirstEmbedDone) + }) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + + return s +} diff --git a/backend/util/llama-go b/backend/util/llama-go new file mode 160000 index 000000000..38cd4a8ab --- /dev/null +++ b/backend/util/llama-go @@ -0,0 +1 @@ +Subproject commit 38cd4a8abd4e3eb3e94283cf62105b734cd543f9 diff --git a/backend/util/singleflight/singleflight_test.go b/backend/util/singleflight/singleflight_test.go index 031922736..4d9c61593 100644 --- a/backend/util/singleflight/singleflight_test.go +++ b/backend/util/singleflight/singleflight_test.go @@ -108,7 +108,7 @@ func TestForget(t *testing.T) { ) go func() { - g.Do("key", func() (i interface{}, e error) { + _, _, _ = g.Do("key", func() (i interface{}, e error) { close(firstStarted) <-unblockFirst close(firstFinished) @@ -181,7 +181,7 @@ func TestPanicDo(t *testing.T) { } }() - g.Do("key", fn) + _, _, _ = g.Do("key", fn) }() } @@ -234,7 +234,7 @@ func TestPanicDoChan(t *testing.T) { if os.Getenv("TEST_PANIC_DOCHAN") != "" { defer func() { - recover() + _ = recover() }() g := new(Group[string, any]) @@ -281,9 +281,9 @@ func TestPanicDoSharedByDoChan(t *testing.T) { g := new(Group[string, any]) go func() { defer func() { - recover() + _ = recover() }() - g.Do("", func() (interface{}, error) { + _, _, _ = g.Do("", func() (interface{}, error) { close(blocked) <-unblock panic("Panicking in Do") diff --git a/backend/util/sqlite/auth.go b/backend/util/sqlite/auth.go index ec29a9c5d..6329db9ac 100644 --- a/backend/util/sqlite/auth.go +++ b/backend/util/sqlite/auth.go @@ -2,7 +2,12 @@ package sqlite // #include // #include -// extern int go_sqlite_auth_tramp(uintptr_t, int, char*, char*, char*, char*); +// #ifdef _WIN32 +// #define SQLITE_GO_EXPORT __declspec(dllexport) +// #else +// #define SQLITE_GO_EXPORT +// #endif +// extern SQLITE_GO_EXPORT int go_sqlite_auth_tramp(uintptr_t, int, char*, char*, char*, char*); // static int c_auth_tramp(void *userData, int action, const char* arg1, const char* arg2, const char* db, const char* trigger) { // return go_sqlite_auth_tramp((uintptr_t)userData, action, (char*)arg1, (char*)arg2, (char*)db, (char*)trigger); // } diff --git a/backend/util/sqlite/auth_test.go b/backend/util/sqlite/auth_test.go index fe892aca2..0fcf49f9b 100644 --- a/backend/util/sqlite/auth_test.go +++ b/backend/util/sqlite/auth_test.go @@ -23,7 +23,7 @@ func TestSetAuthorizer(t *testing.T) { lastAction = info.Action return authResult }) - c.SetAuthorizer(auth) + _ = c.SetAuthorizer(auth) t.Run("Allowed", func(t *testing.T) { authResult = 0 @@ -31,7 +31,7 @@ func TestSetAuthorizer(t *testing.T) { if err != nil { t.Fatal(err) } - stmt.Finalize() + _ = stmt.Finalize() if lastAction != sqlite.SQLITE_SELECT { t.Errorf("action = %q; want SQLITE_SELECT", lastAction) } @@ -41,7 +41,7 @@ func TestSetAuthorizer(t *testing.T) { authResult = sqlite.SQLITE_DENY stmt, _, err := c.PrepareTransient("SELECT 1;") if err == nil { - stmt.Finalize() + _ = stmt.Finalize() t.Fatal("PrepareTransient did not return an error") } if got, want := sqlite.ErrCode(err), sqlite.SQLITE_AUTH; got != want { diff --git a/backend/util/sqlite/backup.go b/backend/util/sqlite/backup.go index e7dfe7ffb..8392ba2c8 100644 --- a/backend/util/sqlite/backup.go +++ b/backend/util/sqlite/backup.go @@ -56,7 +56,7 @@ func (src *Conn) BackupToDB(srcDB, dstPath string) (dst *Conn, err error) { if err != nil { return } - defer b.Finish() + defer func() { _ = b.Finish() }() err = b.Step(-1) return } diff --git a/backend/util/sqlite/blob_test.go b/backend/util/sqlite/blob_test.go index 926292f26..057d20e84 100644 --- a/backend/util/sqlite/blob_test.go +++ b/backend/util/sqlite/blob_test.go @@ -359,8 +359,8 @@ func TestBlobPtrs(t *testing.T) { buf := new(bytes.Buffer) gzw := gzip.NewWriter(buf) - gzw.Write([]byte("hello")) - gzw.Close() + _, _ = gzw.Write([]byte("hello")) + _ = gzw.Close() n := buf.Len() stmt := c.Prep("INSERT INTO blobs (col) VALUES ($col);") @@ -378,10 +378,10 @@ func TestBlobPtrs(t *testing.T) { defer blob.Close() gzw = gzip.NewWriter(blob) - gzw.Write([]byte("hello")) - gzw.Close() + _, _ = gzw.Write([]byte("hello")) + _ = gzw.Close() - blob.Seek(0, 0) + _, _ = blob.Seek(0, 0) gzr, err := gzip.NewReader(blob) if err != nil { diff --git a/backend/util/sqlite/blocking_step.c b/backend/util/sqlite/blocking_step.c index d74071ab8..f890f6cdf 100644 --- a/backend/util/sqlite/blocking_step.c +++ b/backend/util/sqlite/blocking_step.c @@ -20,22 +20,38 @@ unlock_note* unlock_note_alloc() { unlock_note* un = (unlock_note*)malloc(sizeof(unlock_note)); + #ifdef _WIN32 + InitializeConditionVariable(&un->cond); + InitializeCriticalSection(&un->mu); + #else pthread_mutex_init(&un->mu, 0); pthread_cond_init(&un->cond, 0); + #endif return un; } void unlock_note_free(unlock_note* un) { + #ifdef _WIN32 + DeleteCriticalSection(&un->mu); + #else pthread_cond_destroy(&un->cond); pthread_mutex_destroy(&un->mu); + #endif free(un); } void unlock_note_fire(unlock_note* un) { + #ifdef _WIN32 + EnterCriticalSection(&un->mu); + un->fired = 1; + WakeConditionVariable(&un->cond); + LeaveCriticalSection(&un->mu); + #else pthread_mutex_lock(&un->mu); un->fired = 1; pthread_cond_signal(&un->cond); pthread_mutex_unlock(&un->mu); + #endif } static void unlock_notify_cb(void **apArg, int nArg) { @@ -50,11 +66,19 @@ int wait_for_unlock_notify(sqlite3 *db, unlock_note* un) { int res = sqlite3_unlock_notify(db, unlock_notify_cb, (void *)un); if (res == SQLITE_OK) { + #ifdef _WIN32 + EnterCriticalSection(&un->mu); + while (!un->fired) { + SleepConditionVariableCS(&un->cond, &un->mu, INFINITE); + } + LeaveCriticalSection(&un->mu); + #else pthread_mutex_lock(&un->mu); - if (!un->fired) { + while (!un->fired) { pthread_cond_wait(&un->cond, &un->mu); } pthread_mutex_unlock(&un->mu); + #endif } return res; diff --git a/backend/util/sqlite/blocking_step.h b/backend/util/sqlite/blocking_step.h index 309a8c577..c9ddd09d0 100644 --- a/backend/util/sqlite/blocking_step.h +++ b/backend/util/sqlite/blocking_step.h @@ -2,12 +2,22 @@ // See the documentation on Stmt.Step. #include + +#ifdef _WIN32 +#include +#else #include +#endif typedef struct unlock_note { int fired; + #ifdef _WIN32 + CONDITION_VARIABLE cond; + CRITICAL_SECTION mu; + #else pthread_cond_t cond; pthread_mutex_t mu; + #endif } unlock_note; unlock_note* unlock_note_alloc(); diff --git a/backend/util/sqlite/func.go b/backend/util/sqlite/func.go index e08059927..a7af194d4 100644 --- a/backend/util/sqlite/func.go +++ b/backend/util/sqlite/func.go @@ -19,9 +19,14 @@ package sqlite // #include // #include "wrappers.h" // -// extern void func_tramp(sqlite3_context*, int, sqlite3_value**); -// extern void step_tramp(sqlite3_context*, int, sqlite3_value**); -// extern void final_tramp(sqlite3_context*); +// #ifdef _WIN32 +// #define SQLITE_GO_EXPORT __declspec(dllexport) +// #else +// #define SQLITE_GO_EXPORT +// #endif +// extern SQLITE_GO_EXPORT void func_tramp(sqlite3_context*, int, sqlite3_value**); +// extern SQLITE_GO_EXPORT void step_tramp(sqlite3_context*, int, sqlite3_value**); +// extern SQLITE_GO_EXPORT void final_tramp(sqlite3_context*); // // static int go_sqlite3_create_function_v2( // sqlite3 *db, diff --git a/backend/util/sqlite/session.go b/backend/util/sqlite/session.go index eecfb848a..def96cac2 100644 --- a/backend/util/sqlite/session.go +++ b/backend/util/sqlite/session.go @@ -19,8 +19,13 @@ package sqlite // #include // #include "wrappers.h" // -// extern int go_strm_w_tramp(uintptr_t, char*, int); -// extern int go_strm_r_tramp(uintptr_t, char*, int*); +// #ifdef _WIN32 +// #define SQLITE_GO_EXPORT __declspec(dllexport) +// #else +// #define SQLITE_GO_EXPORT +// #endif +// extern SQLITE_GO_EXPORT int go_strm_w_tramp(uintptr_t, char*, int); +// extern SQLITE_GO_EXPORT int go_strm_r_tramp(uintptr_t, char*, int*); // // static int go_sqlite3session_changeset_strm( // sqlite3_session *pSession, diff --git a/backend/util/sqlite/session_test.go b/backend/util/sqlite/session_test.go index 63aea0718..264ed1d13 100644 --- a/backend/util/sqlite/session_test.go +++ b/backend/util/sqlite/session_test.go @@ -73,7 +73,7 @@ func fillSession(t *testing.T) (*sqlite.Conn, *sqlite.Session) { t.Fatal(err) } for i := int64(2); i < 100; i++ { - stmt.Reset() + _ = stmt.Reset() stmt.BindInt64(1, i) stmt.BindText(2, "column2") stmt.BindText(3, "column3") diff --git a/backend/util/sqlite/sqlite.go b/backend/util/sqlite/sqlite.go index 0e3c173bf..3f7b039c4 100644 --- a/backend/util/sqlite/sqlite.go +++ b/backend/util/sqlite/sqlite.go @@ -31,7 +31,6 @@ package sqlite // #cgo CFLAGS: -DSQLITE_DQS=0 // #cgo CFLAGS: -DSQLITE_ENABLE_GEOPOLY // #cgo CFLAGS: -DSQLITE_CORE -// #cgo windows LDFLAGS: -Wl,-Bstatic -lwinpthread -Wl,-Bdynamic // #cgo linux LDFLAGS: -ldl -lm // #cgo linux CFLAGS: -std=c99 // #cgo openbsd LDFLAGS: -lm @@ -51,7 +50,12 @@ package sqlite // return sqlite3_bind_blob(stmt, col, p, n, SQLITE_TRANSIENT); // } // -// extern void log_fn(void* pArg, int code, char* msg); +// #ifdef _WIN32 +// #define SQLITE_GO_EXPORT __declspec(dllexport) +// #else +// #define SQLITE_GO_EXPORT +// #endif +// extern SQLITE_GO_EXPORT void log_fn(void* pArg, int code, char* msg); // static void enable_logging() { // sqlite3_config(SQLITE_CONFIG_LOG, log_fn, NULL); // } diff --git a/backend/util/sqlite/sqlite_test.go b/backend/util/sqlite/sqlite_test.go index 956b1dac9..813ebe55c 100644 --- a/backend/util/sqlite/sqlite_test.go +++ b/backend/util/sqlite/sqlite_test.go @@ -152,7 +152,7 @@ func TestEarlyInterrupt(t *testing.T) { cancel() - stmt, err = c.Prepare("INSERT INTO bartable (foo1, foo2) VALUES ($f1, $f2);") + _, err = c.Prepare("INSERT INTO bartable (foo1, foo2) VALUES ($f1, $f2);") if err == nil { t.Fatal("Prepare err=nil, want prepare to fail") } diff --git a/backend/util/sqlite/sqlitex/kv.go b/backend/util/sqlite/sqlitex/kv.go new file mode 100644 index 000000000..c2dfa8ef3 --- /dev/null +++ b/backend/util/sqlite/sqlitex/kv.go @@ -0,0 +1,56 @@ +package sqlitex + +import ( + "context" + + "seed/backend/util/sqlite" +) + +// SetKV sets a key-value pair in the kv table. +func SetKV[T *sqlite.Conn | *Pool](ctx context.Context, db T, key, value string, replace bool) error { + var conn *sqlite.Conn + switch v := any(db).(type) { + case *sqlite.Conn: + conn = v + case *Pool: + c, release, err := v.Conn(ctx) + if err != nil { + return err + } + defer release() + conn = c + } + + if replace { + return Exec(conn, "INSERT OR REPLACE INTO kv (key, value) VALUES (?, ?);", nil, key, value) + } + + return Exec(conn, "INSERT INTO kv (key, value) VALUES (?, ?);", nil, key, value) +} + +// GetKV gets a value from the kv table. +func GetKV[T *sqlite.Conn | *Pool](ctx context.Context, db T, key string) (string, error) { + var conn *sqlite.Conn + switch v := any(db).(type) { + case *sqlite.Conn: + conn = v + case *Pool: + c, release, err := v.Conn(ctx) + if err != nil { + return "", err + } + defer release() + conn = c + } + + var value string + err := Exec(conn, "SELECT value FROM kv WHERE key = ?;", func(stmt *sqlite.Stmt) error { + value = stmt.ColumnText(0) + return nil + }, key) + if err != nil { + return "", err + } + + return value, nil +} diff --git a/backend/util/sqlitegen/example/schema/schema.go b/backend/util/sqlitegen/example/schema/schema.go index a6f343265..696712a03 100644 --- a/backend/util/sqlitegen/example/schema/schema.go +++ b/backend/util/sqlitegen/example/schema/schema.go @@ -1,7 +1,6 @@ package schema import ( - "io/ioutil" "os" "path/filepath" @@ -34,12 +33,12 @@ func generateSchema() (err error) { return err } - return ioutil.WriteFile("schema.gen.go", code, 0600) + return os.WriteFile("schema.gen.go", code, 0600) } // MakeConn creates a test connection with an example schema. func MakeConn() (conn *sqlite.Conn, closer func() error, err error) { - dir, err := ioutil.TempDir("", "sqlitegen-") + dir, err := os.MkdirTemp("", "sqlitegen-") if err != nil { return nil, nil, err } diff --git a/build/rules/go/go.build_defs b/build/rules/go/go.build_defs index fab811386..094023931 100644 --- a/build/rules/go/go.build_defs +++ b/build/rules/go/go.build_defs @@ -101,7 +101,7 @@ $TOOLS_GO build -trimpath -o $OUT {package} LLVM_TRIPLES = { "darwin/amd64": "x86_64-apple-darwin", "darwin/arm64": "aarch64-apple-darwin", - "windows/amd64": "x86_64-pc-windows-msvc", + "windows/amd64": "x86_64-pc-windows-gnu", "linux/amd64": "x86_64-unknown-linux-gnu", "linux/arm64": "aarch64-unknown-linux-gnu", } diff --git a/build/tools/BUILD.plz b/build/tools/BUILD.plz index baff2df6e..0eb1cca05 100644 --- a/build/tools/BUILD.plz +++ b/build/tools/BUILD.plz @@ -80,6 +80,23 @@ EOF visibility = ["PUBLIC"], ) +# Custom cmake wrapper - uses version directly to avoid mise.toml trust issues +build_rule( + name = "cmake", + outs = ["cmake"], + binary = True, + cmd = """ +cat > $OUT <<'EOF' +#!/bin/sh +# Ignore workspace config to avoid trust issues in Please sandbox +export MISE_IGNORED_CONFIG_PATHS="$WORKSPACE" +exec $SEED_MISE_BIN x cmake@3.31.6 -- cmake "$@" +EOF +""", + output_is_complete = True, + visibility = ["PUBLIC"], +) + mise_binary( name = "go" ) diff --git a/dev b/dev index bb8be1f18..1b7113c26 100755 --- a/dev +++ b/dev @@ -8,6 +8,7 @@ import subprocess import sys + def cmd(cmds: argparse._SubParsersAction, name: str, help: str): """Decorator that registers subcommands as functions to be executed.""" @@ -30,6 +31,7 @@ def run(cmd: str, args: list = [], capture_output=False, env: os._Environ = os.e ) + def main(): if not os.getenv("DIRENV_DIR"): print("Direnv is not enabled. Fix it first! See README.md for instructions.") @@ -50,7 +52,7 @@ def main(): @cmd( cmds, "gen", - "Check all the generated code is up to date. Otherwise run the code generation process to fix it.", + "Check the generated code is up to date and run the code generation process to fix it.", ) def gen(args): targets_to_check = ( @@ -176,7 +178,9 @@ def main(): "Build and run seed-daemon binary for the current platform.", ) def run_backend(args): - return run("plz run //backend:seed-daemon", args=args) + env = os.environ.copy() + env["LLAMA_LOG"] = "error" + return run("plz run //backend:seed-daemon", args=args, env=env) @cmd(cmds, "build-backend", "Build seed-daemon binary for the current platform.") def build_backend(args): diff --git a/frontend/apps/desktop/forge.config.ts b/frontend/apps/desktop/forge.config.ts index 29be6b8d0..7351c5aaf 100644 --- a/frontend/apps/desktop/forge.config.ts +++ b/frontend/apps/desktop/forge.config.ts @@ -20,7 +20,7 @@ const devProjectRoot = path.join(process.cwd(), '../../..') const LLVM_TRIPLES = { 'darwin/x64': 'x86_64-apple-darwin', 'darwin/arm64': 'aarch64-apple-darwin', - 'win32/x64': 'x86_64-pc-windows-msvc.exe', + 'win32/x64': 'x86_64-pc-windows-gnu.exe', 'linux/x64': 'x86_64-unknown-linux-gnu', 'linux/arm64': 'aarch64-unknown-linux-gnu', } @@ -39,6 +39,23 @@ const daemonBinaryPath = path.join( `plz-out/bin/backend/seed-daemon-${getPlatformTriple()}`, ) +const extraResources = [daemonBinaryPath] + +if (process.platform === 'win32') { + const winpthreadRuntimePath = path.join( + devProjectRoot, + 'plz-out/bin/backend/libwinpthread-1.dll', + ) + + if (fs.existsSync(winpthreadRuntimePath)) { + extraResources.push(winpthreadRuntimePath) + } else if (process.env.CI) { + throw new Error( + `Missing Windows runtime dependency at ${winpthreadRuntimePath}`, + ) + } +} + let iconsPath = IS_PROD_DEV ? path.resolve(__dirname, 'assets', 'icons', 'icon') : path.resolve(__dirname, 'assets', 'icons-prod', 'icon') @@ -132,7 +149,7 @@ const config: ForgeConfig = { executableName: IS_PROD_DEV ? 'SeedDev' : 'Seed', appCategoryType: 'public.app-category.productivity', // packageManager: 'yarn', - extraResource: [daemonBinaryPath], + extraResource: extraResources, // beforeCopy: [setLanguages(['en', 'en_US'])], win32metadata: { CompanyName: 'Mintter Inc.', diff --git a/frontend/apps/desktop/src/app-api.ts b/frontend/apps/desktop/src/app-api.ts index 510c849d5..cba898290 100644 --- a/frontend/apps/desktop/src/app-api.ts +++ b/frontend/apps/desktop/src/app-api.ts @@ -26,6 +26,7 @@ import {writeFile} from 'fs-extra' import path from 'path' import z from 'zod' import {deleteAccount} from './app-account-management' +import {restartDaemonWithEmbedding} from './daemon' import {commentsApi} from './app-comments' import {diagnosisApi} from './app-diagnosis' import {draftsApi} from './app-drafts' @@ -425,6 +426,14 @@ export const router = t.router({ getAppInfo: t.procedure.query(() => { return {dataDir: userDataPath, loggingDir: log.loggingDir} }), + + restartDaemonWithEmbedding: t.procedure + .input(z.object({embeddingEnabled: z.boolean()})) + .mutation(async ({input}) => { + log.info('Restarting daemon with embedding setting:', input) + await restartDaemonWithEmbedding(input.embeddingEnabled) + return {success: true} + }), }) export const trpc = router.createCaller({}) diff --git a/frontend/apps/desktop/src/app-experiments.ts b/frontend/apps/desktop/src/app-experiments.ts index 874703a1e..bdf9f6f42 100644 --- a/frontend/apps/desktop/src/app-experiments.ts +++ b/frontend/apps/desktop/src/app-experiments.ts @@ -8,6 +8,15 @@ const EXPERIMENTS_STORAGE_KEY = 'Experiments-v001' let experimentsState: AppExperiments = appStore.get(EXPERIMENTS_STORAGE_KEY) || {} +/** + * Returns the stored embedding enabled setting. + * Used by main.ts to determine daemon startup flags before tRPC is ready. + */ +export function getStoredEmbeddingEnabled(): boolean { + const experiments = appStore.get(EXPERIMENTS_STORAGE_KEY) || {} + return experiments.embeddingEnabled || false +} + export const experimentsApi = t.router({ get: t.procedure.query(async () => { return experimentsState diff --git a/frontend/apps/desktop/src/components/footer.tsx b/frontend/apps/desktop/src/components/footer.tsx index b1177fb2d..e78a4f26d 100644 --- a/frontend/apps/desktop/src/components/footer.tsx +++ b/frontend/apps/desktop/src/components/footer.tsx @@ -1,9 +1,14 @@ import {getUpdateStatusLabel, useUpdateStatus} from '@/components/auto-updater' import {useConnectionSummary} from '@/models/contacts' +import {useDaemonInfo} from '@/models/daemon' import { getActiveDiscoveriesStream, getAggregatedDiscoveryStream, } from '@/models/entities' +import { + Task, + TaskName, +} from '@shm/shared/client/.generated/daemon/v1alpha/daemon_pb' import {COMMIT_HASH, VERSION} from '@shm/shared/constants' import {DiscoveryState} from '@shm/shared/hm-types' import {useResource, useResources} from '@shm/shared/models/entity' @@ -49,6 +54,7 @@ export default function Footer({children}: {children?: ReactNode}) {
+ {children}
@@ -331,3 +337,128 @@ function DiscoveryIndicator() { ) } + +/** + * Get human-readable label for a task name + */ +function getTaskLabel(taskName: TaskName): string { + switch (taskName) { + case TaskName.REINDEXING: + return 'Reindexing Database' + case TaskName.EMBEDDING: + return 'Generating Embeddings' + case TaskName.LOADING_MODEL: + return 'Loading AI Model' + default: + return 'Background Task' + } +} + +/** + * Calculate progress percentage for a task + */ +function getTaskProgress(task: Task): number { + const total = Number(task.total) + const completed = Number(task.completed) + if (total <= 0) return 0 + return Math.round((completed / total) * 100) +} + +/** + * Single task item in the hover card + */ +function DaemonTaskItem({task}: {task: Task}) { + const progress = getTaskProgress(task) + const label = getTaskLabel(task.taskName) + const total = Number(task.total) + const completed = Number(task.completed) + + return ( +
+
+ + {label} + + + {progress}% + +
+ + {total > 0 && ( + + {completed.toLocaleString()} / {total.toLocaleString()} + {task.description && ` - ${task.description}`} + + )} +
+ ) +} + +/** + * Footer indicator showing background daemon tasks with progress + */ +function DaemonTasksIndicator() { + const {data: info} = useDaemonInfo() + + // Get active tasks + const tasks = info?.tasks ?? [] + + // Don't render anything if no tasks + if (tasks.length === 0) return null + + // Build summary text + const taskCount = tasks.length + const summaryText = + taskCount === 1 + ? getTaskLabel(tasks[0].taskName) + : `${taskCount} tasks running` + + // Calculate average progress across all tasks for the inline indicator + const avgProgress = + tasks.length > 0 + ? Math.round( + tasks.reduce( + (sum: number, task: Task) => sum + getTaskProgress(task), + 0, + ) / tasks.length, + ) + : 0 + + return ( + + +
+ + + {summaryText} + + {tasks.length === 1 && ( + + ({avgProgress}%) + + )} +
+ + +
+ + Background Tasks + +
+ {tasks.map((task: Task, index: number) => ( + + ))} +
+
+ + + ) +} diff --git a/frontend/apps/desktop/src/components/search-input.tsx b/frontend/apps/desktop/src/components/search-input.tsx index 5d10342cb..301fe2e32 100644 --- a/frontend/apps/desktop/src/components/search-input.tsx +++ b/frontend/apps/desktop/src/components/search-input.tsx @@ -6,6 +6,7 @@ import {useSelectedAccountId} from '@/selected-account' import {client} from '@/trpc' import {parseDeepLink} from '@/utils/deep-links' import {useTriggerWindowEvent} from '@/utils/window-events' +import {SearchType} from '@shm/shared/client/.generated/entities/v1alpha/entities_pb' import {HYPERMEDIA_SCHEME} from '@shm/shared/constants' import {SearchResult} from '@shm/shared/editor-types' import {UnpackedHypermediaId} from '@shm/shared/hm-types' @@ -106,6 +107,7 @@ export const SearchInput = forwardRef< includeBody: true, contextSize: 48 - deferredSearch.length, perspectiveAccountUid: selectedAccountId ?? undefined, + searchType: SearchType.SEARCH_HYBRID, }) const itemRefs = useRef<(HTMLDivElement | null)[]>([]) let queryItem: null | SearchResult = useMemo(() => { diff --git a/frontend/apps/desktop/src/daemon-path.ts b/frontend/apps/desktop/src/daemon-path.ts index 1c59ab7b6..7cbe6e065 100644 --- a/frontend/apps/desktop/src/daemon-path.ts +++ b/frontend/apps/desktop/src/daemon-path.ts @@ -32,7 +32,7 @@ function getPlatformTriple() { case 'darwin/arm64': return 'aarch64-apple-darwin' case 'win32/x64': - return 'x86_64-pc-windows-msvc' + return 'x86_64-pc-windows-gnu' case 'linux/x64': return 'x86_64-unknown-linux-gnu' case 'linux/arm64': diff --git a/frontend/apps/desktop/src/daemon.ts b/frontend/apps/desktop/src/daemon.ts index 2e92f276f..5da83561e 100644 --- a/frontend/apps/desktop/src/daemon.ts +++ b/frontend/apps/desktop/src/daemon.ts @@ -7,7 +7,7 @@ import { P2P_PORT, VERSION, } from '@shm/shared/constants' -import {spawn} from 'child_process' +import {ChildProcess, spawn} from 'child_process' import {app} from 'electron' import * as readline from 'node:readline' import path from 'path' @@ -23,7 +23,8 @@ const lndhubFlags = ? '-lndhub.mainnet=true' : '-lndhub.mainnet=false' -const daemonArguments = [ +// Base daemon arguments (without embedding flags) +const baseDaemonArguments = [ '-http.port', String(DAEMON_HTTP_PORT), @@ -43,9 +44,34 @@ const daemonArguments = [ '-syncing.no-sync-back=true', lndhubFlags, - `SENTRY_DSN=${__SENTRY_DSN__}`, ] +// Embedding-specific flags +const embeddingFlags = [ + '-llm.embedding.enabled', + '-llm.backend.sleep-between-batches', + '0s', + '-llm.backend.batch-size', + '100', + '-llm.embedding.index-pass-size', + '100', +] + +// Build daemon arguments based on embedding setting +function buildDaemonArguments(embeddingEnabled: boolean): string[] { + if (embeddingEnabled) { + return [...baseDaemonArguments, ...embeddingFlags] + } + return [...baseDaemonArguments] +} + +// For backwards compatibility during initial startup +const daemonArguments = baseDaemonArguments + +// Store daemon process reference for restart capability +let currentDaemonProcess: ChildProcess | null = null +let expectingDaemonClose = false + type ReadyState = {t: 'ready'} type ErrorState = {t: 'error'; message: string} type StartupState = {t: 'startup'} @@ -77,7 +103,9 @@ export function updateGoDaemonState(state: GoDaemonState) { daemonStateHandlers.forEach((handler) => handler(state)) } -export async function startMainDaemon(): Promise<{ +export async function startMainDaemon( + embeddingEnabled: boolean = false, +): Promise<{ httpPort: string | undefined grpcPort: string | undefined p2pPort: string | undefined @@ -95,21 +123,24 @@ export async function startMainDaemon(): Promise<{ const daemonEnv = { ...process.env, SENTRY_RELEASE: VERSION, + SENTRY_DSN: __SENTRY_DSN__, } - // log.info('Daemon with env:', daemonEnv) - // log.info('Daemon with arguments:', daemonArguments) + const args = buildDaemonArguments(embeddingEnabled) + log.info('Starting daemon with arguments:', {args, embeddingEnabled}) - const daemonProcess = spawn(goDaemonExecutablePath, daemonArguments, { + const daemonProcess = spawn(goDaemonExecutablePath, args, { // daemon env cwd: path.join(process.cwd(), '../../..'), env: daemonEnv, stdio: 'pipe', }) + // Store reference for restart capability + currentDaemonProcess = daemonProcess + let lastStderr = '' const stderr = readline.createInterface({input: daemonProcess.stderr}) - let expectingDaemonClose = false await new Promise((resolve, reject) => { stderr.on('line', (line: string) => { lastStderr = line @@ -226,3 +257,144 @@ async function tryUntilSuccess( throw new Error('Timed out: ' + attemptName) } } + +/** + * Restarts the daemon with new embedding configuration. + * This will kill the current daemon process and start a new one with updated flags. + */ +export async function restartDaemonWithEmbedding( + embeddingEnabled: boolean, +): Promise { + if (process.env.SEED_NO_DAEMON_SPAWN) { + log.debug('Daemon restart skipped (SEED_NO_DAEMON_SPAWN)') + return + } + + log.info('Restarting daemon with embedding:', {embeddingEnabled}) + updateGoDaemonState({t: 'startup'}) + + // Kill the current daemon process + if (currentDaemonProcess) { + expectingDaemonClose = true + currentDaemonProcess.kill() + + // Wait for the process to actually close + await new Promise((resolve) => { + if (!currentDaemonProcess) { + resolve() + return + } + const onClose = () => { + currentDaemonProcess?.removeListener('close', onClose) + resolve() + } + currentDaemonProcess.on('close', onClose) + // Timeout after 5 seconds + setTimeout(() => { + currentDaemonProcess?.removeListener('close', onClose) + resolve() + }, 5000) + }) + + currentDaemonProcess = null + } + + // Reset the close expectation flag + expectingDaemonClose = false + + // Start new daemon with updated configuration + const daemonEnv = { + ...process.env, + SENTRY_RELEASE: VERSION, + SENTRY_DSN: __SENTRY_DSN__, + } + + const args = buildDaemonArguments(embeddingEnabled) + log.info('Restarting daemon with arguments:', {args, embeddingEnabled}) + + const daemonProcess = spawn(goDaemonExecutablePath, args, { + cwd: path.join(process.cwd(), '../../..'), + env: daemonEnv, + stdio: 'pipe', + }) + + currentDaemonProcess = daemonProcess + + let lastStderr = '' + const stderr = readline.createInterface({input: daemonProcess.stderr}) + await new Promise((resolve, reject) => { + stderr.on('line', (line: string) => { + lastStderr = line + if (line.includes('DaemonStarted')) { + updateGoDaemonState({t: 'ready'}) + } + log.rawMessage(line) + }) + const stdout = readline.createInterface({input: daemonProcess.stdout}) + stdout.on('line', (line: string) => { + log.rawMessage(line) + }) + daemonProcess.on('error', (err) => { + log.error('Go daemon restart spawn error', {error: err}) + reject(err) + }) + daemonProcess.on('close', (code, signal) => { + if (!expectingDaemonClose) { + updateGoDaemonState({ + t: 'error', + message: 'Service Error: !!!' + lastStderr, + }) + log.error('Go daemon closed after restart', {code, signal}) + } + }) + daemonProcess.on('spawn', () => { + log.debug('Go daemon respawned') + resolve() + }) + }) + + // Wait for daemon to be ready + await tryUntilSuccess( + async () => { + log.debug('Waiting for restarted daemon to boot...') + const info = await grpcClient.daemon.getInfo({}) + if (info.state !== State.ACTIVE) { + if (info.state === State.MIGRATING && info.tasks.length === 1) { + const completed = Number(info.tasks[0].completed) + const total = Number(info.tasks[0].total) + log.info(`Daemon migrating after restart: ${completed}/${total}`) + updateGoDaemonState({ + t: 'migrating', + completed, + total, + }) + } + throw new Error(`Daemon not ready yet: ${info.state}`) + } + log.info('Restarted daemon is ready') + updateGoDaemonState({t: 'ready'}) + }, + 'waiting for restarted daemon gRPC to be ready', + 200, + 10 * 60 * 1_000, + ) + + // Also check HTTP endpoint + await tryUntilSuccess( + async () => { + log.debug('Checking HTTP endpoint health after restart...') + const response = await fetch( + `http://localhost:${DAEMON_HTTP_PORT}/debug/version`, + ) + if (!response.ok) { + throw new Error(`HTTP endpoint not ready: ${response.status}`) + } + log.info('HTTP endpoint is ready after restart') + }, + 'waiting for restarted daemon HTTP to be ready', + 200, + 30_000, + ) + + log.info('Daemon restart complete', {embeddingEnabled}) +} diff --git a/frontend/apps/desktop/src/main.ts b/frontend/apps/desktop/src/main.ts index a2b437f6d..60a5bff4c 100644 --- a/frontend/apps/desktop/src/main.ts +++ b/frontend/apps/desktop/src/main.ts @@ -74,6 +74,7 @@ import { import {defaultRoute} from '@shm/shared/routes' import {initCommentDrafts} from './app-comments' import {initDrafts} from './app-drafts' +import {getStoredEmbeddingEnabled} from './app-experiments' import { getOnboardingState, setInitialAccountIdCount, @@ -253,7 +254,9 @@ async function startDaemonWithLoadingWindow(): Promise { try { // Start daemon - this spawns the process and polls until ACTIVE // Daemon will send state updates (startup, migrating, etc) to loading window - await startMainDaemon() + const embeddingEnabled = getStoredEmbeddingEnabled() + logger.info('[MAIN]: Starting daemon with embedding:', {embeddingEnabled}) + await startMainDaemon(embeddingEnabled) logger.info('[MAIN]: Daemon is ACTIVE') } finally { // Cleanup: unsubscribe if still subscribed diff --git a/frontend/apps/desktop/src/models/daemon.ts b/frontend/apps/desktop/src/models/daemon.ts index dbccdf0d2..55a7fbfb4 100644 --- a/frontend/apps/desktop/src/models/daemon.ts +++ b/frontend/apps/desktop/src/models/daemon.ts @@ -26,6 +26,11 @@ export type NamedKey = { publicKey: string } +// Default interval for daemon info polling (when no tasks are active) +const DEFAULT_DAEMON_INFO_INTERVAL = 10_000 +// Fast interval for daemon info polling (when tasks are active) +const ACTIVE_TASKS_DAEMON_INFO_INTERVAL = 2_000 + function queryDaemonInfo( grpcClient: GRPCClient, opts: UseQueryOptions | FetchQueryOptions = {}, @@ -43,12 +48,45 @@ function queryDaemonInfo( } return null }, - refetchInterval: 10_000, + refetchInterval: DEFAULT_DAEMON_INFO_INTERVAL, useErrorBoundary: false, } } + +/** + * Hook to get daemon info with smart polling. + * Polls every 2s when there are active tasks, otherwise every 10s. + */ export function useDaemonInfo(opts: UseQueryOptions = {}) { - return useQuery(queryDaemonInfo(grpcClient, opts)) + // Track whether we have active tasks to determine polling interval + const [hasActiveTasks, setHasActiveTasks] = useState(false) + + const query = useQuery({ + queryKey: [queryKeys.GET_DAEMON_INFO], + queryFn: async () => { + try { + return await grpcClient.daemon.getInfo({}) + } catch (error) { + if (error) { + console.log('error check make sure not set up condition..', error) + } + } + return null + }, + refetchInterval: hasActiveTasks + ? ACTIVE_TASKS_DAEMON_INFO_INTERVAL + : DEFAULT_DAEMON_INFO_INTERVAL, + useErrorBoundary: false, + ...opts, + }) + + // Update hasActiveTasks based on query data + useEffect(() => { + const tasksCount = query.data?.tasks?.length ?? 0 + setHasActiveTasks(tasksCount > 0) + }, [query.data?.tasks?.length]) + + return query } export function useMnemonics( diff --git a/frontend/apps/desktop/src/pages/settings.tsx b/frontend/apps/desktop/src/pages/settings.tsx index 8fef8486a..9017dcd51 100644 --- a/frontend/apps/desktop/src/pages/settings.tsx +++ b/frontend/apps/desktop/src/pages/settings.tsx @@ -281,11 +281,101 @@ export function DeveloperSettings() { const writeExperiments = useWriteExperiments() const enabledDevTools = experiments?.developerTools const enabledPubContentDevMenu = experiments?.pubContentDevMenu + const embeddingEnabled = experiments?.embeddingEnabled + const [showEmbeddingConfirm, setShowEmbeddingConfirm] = useState(false) + const [pendingEmbeddingState, setPendingEmbeddingState] = useState(false) + const restartDaemon = useMutation({ + mutationFn: (enabled: boolean) => + client.restartDaemonWithEmbedding.mutate({embeddingEnabled: enabled}), + onSuccess: () => { + toast.success( + pendingEmbeddingState + ? 'Embedding enabled. Daemon restarted.' + : 'Embedding disabled. Daemon restarted.', + ) + }, + onError: (error: unknown) => { + toast.error('Failed to restart daemon: ' + String(error)) + }, + }) const openDraftLogs = useMutation({ mutationFn: () => client.diagnosis.openDraftLogFolder.mutate(), }) + + function handleEmbeddingToggle() { + const newState = !embeddingEnabled + setPendingEmbeddingState(newState) + setShowEmbeddingConfirm(true) + } + + function confirmEmbeddingChange() { + setShowEmbeddingConfirm(false) + writeExperiments.mutate({embeddingEnabled: pendingEmbeddingState}) + restartDaemon.mutate(pendingEmbeddingState) + } + return ( <> + + + Enable AI-powered document embeddings for semantic search and related + content features. This will restart the background service. + +
+ {embeddingEnabled ? :
} + +
+ + + + + + {pendingEmbeddingState + ? 'Enable Embedding?' + : 'Disable Embedding?'} + + + {pendingEmbeddingState + ? 'This will restart the background service with AI embedding features enabled. The app may be briefly unresponsive during restart.' + : 'This will restart the background service with AI embedding features disabled. The app may be briefly unresponsive during restart.'} + +
+ + + + + + +
+ + + Adds features across the app for helping diagnose issues. Mostly diff --git a/frontend/packages/shared/src/api-search.ts b/frontend/packages/shared/src/api-search.ts index f2bf8ca19..530dd5623 100644 --- a/frontend/packages/shared/src/api-search.ts +++ b/frontend/packages/shared/src/api-search.ts @@ -8,14 +8,21 @@ export const Search: HMRequestImplementation = { grpcClient: GRPCClient, input: HMSearchInput, ): Promise { - const {query, accountUid, includeBody, contextSize, perspectiveAccountUid} = - input + const { + query, + accountUid, + includeBody, + contextSize, + perspectiveAccountUid, + searchType, + } = input const result = await grpcClient.entities.searchEntities({ query, includeBody, contextSize, accountUid, loggedAccountUid: perspectiveAccountUid, + searchType, }) return { searchQuery: query, diff --git a/frontend/packages/shared/src/client/.generated/daemon/v1alpha/daemon_connect.ts b/frontend/packages/shared/src/client/.generated/daemon/v1alpha/daemon_connect.ts index 5cbcea5dd..7603d5eab 100644 --- a/frontend/packages/shared/src/client/.generated/daemon/v1alpha/daemon_connect.ts +++ b/frontend/packages/shared/src/client/.generated/daemon/v1alpha/daemon_connect.ts @@ -1,4 +1,4 @@ -// @generated by protoc-gen-connect-es v1.6.1 with parameter "target=ts,import_extension=none" +// @generated by protoc-gen-connect-es v1.4.0 with parameter "target=ts,import_extension=none" // @generated from file daemon/v1alpha/daemon.proto (package com.seed.daemon.v1alpha, syntax proto3) /* eslint-disable */ // @ts-nocheck diff --git a/frontend/packages/shared/src/client/.generated/daemon/v1alpha/daemon_pb.ts b/frontend/packages/shared/src/client/.generated/daemon/v1alpha/daemon_pb.ts index b880e509c..83c7f9786 100644 --- a/frontend/packages/shared/src/client/.generated/daemon/v1alpha/daemon_pb.ts +++ b/frontend/packages/shared/src/client/.generated/daemon/v1alpha/daemon_pb.ts @@ -1,4 +1,4 @@ -// @generated by protoc-gen-es v1.4.1 with parameter "target=ts,import_extension=none" +// @generated by protoc-gen-es v1.10.0 with parameter "target=ts,import_extension=none" // @generated from file daemon/v1alpha/daemon.proto (package com.seed.daemon.v1alpha, syntax proto3) /* eslint-disable */ // @ts-nocheck @@ -60,11 +60,27 @@ export enum TaskName { * @generated from enum value: REINDEXING = 1; */ REINDEXING = 1, + + /** + * Task for generating embeddings. + * + * @generated from enum value: EMBEDDING = 2; + */ + EMBEDDING = 2, + + /** + * Task for loading a machine learning model. + * + * @generated from enum value: LOADING_MODEL = 3; + */ + LOADING_MODEL = 3, } // Retrieve enum metadata with: proto3.getEnumType(TaskName) proto3.util.setEnumType(TaskName, "com.seed.daemon.v1alpha.TaskName", [ { no: 0, name: "TASK_NAME_UNSPECIFIED" }, { no: 1, name: "REINDEXING" }, + { no: 2, name: "EMBEDDING" }, + { no: 3, name: "LOADING_MODEL" }, ]); /** diff --git a/frontend/packages/shared/src/client/.generated/entities/v1alpha/entities_connect.ts b/frontend/packages/shared/src/client/.generated/entities/v1alpha/entities_connect.ts index 58e7a5694..4a9c9737d 100644 --- a/frontend/packages/shared/src/client/.generated/entities/v1alpha/entities_connect.ts +++ b/frontend/packages/shared/src/client/.generated/entities/v1alpha/entities_connect.ts @@ -1,4 +1,4 @@ -// @generated by protoc-gen-connect-es v1.6.1 with parameter "target=ts,import_extension=none" +// @generated by protoc-gen-connect-es v1.4.0 with parameter "target=ts,import_extension=none" // @generated from file entities/v1alpha/entities.proto (package com.seed.entities.v1alpha, syntax proto3) /* eslint-disable */ // @ts-nocheck diff --git a/frontend/packages/shared/src/client/.generated/entities/v1alpha/entities_pb.ts b/frontend/packages/shared/src/client/.generated/entities/v1alpha/entities_pb.ts index a9a4db303..9b83901e1 100644 --- a/frontend/packages/shared/src/client/.generated/entities/v1alpha/entities_pb.ts +++ b/frontend/packages/shared/src/client/.generated/entities/v1alpha/entities_pb.ts @@ -1,4 +1,4 @@ -// @generated by protoc-gen-es v1.4.1 with parameter "target=ts,import_extension=none" +// @generated by protoc-gen-es v1.10.0 with parameter "target=ts,import_extension=none" // @generated from file entities/v1alpha/entities.proto (package com.seed.entities.v1alpha, syntax proto3) /* eslint-disable */ // @ts-nocheck @@ -41,6 +41,74 @@ proto3.util.setEnumType(DiscoveryTaskState, "com.seed.entities.v1alpha.Discovery { no: 2, name: "DISCOVERY_TASK_COMPLETED" }, ]); +/** + * Describes the state of the discovery task. + * + * @generated from enum com.seed.entities.v1alpha.SearchType + */ +export enum SearchType { + /** + * Keyword-based search. + * + * @generated from enum value: SEARCH_KEYWORD = 0; + */ + SEARCH_KEYWORD = 0, + + /** + * Semantic search. + * + * @generated from enum value: SEARCH_SEMANTIC = 1; + */ + SEARCH_SEMANTIC = 1, + + /** + * Hybrid search. with RRFusion. + * + * @generated from enum value: SEARCH_HYBRID = 2; + */ + SEARCH_HYBRID = 2, +} +// Retrieve enum metadata with: proto3.getEnumType(SearchType) +proto3.util.setEnumType(SearchType, "com.seed.entities.v1alpha.SearchType", [ + { no: 0, name: "SEARCH_KEYWORD" }, + { no: 1, name: "SEARCH_SEMANTIC" }, + { no: 2, name: "SEARCH_HYBRID" }, +]); + +/** + * Content type to filter search results by. + * + * @generated from enum com.seed.entities.v1alpha.ContentTypeFilter + */ +export enum ContentTypeFilter { + /** + * @generated from enum value: CONTENT_TYPE_TITLE = 0; + */ + CONTENT_TYPE_TITLE = 0, + + /** + * @generated from enum value: CONTENT_TYPE_DOCUMENT = 1; + */ + CONTENT_TYPE_DOCUMENT = 1, + + /** + * @generated from enum value: CONTENT_TYPE_COMMENT = 2; + */ + CONTENT_TYPE_COMMENT = 2, + + /** + * @generated from enum value: CONTENT_TYPE_CONTACT = 3; + */ + CONTENT_TYPE_CONTACT = 3, +} +// Retrieve enum metadata with: proto3.getEnumType(ContentTypeFilter) +proto3.util.setEnumType(ContentTypeFilter, "com.seed.entities.v1alpha.ContentTypeFilter", [ + { no: 0, name: "CONTENT_TYPE_TITLE" }, + { no: 1, name: "CONTENT_TYPE_DOCUMENT" }, + { no: 2, name: "CONTENT_TYPE_COMMENT" }, + { no: 3, name: "CONTENT_TYPE_CONTACT" }, +]); + /** * Request to get a change by ID. * @@ -796,13 +864,13 @@ export class DeletedEntity extends Message { } /** - * Request to + * Request to search entities. * * @generated from message com.seed.entities.v1alpha.SearchEntitiesRequest */ export class SearchEntitiesRequest extends Message { /** - * Query to find. We Ssupport wildcards and phrases. + * Query to find. We support wildcards and phrases. * See https://sqlite.org/fts5.html#full_text_query_syntax. * * @generated from field: string query = 1; @@ -810,11 +878,11 @@ export class SearchEntitiesRequest extends Message { query = ""; /** - * Whether to look into all content available or just the titles. - * If false, comments are not included in the search. - * Default is false. + * Deprecated, use content_type_filters instead to specify + * which content types to include in the search. * - * @generated from field: bool include_body = 2; + * @generated from field: bool include_body = 2 [deprecated = true]; + * @deprecated */ includeBody = false; @@ -828,10 +896,10 @@ export class SearchEntitiesRequest extends Message { contextSize = 0; /** - * Optional. The account uid to filter the search by. - * If not set, the search will be performed across all accounts. + * Deprecated. Use iri_filter instead. * - * @generated from field: string account_uid = 4; + * @generated from field: string account_uid = 4 [deprecated = true]; + * @deprecated */ accountUid = ""; @@ -844,6 +912,55 @@ export class SearchEntitiesRequest extends Message { */ loggedAccountUid = ""; + /** + * Optional. Type of search to perform. Could be keyword, semantic or hybrid. + * if not set, keyword search is used. + * + * @generated from field: com.seed.entities.v1alpha.SearchType search_type = 6; + */ + searchType = SearchType.SEARCH_KEYWORD; + + /** + * Optional. hm:// URL with optional GLOB wildcards to scope search. + * Examples: "hm:///cars/honda" (single doc), "hm:///cars/*" (subtree). + * When empty, falls back to account_uid if set, otherwise matches all. + * + * @generated from field: string iri_filter = 7; + */ + iriFilter = ""; + + /** + * Optional. Fine-grained content type selection. Overrides include_body when set. + * When empty, legacy behavior (title + body types based on include_body). + * + * @generated from field: repeated com.seed.entities.v1alpha.ContentTypeFilter content_type_filter = 8; + */ + contentTypeFilter: ContentTypeFilter[] = []; + + /** + * Optional. Authority weight for citation-based ranking. Range [0, 1]. + * 0 (default) disables authority scoring. Higher values increase citation influence. + * Final score: (1-weight)*textRRF + 0.7*weight*docAuthRRF + 0.3*weight*authorAuthRRF. + * + * @generated from field: float authority_weight = 9; + */ + authorityWeight = 0; + + /** + * Optional. Maximum number of results per page. + * When 0 (default), all results are returned (backwards compatible). + * + * @generated from field: int32 page_size = 10; + */ + pageSize = 0; + + /** + * Optional. Token from a previous SearchEntitiesResponse to get the next page. + * + * @generated from field: string page_token = 11; + */ + pageToken = ""; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -857,6 +974,12 @@ export class SearchEntitiesRequest extends Message { { no: 3, name: "context_size", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, { no: 4, name: "account_uid", kind: "scalar", T: 9 /* ScalarType.STRING */ }, { no: 5, name: "logged_account_uid", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 6, name: "search_type", kind: "enum", T: proto3.getEnumType(SearchType) }, + { no: 7, name: "iri_filter", kind: "scalar", T: 9 /* ScalarType.STRING */ }, + { no: 8, name: "content_type_filter", kind: "enum", T: proto3.getEnumType(ContentTypeFilter), repeated: true }, + { no: 9, name: "authority_weight", kind: "scalar", T: 2 /* ScalarType.FLOAT */ }, + { no: 10, name: "page_size", kind: "scalar", T: 5 /* ScalarType.INT32 */ }, + { no: 11, name: "page_token", kind: "scalar", T: 9 /* ScalarType.STRING */ }, ]); static fromBinary(bytes: Uint8Array, options?: Partial): SearchEntitiesRequest { diff --git a/frontend/packages/shared/src/hm-types.ts b/frontend/packages/shared/src/hm-types.ts index a615fff5f..81d748761 100644 --- a/frontend/packages/shared/src/hm-types.ts +++ b/frontend/packages/shared/src/hm-types.ts @@ -1471,6 +1471,7 @@ export const HMSearchInputSchema = z.object({ includeBody: z.boolean().optional(), contextSize: z.number().optional(), perspectiveAccountUid: z.string().optional(), + searchType: z.number().optional(), }) export type HMSearchInput = z.infer diff --git a/frontend/packages/shared/src/models/search.ts b/frontend/packages/shared/src/models/search.ts index 7c4a5fda0..bea79cd0a 100644 --- a/frontend/packages/shared/src/models/search.ts +++ b/frontend/packages/shared/src/models/search.ts @@ -1,5 +1,6 @@ import {Timestamp} from '@bufbuild/protobuf' import {useQuery} from '@tanstack/react-query' +import {SearchType} from '../client/.generated/entities/v1alpha/entities_pb' import {HMDocument, HMSearchRequest, UnpackedHypermediaId} from '../hm-types' import {packHmId} from '../utils/entity-id-url' import {queryKeys} from './query-keys' @@ -29,12 +30,14 @@ export function useSearch( includeBody = false, contextSize = 48, perspectiveAccountUid, + searchType, }: { enabled?: boolean accountUid?: string includeBody?: boolean contextSize?: number perspectiveAccountUid?: string + searchType?: SearchType } = {}, ) { const client = useUniversalClient() @@ -46,6 +49,7 @@ export function useSearch( query, includeBody, contextSize, + searchType, ], queryFn: async () => { const out = await client.request('Search', { @@ -54,6 +58,7 @@ export function useSearch( accountUid: accountUid || undefined, includeBody: includeBody || false, contextSize: contextSize || 48, + searchType, }) const alreadySeenIds = new Set() const entities: SearchResultItem[] = [] diff --git a/frontend/packages/shared/src/routing.tsx b/frontend/packages/shared/src/routing.tsx index bfe4e7ed8..4427fe99b 100644 --- a/frontend/packages/shared/src/routing.tsx +++ b/frontend/packages/shared/src/routing.tsx @@ -26,6 +26,7 @@ export const appExperimentsSchema = z developerTools: z.boolean().optional(), pubContentDevMenu: z.boolean().optional(), newLibrary: z.boolean().optional(), + embeddingEnabled: z.boolean().optional(), }) .strict() export type AppExperiments = z.infer diff --git a/go.mod b/go.mod index a5cdc5736..c6876b854 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/multiformats/go-multibase v0.2.0 github.com/multiformats/go-multicodec v0.9.2 github.com/multiformats/go-multihash v0.2.3 + github.com/ollama/ollama v0.14.2 github.com/peterbourgon/ff/v4 v4.0.0-alpha.4 github.com/peterbourgon/trc v0.0.3 github.com/polydawn/refmt v0.89.0 @@ -58,7 +59,7 @@ require ( go.uber.org/zap v1.27.0 golang.org/x/exp v0.0.0-20250911091902-df9299821621 golang.org/x/sync v0.17.0 - golang.org/x/text v0.29.0 + golang.org/x/text v0.30.0 google.golang.org/grpc v1.75.0 google.golang.org/protobuf v1.36.9 roci.dev/fracdex v0.0.0-00010101000000-000000000000 @@ -70,7 +71,9 @@ require ( github.com/abiosoft/ishell v2.0.0+incompatible // indirect github.com/abiosoft/readline v0.0.0-20180607040430-155bce2042db // indirect github.com/alessio/shellescape v1.4.1 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/danieljoos/wincred v1.2.0 // indirect github.com/ebitengine/purego v0.9.0 // indirect github.com/fatih/color v1.12.0 // indirect @@ -84,6 +87,7 @@ require ( github.com/ipfs/go-dsqueue v0.0.5 // indirect github.com/libp2p/go-libp2p-record v0.3.1 // indirect github.com/libp2p/go-yamux/v5 v5.0.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mschoch/smat v0.2.0 // indirect @@ -110,10 +114,11 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect github.com/wlynxg/anet v0.0.5 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect - golang.org/x/telemetry v0.0.0-20250908211612-aef8a434d053 // indirect + golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 // indirect golang.org/x/time v0.12.0 // indirect google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.5.1 // indirect @@ -223,6 +228,7 @@ require ( github.com/rs/cors v1.7.0 // indirect github.com/sahilm/fuzzy v0.1.1 github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/tcpipuk/llama-go v0.0.0-20260108175825-f54e6b8263d7 github.com/tklauser/go-sysconf v0.3.15 // indirect github.com/tklauser/numcpus v0.10.0 // indirect github.com/whyrusleeping/cbor-gen v0.3.1 // indirect @@ -239,12 +245,12 @@ require ( go.uber.org/dig v1.19.0 // indirect go.uber.org/fx v1.24.0 // indirect go.uber.org/mock v0.5.2 // indirect - golang.org/x/crypto v0.42.0 // indirect - golang.org/x/mod v0.28.0 // indirect - golang.org/x/net v0.44.0 + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/mod v0.30.0 // indirect + golang.org/x/net v0.46.0 golang.org/x/sys v0.37.0 // indirect - golang.org/x/term v0.35.0 - golang.org/x/tools v0.37.0 // indirect + golang.org/x/term v0.36.0 + golang.org/x/tools v0.38.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect gonum.org/v1/gonum v0.16.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250825161204-c5933d9347a5 // indirect @@ -253,7 +259,10 @@ require ( nhooyr.io/websocket v1.8.7 // indirect ) -replace roci.dev/fracdex => github.com/rocicorp/fracdex v0.0.0-20231009204907-ebc26eac9486 +replace ( + github.com/tcpipuk/llama-go => ./backend/util/llama-go + roci.dev/fracdex => github.com/rocicorp/fracdex v0.0.0-20231009204907-ebc26eac9486 +) // LND imports etcd, which imports some very old version of OpenTelemetry, // and it break the build in many different but miserable ways. diff --git a/go.sum b/go.sum index 5cbd12488..16d305414 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RoaringBitmap/roaring/v2 v2.4.2 h1:ew/INI7HLRyYK+dCbF6FcUwoe2Q0q5HCV7WafY9ljBk= github.com/RoaringBitmap/roaring/v2 v2.4.2/go.mod h1:FiJcsfkGje/nZBZgCu0ZxCPOKD/hVXDS2dXi7/eUFE0= @@ -70,6 +72,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:W github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -138,12 +142,18 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE github.com/btcsuite/winsvc v1.0.0 h1:J9B4L7e3oqhXOcm+2IuNApwzQec85lE+QaikUcCs+dk= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/burdiyan/go-erriter v0.0.0-20251126131818-84c9a62b84d2 h1:IiIGGudmB/7G21DRscg3gHNDVvh3FeqNYTNl6seMc3w= github.com/burdiyan/go-erriter v0.0.0-20251126131818-84c9a62b84d2/go.mod h1:+6ibPBKYd5uvc3fSOlPj2Uug6sTob0Z+4e6UqSuBtsU= github.com/burdiyan/go/mainutil v0.0.0-20200124222818-6f87e0e684b6 h1:6H15Dgf4zZ8KlEXg3gHXVeSaJ7lmxmPwTGuhtuZuL2w= github.com/burdiyan/go/mainutil v0.0.0-20200124222818-6f87e0e684b6/go.mod h1:rw0aHTLAgD7uczBMUzhtLU8+OH5NZbJOg56QnIC6YF0= github.com/bwmarrin/discordgo v0.28.1 h1:gXsuo2GBO7NbR6uqmrrBDplPUx2T3nzu775q/Rd1aG4= github.com/bwmarrin/discordgo v0.28.1/go.mod h1:NJZpH+1AfhIcyQsPeuBKsUtYrRnjkyu0kIVMCHkZtRY= +github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc0= +github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= +github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= @@ -165,6 +175,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWs github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -270,6 +284,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/gabriel-vasile/mimetype v1.4.10 h1:zyueNbySn/z8mJZHLt6IPw0KoZsiQNszIpU+bX4+ZK0= +github.com/gabriel-vasile/mimetype v1.4.10/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gammazero/chanqueue v1.1.1 h1:n9Y+zbBxw2f7uUE9wpgs0rOSkP/I/yhDLiNuhyVjojQ= github.com/gammazero/chanqueue v1.1.1/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= github.com/gammazero/deque v1.1.0 h1:OyiyReBbnEG2PP0Bnv1AASLIYvyKqIFN5xfl1t8oGLo= @@ -281,8 +297,8 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= -github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= +github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= +github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= @@ -305,16 +321,19 @@ github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= -github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= -github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= +github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= +github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -324,8 +343,8 @@ github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -391,6 +410,8 @@ github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20250923004556-9e5a51aed1e8 h1:ZI8gCoCjGzPsum4L21jHdQs8shFBIQih1TM9Rd/c+EQ= +github.com/google/pprof v0.0.0-20250923004556-9e5a51aed1e8/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -546,6 +567,7 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= @@ -595,8 +617,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= +github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-buffer-pool v0.1.0 h1:oK4mSFcQz7cTQIfqbe4MIj9gLW+mnanjyFtc6cdF0Y8= @@ -665,6 +687,8 @@ github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/marcopolo/simnet v0.0.1 h1:rSMslhPz6q9IvJeFWDoMGxMIrlsbXau3NkuIXHGJxfg= github.com/marcopolo/simnet v0.0.1/go.mod h1:WDaQkgLAjqDUEBAOXz22+1j6wXKfGlC5sD5XWt3ddOs= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= @@ -768,14 +792,21 @@ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn github.com/oklog/ulid/v2 v2.1.0 h1:+9lhoxAP56we25tyYETBBY1YLA2SaoLvUFgrP2miPJU= github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/ollama/ollama v0.14.2 h1:nPPaf5I6aMpPr94Au4syTeyQUqR2ctojryUl4aq7e5g= +github.com/ollama/ollama v0.14.2/go.mod h1:4Yn3jw2hZ4VqyJ1XciYawDRE8bzv4RT3JiVZR1kCfwE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.25.3 h1:Ty8+Yi/ayDAGtk4XxmmfUy4GabvM+MegeB4cDLRi6nw= +github.com/onsi/ginkgo/v2 v2.25.3/go.mod h1:43uiyQC4Ed2tkOzLsEYm7hnrb7UJTWHYNsuy3bG/snE= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= @@ -794,8 +825,8 @@ github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwp github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= -github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= +github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/peterbourgon/ff/v4 v4.0.0-alpha.4 h1:aiqS8aBlF9PsAKeMddMSfbwp3smONCn3UO8QfUg0Z7Y= github.com/peterbourgon/ff/v4 v4.0.0-alpha.4/go.mod h1:H/13DK46DKXy7EaIxPhk2Y0EC8aubKm35nBjBe8AAGc= @@ -1037,13 +1068,15 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= +github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -1061,6 +1094,8 @@ github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/wlynxg/anet v0.0.3/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= @@ -1139,6 +1174,8 @@ go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= go.uber.org/dig v1.19.0 h1:BACLhebsYdpQ7IROQ1AGPjrXcP5dF80U3gKoFzbaq/4= go.uber.org/dig v1.19.0/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.24.0 h1:wE8mruvpg2kiiL1Vqd0CC+tr0/24XIB10Iwp2lLWzkg= @@ -1162,7 +1199,11 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= +golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= +golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -1185,8 +1226,8 @@ golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= -golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= -golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1218,8 +1259,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= -golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= +golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= +golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1264,8 +1305,8 @@ golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= -golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= -golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1353,8 +1394,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/telemetry v0.0.0-20250908211612-aef8a434d053 h1:dHQOQddU4YHS5gY33/6klKjq7Gp3WwMyOXGNp5nzRj8= -golang.org/x/telemetry v0.0.0-20250908211612-aef8a434d053/go.mod h1:+nZKN+XVh4LCiA9DV3ywrzN4gumyCnKjau3NGb9SGoE= +golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8 h1:LvzTn0GQhWuvKH/kVRS3R3bVAsdQWI7hvfLHGgh9+lU= +golang.org/x/telemetry v0.0.0-20251008203120-078029d740a8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -1362,8 +1403,8 @@ golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= -golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= -golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -1375,8 +1416,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1417,8 +1458,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= -golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/mise.toml b/mise.toml index d9c9327a7..8e826f935 100644 --- a/mise.toml +++ b/mise.toml @@ -7,6 +7,96 @@ go = "1.25.4" node = "22.2.0" protoc = "24.4" pnpm = "9.15.0" +cmake = "3.31.6" +golangci-lint = "2.8.0" [settings] experimental = true + +[env] +_.file = ".env" + +# GPU acceleration for llama.cpp: +# - macOS: Metal (built-in, no extra packages needed) +# - Linux: CPU-only for local dev (Vulkan used in CI/production only) +# +# If you need Vulkan on Linux for local GPU testing: +# Fedora/RHEL: sudo dnf install vulkan-headers vulkan-loader-devel glslang gcc-c++ +# Ubuntu/Debian: sudo apt install libvulkan-dev vulkan-tools glslc g++ + +[tasks.ensure-submodule] +run = ''' +if [ -f .gitmodules ] && [ ! -f backend/util/llama-go/Makefile ]; then + echo "Initializing git submodules (llama-go + llama.cpp)..." + git submodule update --init --recursive +fi +''' +hide = true + +[tasks.ensure-model] +run = ''' +MODEL="backend/llm/backends/llamacpp/models/granite-embedding-107m-multilingual-Q8_0.gguf" +if [ ! -f "$MODEL" ]; then + mkdir -p "$(dirname "$MODEL")" + echo "Downloading GGUF embedding model..." + curl -fSL --progress-bar -o "$MODEL" \ + "https://huggingface.co/keisuke-miyako/granite-embedding-107m-multilingual-gguf-q8_0/resolve/main/granite-embedding-107m-multilingual-Q8_0.gguf?download=true" +fi +''' +hide = true + +[tasks.ensure-llama-libs] +depends = ["ensure-submodule"] +run = ''' +LLAMA_GO_DIR="backend/util/llama-go" +NEEDS_BUILD=false + +if [ ! -f "$LLAMA_GO_DIR/libbinding.a" ]; then + NEEDS_BUILD=true +elif [ "$(uname -s)" = "Darwin" ] && [ ! -f "$LLAMA_GO_DIR/libggml-blas.a" ]; then + # macOS requires Metal build. libggml-blas.a only exists after a Metal build. + # If missing, a stale CPU-only build exists and must be replaced. + NEEDS_BUILD=true +fi + +if [ "$NEEDS_BUILD" = "true" ]; then + cd "$LLAMA_GO_DIR" + if [ "$(uname -s)" = "Darwin" ]; then + echo "Building llama.cpp libraries with Metal (this may take a few minutes)..." + BUILD_TYPE=metal CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF" make libbinding.a + cp build/bin/ggml-metal.metal . 2>/dev/null || true + # Stub for Vulkan (not used on macOS). + touch libggml-vulkan.a + else + echo "Building llama.cpp libraries, CPU-only (this may take a few minutes)..." + CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF -DGGML_VULKAN=OFF -DGGML_METAL=OFF -DGGML_CUDA=OFF -DGGML_HIP=OFF -DGGML_SYCL=OFF -DGGML_BLAS=OFF" make libbinding.a + # Stubs for GPU libraries (not used in CPU-only build, but needed for linking). + touch libggml-vulkan.a + touch libggml-metal.a + fi + echo "llama.cpp libraries built successfully." +fi +''' +hide = true + +[tasks.check-gpu] +description = "Check if GPU acceleration dependencies are installed for your platform" +run = ''' +if [ "$(uname -s)" = "Darwin" ]; then + echo "macOS: Metal support is built-in and always enabled" +else + echo "Linux: local dev uses CPU-only builds. Vulkan is used in CI/production." + echo "" + echo "To check Vulkan availability (optional):" + if command -v glslc >/dev/null 2>&1; then + echo " glslc: found" + else + echo " glslc: not found (install: sudo dnf install glslang / sudo apt install glslc)" + fi + if pkg-config --exists vulkan 2>/dev/null; then + echo " vulkan: found" + else + echo " vulkan: not found (install: sudo dnf install vulkan-headers vulkan-loader-devel)" + fi +fi +''' diff --git a/proto/daemon/v1alpha/daemon.proto b/proto/daemon/v1alpha/daemon.proto index 25de80c9e..6af6835f1 100644 --- a/proto/daemon/v1alpha/daemon.proto +++ b/proto/daemon/v1alpha/daemon.proto @@ -246,6 +246,12 @@ enum TaskName { // Task for reindexing the database. REINDEXING = 1; + + // Task for generating embeddings. + EMBEDDING = 2; + + // Task for loading a machine learning model. + LOADING_MODEL = 3; } // Description of a task that the daemon is performing. diff --git a/proto/daemon/v1alpha/go.gensum b/proto/daemon/v1alpha/go.gensum index f65b66c6b..d8cdcf6d8 100644 --- a/proto/daemon/v1alpha/go.gensum +++ b/proto/daemon/v1alpha/go.gensum @@ -1,2 +1,2 @@ -srcs: 3cb88f9722f6be3f203db2935d7cabc5 -outs: 1652b1b1af3b53c1a507522122e044dc +srcs: cbb4bb808c8fcda2d5db4646c2832881 +outs: 309fe02254dd5f37196cb2a9dfa162c3 diff --git a/proto/daemon/v1alpha/js.gensum b/proto/daemon/v1alpha/js.gensum index 35da5d91c..aafbd0b17 100644 --- a/proto/daemon/v1alpha/js.gensum +++ b/proto/daemon/v1alpha/js.gensum @@ -1,2 +1,2 @@ -srcs: 3cb88f9722f6be3f203db2935d7cabc5 -outs: 014757e1c49bfa917487ba7ef42c4e84 +srcs: cbb4bb808c8fcda2d5db4646c2832881 +outs: 1e203572e934e1b578170ebcd64947ad diff --git a/proto/entities/v1alpha/entities.proto b/proto/entities/v1alpha/entities.proto index e09cca30c..75f5baefd 100644 --- a/proto/entities/v1alpha/entities.proto +++ b/proto/entities/v1alpha/entities.proto @@ -252,30 +252,74 @@ message DeletedEntity { // Further metadata about the deleted entity, title, etc ... string metadata = 4; } -// Request to + +// Describes the state of the discovery task. +enum SearchType { + // Keyword-based search. + SEARCH_KEYWORD = 0; + + // Semantic search. + SEARCH_SEMANTIC = 1; + + // Hybrid search. with RRFusion. + SEARCH_HYBRID = 2; +} + +// Content type to filter search results by. +enum ContentTypeFilter { + CONTENT_TYPE_TITLE = 0; + CONTENT_TYPE_DOCUMENT = 1; + CONTENT_TYPE_COMMENT = 2; + CONTENT_TYPE_CONTACT = 3; +} + +// Request to search entities. message SearchEntitiesRequest { - // Query to find. We Ssupport wildcards and phrases. + // Query to find. We support wildcards and phrases. // See https://sqlite.org/fts5.html#full_text_query_syntax. string query = 1; - // Whether to look into all content available or just the titles. - // If false, comments are not included in the search. - // Default is false. - bool include_body = 2; + // Deprecated, use content_type_filters instead to specify + // which content types to include in the search. + bool include_body = 2 [deprecated = true]; // Optional. The size of the text accompanying the search match. // Half of the size is before the match, and half after. // Default is 48 runes. int32 context_size = 3; - // Optional. The account uid to filter the search by. - // If not set, the search will be performed across all accounts. - string account_uid = 4; + // Deprecated. Use iri_filter instead. + string account_uid = 4 [deprecated = true]; // Optional. The account uid the user is logged in with. // This is used to filter out contacts that the user doesn't have access to. // If not set, we won't provide any contact entities in the response. string logged_account_uid = 5; + + // Optional. Type of search to perform. Could be keyword, semantic or hybrid. + // if not set, keyword search is used. + SearchType search_type = 6; + + // Optional. hm:// URL with optional GLOB wildcards to scope search. + // Examples: "hm:///cars/honda" (single doc), "hm:///cars/*" (subtree). + // When empty, falls back to account_uid if set, otherwise matches all. + string iri_filter = 7; + + // Optional. Fine-grained content type selection. Overrides include_body when set. + // When empty, legacy behavior (title + body types based on include_body). + repeated ContentTypeFilter content_type_filter = 8; + + // Optional. Authority weight for citation-based ranking. Range [0, 1]. + // 0 (default) disables authority scoring. Higher values increase citation influence. + // Final score: (1-weight)*textRRF + 0.7*weight*docAuthRRF + 0.3*weight*authorAuthRRF. + float authority_weight = 9; + + // Optional. Maximum number of results per page. + // When 0 (default), all results are returned (backwards compatible). + int32 page_size = 10; + + // Optional. Token from a previous SearchEntitiesResponse to get the next page. + string page_token = 11; } // A list of entities matching the request. diff --git a/proto/entities/v1alpha/go.gensum b/proto/entities/v1alpha/go.gensum index c95b675d8..ddb0fee66 100644 --- a/proto/entities/v1alpha/go.gensum +++ b/proto/entities/v1alpha/go.gensum @@ -1,2 +1,2 @@ -srcs: 4c13f9e29653e29e87be52a564a603b6 -outs: cd9ff19ce97c986f5dfac244cca94e51 +srcs: 9136e7596622d5771d7efd8eeccddff8 +outs: b188774ced3890622882d81281aca68f diff --git a/proto/entities/v1alpha/js.gensum b/proto/entities/v1alpha/js.gensum index 71c64a8c3..f59b8ea57 100644 --- a/proto/entities/v1alpha/js.gensum +++ b/proto/entities/v1alpha/js.gensum @@ -1,2 +1,2 @@ -srcs: 4c13f9e29653e29e87be52a564a603b6 -outs: baa2f0b94be3a8eb60b612803fbe022e +srcs: 9136e7596622d5771d7efd8eeccddff8 +outs: aed50b569196f7673e19b4c924e4538d diff --git a/tests/integration/daemon.ts b/tests/integration/daemon.ts index 959f84c82..b218cf3bb 100644 --- a/tests/integration/daemon.ts +++ b/tests/integration/daemon.ts @@ -3,179 +3,184 @@ * Spawns a seed-daemon process with custom ports and data directory. */ -import {spawn, ChildProcess} from 'child_process' -import * as readline from 'node:readline' -import path from 'path' +import { spawn, ChildProcess } from "child_process"; +import * as readline from "node:readline"; +import path from "path"; export type DaemonConfig = { - httpPort: number - grpcPort: number - p2pPort: number - dataDir: string -} + httpPort: number; + grpcPort: number; + p2pPort: number; + dataDir: string; +}; export type DaemonInstance = { - process: ChildProcess - config: DaemonConfig - kill: () => Promise - waitForReady: () => Promise -} + process: ChildProcess; + config: DaemonConfig; + kill: () => Promise; + waitForReady: () => Promise; +}; function getDaemonBinaryPath(): string { - const platform = process.platform - const arch = process.arch + const platform = process.platform; + const arch = process.arch; - let triple: string + let triple: string; switch (`${platform}/${arch}`) { - case 'darwin/x64': - triple = 'x86_64-apple-darwin' - break - case 'darwin/arm64': - triple = 'aarch64-apple-darwin' - break - case 'win32/x64': - triple = 'x86_64-pc-windows-msvc' - break - case 'linux/x64': - triple = 'x86_64-unknown-linux-gnu' - break - case 'linux/arm64': - triple = 'aarch64-unknown-linux-gnu' - break + case "darwin/x64": + triple = "x86_64-apple-darwin"; + break; + case "darwin/arm64": + triple = "aarch64-apple-darwin"; + break; + case "win32/x64": + triple = "x86_64-pc-windows-gnu"; + break; + case "linux/x64": + triple = "x86_64-unknown-linux-gnu"; + break; + case "linux/arm64": + triple = "aarch64-unknown-linux-gnu"; + break; default: - throw new Error(`Unsupported platform: ${platform}/${arch}`) + throw new Error(`Unsupported platform: ${platform}/${arch}`); } // tests/integration/daemon.ts -> repo root is ../.. - const repoRoot = path.resolve(__dirname, '../..') - return path.join(repoRoot, `plz-out/bin/backend/seed-daemon-${triple}`) + const repoRoot = path.resolve(__dirname, "../.."); + return path.join(repoRoot, `plz-out/bin/backend/seed-daemon-${triple}`); } -export async function spawnDaemon(config: DaemonConfig): Promise { - const binaryPath = getDaemonBinaryPath() +export async function spawnDaemon( + config: DaemonConfig, +): Promise { + const binaryPath = getDaemonBinaryPath(); const args = [ - '-http.port', + "-http.port", String(config.httpPort), - '-grpc.port', + "-grpc.port", String(config.grpcPort), - '-p2p.port', + "-p2p.port", String(config.p2pPort), - '-log-level=debug', - '-data-dir', + "-log-level=debug", + "-data-dir", config.dataDir, - '-syncing.smart=true', - '-syncing.no-sync-back=true', - '-lndhub.mainnet=false', - ] + "-syncing.smart=true", + "-syncing.no-sync-back=true", + "-lndhub.mainnet=false", + ]; - console.log(`[Daemon] Spawning: ${binaryPath}`) - console.log(`[Daemon] Args: ${args.join(' ')}`) + console.log(`[Daemon] Spawning: ${binaryPath}`); + console.log(`[Daemon] Args: ${args.join(" ")}`); const daemonProcess = spawn(binaryPath, args, { - stdio: 'pipe', + stdio: "pipe", env: { ...process.env, }, - }) - - let isReady = false - let readyResolve: (() => void) | null = null - let readyReject: ((err: Error) => void) | null = null - - const stderr = readline.createInterface({input: daemonProcess.stderr!}) - stderr.on('line', (line: string) => { - console.log(`[Daemon stderr] ${line}`) - if (line.includes('DaemonStarted')) { - isReady = true - readyResolve?.() + }); + + let isReady = false; + let readyResolve: (() => void) | null = null; + let readyReject: ((err: Error) => void) | null = null; + + const stderr = readline.createInterface({ input: daemonProcess.stderr! }); + stderr.on("line", (line: string) => { + console.log(`[Daemon stderr] ${line}`); + if (line.includes("DaemonStarted")) { + isReady = true; + readyResolve?.(); } - }) + }); - const stdout = readline.createInterface({input: daemonProcess.stdout!}) - stdout.on('line', (line: string) => { - console.log(`[Daemon stdout] ${line}`) - }) + const stdout = readline.createInterface({ input: daemonProcess.stdout! }); + stdout.on("line", (line: string) => { + console.log(`[Daemon stdout] ${line}`); + }); - daemonProcess.on('error', (err) => { - console.error('[Daemon] Spawn error:', err) - readyReject?.(err) - }) + daemonProcess.on("error", (err) => { + console.error("[Daemon] Spawn error:", err); + readyReject?.(err); + }); - daemonProcess.on('close', (code, signal) => { - console.log(`[Daemon] Closed with code=${code}, signal=${signal}`) + daemonProcess.on("close", (code, signal) => { + console.log(`[Daemon] Closed with code=${code}, signal=${signal}`); if (!isReady) { - readyReject?.(new Error(`Daemon exited before ready: code=${code}`)) + readyReject?.(new Error(`Daemon exited before ready: code=${code}`)); } - }) + }); const waitForReady = async (): Promise => { - if (isReady) return + if (isReady) return; await new Promise((resolve, reject) => { - readyResolve = resolve - readyReject = reject + readyResolve = resolve; + readyReject = reject; // Timeout after 60 seconds setTimeout(() => { - reject(new Error('Daemon startup timeout (60s)')) - }, 60_000) - }) + reject(new Error("Daemon startup timeout (60s)")); + }, 60_000); + }); // Also wait for HTTP endpoint to be ready - await waitForHttpReady(config.httpPort) - } + await waitForHttpReady(config.httpPort); + }; const kill = (): Promise => { return new Promise((resolve) => { - console.log('[Daemon] Killing process...') + console.log("[Daemon] Killing process..."); // Close readline interfaces to prevent "Channel closed" errors - stderr.close() - stdout.close() + stderr.close(); + stdout.close(); if (daemonProcess.exitCode !== null) { // Already exited - resolve() - return + resolve(); + return; } - daemonProcess.once('close', () => resolve()) - daemonProcess.kill() + daemonProcess.once("close", () => resolve()); + daemonProcess.kill(); // Force kill after 5s if graceful shutdown fails setTimeout(() => { if (daemonProcess.exitCode === null) { - daemonProcess.kill('SIGKILL') + daemonProcess.kill("SIGKILL"); } - resolve() - }, 5000) - }) - } + resolve(); + }, 5000); + }); + }; return { process: daemonProcess, config, kill, waitForReady, - } + }; } -async function waitForHttpReady(port: number, timeoutMs = 30_000): Promise { - const startTime = Date.now() +async function waitForHttpReady( + port: number, + timeoutMs = 30_000, +): Promise { + const startTime = Date.now(); while (Date.now() - startTime < timeoutMs) { try { - const response = await fetch(`http://localhost:${port}/debug/version`) + const response = await fetch(`http://localhost:${port}/debug/version`); if (response.ok) { - console.log(`[Daemon] HTTP endpoint ready on port ${port}`) - return + console.log(`[Daemon] HTTP endpoint ready on port ${port}`); + return; } } catch (e) { // Not ready yet } - await new Promise((resolve) => setTimeout(resolve, 200)) + await new Promise((resolve) => setTimeout(resolve, 200)); } - throw new Error(`HTTP endpoint not ready after ${timeoutMs}ms`) + throw new Error(`HTTP endpoint not ready after ${timeoutMs}ms`); }