diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 01d9b872..eeef3631 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,58 +14,6 @@ jobs: - name: Typecheck files run: yarn typecheck - test-android: - runs-on: ubuntu-latest - env: - TURBO_CACHE_DIR: .turbo/android - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Turn off addons - run: | - node ./scripts/turnOffEverything.js - - - name: Setup - uses: ./.github/actions/setup - - - name: Install JDK - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '17' - - - name: Finalize Android SDK - run: | - /bin/bash -c "yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --licenses > /dev/null" - - - name: Cache Gradle - uses: actions/cache@v4 - with: - path: | - ~/.gradle/wrapper - ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('example/android/gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- - - - name: Enable KVM - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - - name: Give execute permissions to script - run: chmod +x ./scripts/test-android.sh - - - name: run tests - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: 29 - emulator-options: -no-window -no-boot-anim -no-audio -no-snapshot-load - script: | - ./scripts/test-android.sh - test-ios: runs-on: macos-latest steps: @@ -234,17 +182,32 @@ jobs: run: | ./scripts/test-ios.sh - test-android-sqlcipher: + test-android: runs-on: ubuntu-latest - env: - TURBO_CACHE_DIR: .turbo/android + timeout-minutes: 20 steps: + - name: Free Disk Space (Ubuntu) + uses: insightsengineering/disk-space-reclaimer@v1 + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tools-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: false + dotnet: true + haskell: true + large-packages: true + swap-storage: true + docker-images: true + - name: Checkout uses: actions/checkout@v4 - - name: Turn on SQLCipher + - name: Turn off addons run: | - node ./scripts/turnOnSQLCipher.js + node ./scripts/turnOffEverything.js - name: Setup uses: ./.github/actions/setup @@ -252,22 +215,84 @@ jobs: - name: Install JDK uses: actions/setup-java@v3 with: - distribution: 'zulu' - java-version: '17' + distribution: "temurin" + java-version: "17" - - name: Finalize Android SDK + - name: Enable KVM run: | - /bin/bash -c "yes | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --licenses > /dev/null" + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm - - name: Cache Gradle - uses: actions/cache@v4 + - name: Gradle cache + uses: gradle/actions/setup-gradle@v3 + + # - name: AVD cache + # uses: actions/cache@v4 + # id: avd-cache + # with: + # path: | + # ~/.android/avd/* + # ~/.android/adb* + # key: avd-29 + + - name: create AVD and generate snapshot for caching + # if: steps.avd-cache.outputs.cache-hit != 'true' + uses: reactivecircus/android-emulator-runner@v2 with: - path: | - ~/.gradle/wrapper - ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('example/android/gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - ${{ runner.os }}-gradle- + api-level: 29 + force-avd-creation: false + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + disable-animations: false + script: echo "Generated AVD snapshot for caching." + + - name: Give execute permissions to script + run: chmod +x ./scripts/test-android.sh + + - name: run tests + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 29 + force-avd-creation: false + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + disable-animations: true + script: | + ./scripts/test-android.sh + + test-android-sqlcipher: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Free Disk Space (Ubuntu) + uses: insightsengineering/disk-space-reclaimer@v1 + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tools-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: false + dotnet: true + haskell: true + large-packages: true + swap-storage: true + docker-images: true + - name: Checkout + uses: actions/checkout@v4 + + - name: Turn on SQLCipher + run: | + node ./scripts/turnOnSQLCipher.js + + - name: Setup + uses: ./.github/actions/setup + + - name: Install JDK + uses: actions/setup-java@v3 + with: + distribution: "temurin" + java-version: "17" - name: Enable KVM run: | @@ -275,6 +300,28 @@ jobs: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm + - name: Gradle cache + uses: gradle/actions/setup-gradle@v3 + + # - name: AVD cache + # uses: actions/cache@v4 + # id: avd-cache + # with: + # path: | + # ~/.android/avd/* + # ~/.android/adb* + # key: avd-29 + + - name: create AVD and generate snapshot for caching + # if: steps.avd-cache.outputs.cache-hit != 'true' + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 29 + force-avd-creation: false + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + disable-animations: false + script: echo "Generated AVD snapshot for caching." + - name: Give execute permissions to script run: chmod +x ./scripts/test-android.sh @@ -282,7 +329,9 @@ jobs: uses: reactivecircus/android-emulator-runner@v2 with: api-level: 29 - emulator-options: -no-window -no-boot-anim -no-audio -no-snapshot-load + force-avd-creation: false + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + disable-animations: true script: | adb wait-for-device adb shell input keyevent 82 @@ -303,8 +352,8 @@ jobs: - name: Install JDK uses: actions/setup-java@v3 with: - distribution: 'zulu' - java-version: '17' + distribution: "zulu" + java-version: "17" - name: Finalize Android SDK run: | diff --git a/android/src/main/libsqlitevec/arm64-v8a/libsqlite_vec.so b/android/src/main/libsqlitevec/arm64-v8a/libsqlite_vec.so old mode 100644 new mode 100755 index d92bc31a..cba097c9 Binary files a/android/src/main/libsqlitevec/arm64-v8a/libsqlite_vec.so and b/android/src/main/libsqlitevec/arm64-v8a/libsqlite_vec.so differ diff --git a/android/src/main/libsqlitevec/armeabi-v7a/libsqlite_vec.so b/android/src/main/libsqlitevec/armeabi-v7a/libsqlite_vec.so old mode 100644 new mode 100755 index 95301544..1ded0a38 Binary files a/android/src/main/libsqlitevec/armeabi-v7a/libsqlite_vec.so and b/android/src/main/libsqlitevec/armeabi-v7a/libsqlite_vec.so differ diff --git a/android/src/main/libsqlitevec/x86/libsqlite_vec.so b/android/src/main/libsqlitevec/x86/libsqlite_vec.so old mode 100644 new mode 100755 index 369d0978..880836c5 Binary files a/android/src/main/libsqlitevec/x86/libsqlite_vec.so and b/android/src/main/libsqlitevec/x86/libsqlite_vec.so differ diff --git a/android/src/main/libsqlitevec/x86_64/libsqlite_vec.so b/android/src/main/libsqlitevec/x86_64/libsqlite_vec.so old mode 100644 new mode 100755 index 42b041d8..f6e70efc Binary files a/android/src/main/libsqlitevec/x86_64/libsqlite_vec.so and b/android/src/main/libsqlitevec/x86_64/libsqlite_vec.so differ diff --git a/cpp/PreparedStatementHostObject.h b/cpp/PreparedStatementHostObject.h index c930526e..5621d4e5 100644 --- a/cpp/PreparedStatementHostObject.h +++ b/cpp/PreparedStatementHostObject.h @@ -40,7 +40,6 @@ class PreparedStatementHostObject : public jsi::HostObject { jsi::Value get(jsi::Runtime &rt, const jsi::PropNameID &propNameID) override; private: - std::shared_ptr _thread_pool; #ifdef OP_SQLITE_USE_LIBSQL DB _db; libsql_stmt_t _stmt; @@ -49,6 +48,7 @@ class PreparedStatementHostObject : public jsi::HostObject { // This shouldn't be de-allocated until sqlite3_finalize is called on it sqlite3_stmt *_stmt; #endif + std::shared_ptr _thread_pool; }; } // namespace opsqlite diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 8cec003a..84a52561 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -2027,7 +2027,7 @@ EXTERNAL SOURCES: SPEC CHECKSUMS: FBLazyVector: 2e5b5553df729e080483373db6f045201ff4e6db hermes-engine: 273e30e7fb618279934b0b95ffab60ecedb7acf5 - op-sqlite: 137a619945c21df76fda2744680061b8f3251839 + op-sqlite: a0d8cbee3fc5f8c8b67272c3b07c3b4f07b5b17f OpServer: 9b3ebdeeb095950e760e3c39853cd06849421b35 RCTDeprecation: c6b36da89aa26090c8684d29c2868dcca2cd4554 RCTRequired: 1413a0844770d00fa1f1bb2da4680adfa8698065 diff --git a/example/package.json b/example/package.json index 4440166c..25ff2828 100644 --- a/example/package.json +++ b/example/package.json @@ -45,13 +45,13 @@ "node": ">=18" }, "op-sqlite": { - "libsql": true, + "libsql": false, "sqlcipher": false, "iosSqlite": false, "fts5": true, "rtree": true, "crsqlite": false, - "sqliteVec": false, + "sqliteVec": true, "performanceMode": true, "tokenizers": [ "wordtokenizer", diff --git a/example/src/App.tsx b/example/src/App.tsx index ea529267..a3e1c97d 100644 --- a/example/src/App.tsx +++ b/example/src/App.tsx @@ -17,6 +17,7 @@ export default function App() { const [openTime, setOpenTime] = useState(0); useEffect(() => { + console.log("App has started 🟢") const work = async () => { let start = performance.now(); open({ @@ -26,9 +27,11 @@ export default function App() { try { const results = await runTests(); + console.log("TESTS FINISHED 🟢") setServerResults(allTestsPassed(results)); setResults(results); } catch (e) { + console.log(`TEST FAILED 🟥 ${e}`) setServerResults(false); } diff --git a/example/src/server.ts b/example/src/server.ts index ed6ce773..ecbea647 100644 --- a/example/src/server.ts +++ b/example/src/server.ts @@ -13,12 +13,12 @@ server.get('/ping', async (_req, res) => { server.get('/results', async (_req, res) => { res.statusCode = 200; + res.contentType = 'application/json'; res.content = JSON.stringify({passed}); - // res.json({passed}, 200); }); server.listen(9000); -// console.log('Server listening on port 9000'); +console.log('🟢 HTTP Server listening on port 9000'); export function stopServer() { server.stop(); diff --git a/example/src/tests/queries.ts b/example/src/tests/queries.ts index d65cb72a..8fe2f997 100644 --- a/example/src/tests/queries.ts +++ b/example/src/tests/queries.ts @@ -14,6 +14,7 @@ import { describe, it, } from '@op-engineering/op-test'; +// import pkg from '../../package.json' describe('Queries tests', () => { let db: DB; @@ -775,4 +776,50 @@ describe('Queries tests', () => { const res = db.executeSync('PRAGMA user_version'); expect(res.rows).toDeepEqual([{user_version: 0}]); }); + + +// const sqliteVecEnabled = pkg?.['op-sqlite']?.sqliteVec === true; +// if (sqliteVecEnabled) { +// it('sqlite-vec extension: vector similarity search', async () => { +// // Create a virtual table for storing vectors +// await db.execute(` +// CREATE VIRTUAL TABLE vec_items USING vec0( +// embedding FLOAT[8] +// ) +// `); + +// // Insert some sample vectors +// await db.execute(` +// INSERT INTO vec_items(rowid, embedding) +// VALUES +// (1, '[-0.200, 0.250, 0.341, -0.211, 0.645, 0.935, -0.316, -0.924]'), +// (2, '[0.443, -0.501, 0.355, -0.771, 0.707, -0.708, -0.185, 0.362]'), +// (3, '[0.716, -0.927, 0.134, 0.052, -0.669, 0.793, -0.634, -0.162]'), +// (4, '[-0.710, 0.330, 0.656, 0.041, -0.990, 0.726, 0.385, -0.958]') +// `); + +// // Perform KNN query to find the 2 nearest neighbors +// const queryVector = '[0.890, 0.544, 0.825, 0.961, 0.358, 0.0196, 0.521, 0.175]'; +// const result = await db.execute(` +// SELECT rowid, distance +// FROM vec_items +// WHERE embedding MATCH ? +// ORDER BY distance +// LIMIT 2 +// `, [queryVector]); + +// // Verify results +// expect(result.rows.length).toEqual(2); +// expect(result.rows[0]!.rowid).toEqual(2); +// expect(result.rows[1]!.rowid).toEqual(1); + +// // Verify distances are positive numbers +// const distance0 = result.rows[0]!.distance as number; +// const distance1 = result.rows[1]!.distance as number; +// expect(typeof distance0).toEqual('number'); +// expect(distance0 > 0).toBeTruthy(); +// expect(distance1 > 0).toBeTruthy(); +// }); +// } + }); diff --git a/ios/sqlitevec.xcframework/ios-arm64/sqlitevec.framework/sqlitevec b/ios/sqlitevec.xcframework/ios-arm64/sqlitevec.framework/sqlitevec old mode 100644 new mode 100755 index 00e6bd66..04a97a14 Binary files a/ios/sqlitevec.xcframework/ios-arm64/sqlitevec.framework/sqlitevec and b/ios/sqlitevec.xcframework/ios-arm64/sqlitevec.framework/sqlitevec differ diff --git a/ios/sqlitevec.xcframework/ios-arm64_x86_64-simulator/sqlitevec.framework/sqlitevec b/ios/sqlitevec.xcframework/ios-arm64_x86_64-simulator/sqlitevec.framework/sqlitevec old mode 100644 new mode 100755 index 48df9015..9732bd81 Binary files a/ios/sqlitevec.xcframework/ios-arm64_x86_64-simulator/sqlitevec.framework/sqlitevec and b/ios/sqlitevec.xcframework/ios-arm64_x86_64-simulator/sqlitevec.framework/sqlitevec differ diff --git a/scripts/poll-in-app-server.js b/scripts/poll-in-app-server.js index b7d5b451..849787fd 100644 --- a/scripts/poll-in-app-server.js +++ b/scripts/poll-in-app-server.js @@ -2,12 +2,20 @@ const http = require('http'); async function pollInAppServer() { const startTime = Date.now(); - const maxDuration = 5 * 60 * 1000; // 5 minutes - const pollInterval = 5000; // 1 second + const maxDuration = 5 * 60 * 1000; // 3 minutes - tests can take time on CI + const pollInterval = 5000; // + + // Do an initial ping into the server + + try { + await makeHttpRequest('http://127.0.0.1:9000/ping') + console.log("🟢 Ping success") + } catch(e) { + console.error("Ping failed!") + } while (Date.now() - startTime < maxDuration) { try { - console.log('Polling in-app server for results...'); const response = await makeHttpRequest('http://127.0.0.1:9000/results'); if (response !== null) { @@ -29,7 +37,7 @@ async function pollInAppServer() { await new Promise((resolve) => setTimeout(resolve, pollInterval)); } - console.error('Polling failed after 5 minutes'); + console.error(`Polling timed out after ${Math.round(maxDuration/1000)} seconds`); process.exit(1); } diff --git a/scripts/test-android.sh b/scripts/test-android.sh index f76e9358..bc2ecfbf 100755 --- a/scripts/test-android.sh +++ b/scripts/test-android.sh @@ -4,9 +4,25 @@ set -ex cd example || exit adb wait-for-device +echo "Waiting for boot to complete..." +adb shell 'while [[ -z $(getprop sys.boot_completed) ]]; do sleep 1; done' +echo "Boot completed!" adb shell input keyevent 82 adb forward tcp:9000 tcp:9000 -JAVA_OPTS=-XX:MaxHeapSize=6g yarn run:android:release +# JAVA_OPTS=-XX:MaxHeapSize=6g yarn run:android:release -node ../scripts/poll-in-app-server.js \ No newline at end of file +yarn run:android:release + +# echo "Waiting 20 seconds for app to fully initialize and tests to start..." +# sleep 80 + +node ../scripts/poll-in-app-server.js || { + echo "❌ poll-in-app-server failed, printing device logs from app launch..." + adb logcat -d "*:E" # Show only errors first + echo "" + echo "=== Full logcat from app launch ===" + adb logcat -d | grep -E "(com.op.sqlite|ReactNative|FATAL|AndroidRuntime)" || adb logcat -d | tail -200 + kill $APP_PID 2>/dev/null || true + exit 1 +} \ No newline at end of file diff --git a/scripts/turnOffEverything.js b/scripts/turnOffEverything.js index 16b084a7..fc4f8e15 100644 --- a/scripts/turnOffEverything.js +++ b/scripts/turnOffEverything.js @@ -12,7 +12,7 @@ packageJson['op-sqlite']['iosSqlite'] = false; packageJson['op-sqlite']['fts5'] = true; packageJson['op-sqlite']['rtree'] = true; packageJson['op-sqlite']['crsqlite'] = false; -packageJson['op-sqlite']['sqliteVec'] = true; +packageJson['op-sqlite']['sqliteVec'] = false; // Save the updated package.json file fs.writeFileSync( diff --git a/scripts/turnOnLibsql.js b/scripts/turnOnLibsql.js index d2692b2b..a0e32549 100644 --- a/scripts/turnOnLibsql.js +++ b/scripts/turnOnLibsql.js @@ -8,6 +8,7 @@ packageJson['op-sqlite']['libsql'] = true; packageJson['op-sqlite']['sqlcipher'] = false; packageJson['op-sqlite']['ioSqlite'] = false; delete packageJson['op-sqlite']['tokenizers']; +packageJson['op-sqlite']['sqliteVec'] = false; // Save the updated package.json file fs.writeFileSync( diff --git a/scripts/turnOnSQLCipher.js b/scripts/turnOnSQLCipher.js index 92d5110f..5dbc3205 100644 --- a/scripts/turnOnSQLCipher.js +++ b/scripts/turnOnSQLCipher.js @@ -7,6 +7,7 @@ const packageJson = JSON.parse(fs.readFileSync('./example/package.json')); packageJson['op-sqlite']['sqlcipher'] = true; packageJson['op-sqlite']['libsql'] = false; packageJson['op-sqlite']['iosSqlite'] = false; +packageJson['op-sqlite']['sqliteVec'] = false; // Save the updated package.json file fs.writeFileSync( diff --git a/scripts/update-sqlitevec.sh b/scripts/update-sqlitevec.sh new file mode 100755 index 00000000..667fa0c4 --- /dev/null +++ b/scripts/update-sqlitevec.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +# Change to the iOS directory +cd "$(dirname "$0")/../ios" + +echo "📦 Updating sqlitevec.xcframework binaries..." + +# Device (ios-arm64) +echo "🔨 Processing device binary (ios-arm64)..." +cp vec0-ios-aarch64.dylib sqlitevec +install_name_tool -id @rpath/sqlitevec.framework/sqlitevec sqlitevec +codesign -f -s - --identifier com.op.sqlitevec sqlitevec +mv sqlitevec sqlitevec.xcframework/ios-arm64/sqlitevec.framework/ + +# Simulator (ios-arm64_x86_64-simulator) - create fat binary +echo "🔨 Processing simulator binaries (arm64 + x86_64)..." +lipo -create vec0-iossimulator-aarch64.dylib vec0-iossimulator-x86_64.dylib -output sqlitevec +install_name_tool -id @rpath/sqlitevec.framework/sqlitevec sqlitevec +codesign -f -s - --identifier com.op.sqlitevec sqlitevec +mv sqlitevec sqlitevec.xcframework/ios-arm64_x86_64-simulator/sqlitevec.framework/ + +echo "✅ sqlitevec.xcframework updated successfully!"