v1.1.0

cpp/HybridCactus.cpp

@@ -75,6 +75,53 @@ std::shared_ptr<Promise<std::string>> HybridCactus::complete(
   });
 }
 
+std::shared_ptr<Promise<std::string>> HybridCactus::transcribe(
+    const std::string &audioFilePath, const std::string &prompt,
+    double responseBufferSize, const std::optional<std::string> &optionsJson,
+    const std::optional<std::function<void(const std::string & /* token */,
+                                           double /* tokenId */)>> &callback) {
+  return Promise<std::string>::async([this, audioFilePath, prompt, optionsJson,
+                                      callback,
+                                      responseBufferSize]() -> std::string {
+    std::lock_guard<std::mutex> lock(this->_modelMutex);
+
+    if (!this->_model) {
+      throw std::runtime_error("Cactus model is not initialized");
+    }
+
+    struct CallbackCtx {
+      const std::function<void(const std::string & /* token */,
+                               double /* tokenId */)> *callback;
+    } callbackCtx{callback.has_value() ? &callback.value() : nullptr};
+
+    auto cactusTokenCallback = [](const char *token, uint32_t tokenId,
+                                  void *userData) {
+      auto *callbackCtx = static_cast<CallbackCtx *>(userData);
+      if (!callbackCtx || !callbackCtx->callback || !(*callbackCtx->callback))
+        return;
+      (*callbackCtx->callback)(token, tokenId);
+    };
+
+    std::string responseBuffer;
+    responseBuffer.resize(responseBufferSize);
+
+    int result =
+        cactus_transcribe(this->_model, audioFilePath.c_str(), prompt.c_str(),
+                          responseBuffer.data(), responseBufferSize,
+                          optionsJson ? optionsJson->c_str() : nullptr,
+                          cactusTokenCallback, &callbackCtx);
+
+    if (result < 0) {
+      throw std::runtime_error("Cactus transcription failed");
+    }
+
+    // Remove null terminator
+    responseBuffer.resize(strlen(responseBuffer.c_str()));
+
+    return responseBuffer;
+  });
+}
+
 std::shared_ptr<Promise<std::vector<double>>>
 HybridCactus::embed(const std::string &text, double embeddingBufferSize) {
   return Promise<std::vector<double>>::async(
@@ -103,6 +150,64 @@ HybridCactus::embed(const std::string &text, double embeddingBufferSize) {
       });
 }
 
+std::shared_ptr<Promise<std::vector<double>>>
+HybridCactus::imageEmbed(const std::string &imagePath,
+                         double embeddingBufferSize) {
+  return Promise<std::vector<double>>::async(
+      [this, imagePath, embeddingBufferSize]() -> std::vector<double> {
+        std::lock_guard<std::mutex> lock(this->_modelMutex);
+
+        if (!this->_model) {
+          throw std::runtime_error("Cactus model is not initialized");
+        }
+
+        std::vector<float> embeddingBuffer(embeddingBufferSize);
+        size_t embeddingDim;
+
+        int result = cactus_image_embed(
+            this->_model, imagePath.c_str(), embeddingBuffer.data(),
+            embeddingBufferSize * sizeof(float), &embeddingDim);
+
+        if (result < 0) {
+          throw std::runtime_error("Cactus image embedding failed");
+        }
+
+        embeddingBuffer.resize(embeddingDim);
+
+        return std::vector<double>(embeddingBuffer.begin(),
+                                   embeddingBuffer.end());
+      });
+}
+
+std::shared_ptr<Promise<std::vector<double>>>
+HybridCactus::audioEmbed(const std::string &audioPath,
+                         double embeddingBufferSize) {
+  return Promise<std::vector<double>>::async(
+      [this, audioPath, embeddingBufferSize]() -> std::vector<double> {
+        std::lock_guard<std::mutex> lock(this->_modelMutex);
+
+        if (!this->_model) {
+          throw std::runtime_error("Cactus model is not initialized");
+        }
+
+        std::vector<float> embeddingBuffer(embeddingBufferSize);
+        size_t embeddingDim;
+
+        int result = cactus_audio_embed(
+            this->_model, audioPath.c_str(), embeddingBuffer.data(),
+            embeddingBufferSize * sizeof(float), &embeddingDim);
+
+        if (result < 0) {
+          throw std::runtime_error("Cactus audio embedding failed");
+        }
+
+        embeddingBuffer.resize(embeddingDim);
+
+        return std::vector<double>(embeddingBuffer.begin(),
+                                   embeddingBuffer.end());
+      });
+}
+
 std::shared_ptr<Promise<void>> HybridCactus::reset() {
   return Promise<void>::async([this]() -> void {
     std::lock_guard<std::mutex> lock(this->_modelMutex);

cpp/HybridCactus.hpp

@@ -23,9 +23,22 @@ class HybridCactus : public HybridCactusSpec {
                                              double /* tokenId */)>> &callback)
       override;
 
+  std::shared_ptr<Promise<std::string>> transcribe(
+      const std::string &audioFilePath, const std::string &prompt,
+      double responseBufferSize, const std::optional<std::string> &optionsJson,
+      const std::optional<std::function<void(const std::string & /* token */,
+                                             double /* tokenId */)>> &callback)
+      override;
+
   std::shared_ptr<Promise<std::vector<double>>>
   embed(const std::string &text, double embeddingBufferSize) override;
 
+  std::shared_ptr<Promise<std::vector<double>>>
+  imageEmbed(const std::string &imagePath, double embeddingBufferSize) override;
+
+  std::shared_ptr<Promise<std::vector<double>>>
+  audioEmbed(const std::string &audioPath, double embeddingBufferSize) override;
+
   std::shared_ptr<Promise<void>> reset() override;
 
   std::shared_ptr<Promise<void>> stop() override;

cpp/cactus_ffi.h

@@ -33,6 +33,17 @@ CACTUS_FFI_EXPORT int cactus_complete(
     void* user_data
 );
 
+CACTUS_FFI_EXPORT int cactus_transcribe(
+    cactus_model_t model,
+    const char* audio_file_path,
+    const char* prompt,
+    char* response_buffer,
+    size_t buffer_size,
+    const char* options_json,
+    cactus_token_callback callback,
+    void* user_data
+);
+
 
 CACTUS_FFI_EXPORT int cactus_embed(
     cactus_model_t model,
@@ -42,6 +53,22 @@ CACTUS_FFI_EXPORT int cactus_embed(
     size_t* embedding_dim
 );
 
+CACTUS_FFI_EXPORT int cactus_image_embed(
+    cactus_model_t model,
+    const char* image_path,
+    float* embeddings_buffer,
+    size_t buffer_size,
+    size_t* embedding_dim
+);
+
+CACTUS_FFI_EXPORT int cactus_audio_embed(
+    cactus_model_t model,
+    const char* audio_path,
+    float* embeddings_buffer,
+    size_t buffer_size,
+    size_t* embedding_dim
+);
+
 CACTUS_FFI_EXPORT void cactus_reset(cactus_model_t model);
 
 CACTUS_FFI_EXPORT void cactus_stop(cactus_model_t model);

example/ios/Podfile.lock

@@ -1,6 +1,6 @@
 PODS:
   - boost (1.84.0)
-  - Cactus (1.0.2):
+  - Cactus (1.1.0):
     - boost
     - DoubleConversion
     - fast_float
@@ -1808,6 +1808,34 @@ PODS:
     - React-RCTFBReactNativeSpec
     - ReactCommon/turbomodule/core
     - SocketRocket
+  - react-native-document-picker (11.0.0):
+    - boost
+    - DoubleConversion
+    - fast_float
+    - fmt
+    - glog
+    - hermes-engine
+    - RCT-Folly
+    - RCT-Folly/Fabric
+    - RCTRequired
+    - RCTTypeSafety
+    - React-Core
+    - React-debug
+    - React-Fabric
+    - React-featureflags
+    - React-graphics
+    - React-ImageManager
+    - React-jsi
+    - React-NativeModulesApple
+    - React-RCTFabric
+    - React-renderercss
+    - React-rendererdebug
+    - React-utils
+    - ReactCodegen
+    - ReactCommon/turbomodule/bridging
+    - ReactCommon/turbomodule/core
+    - SocketRocket
+    - Yoga
   - react-native-image-picker (8.2.1):
     - boost
     - DoubleConversion
@@ -2417,6 +2445,7 @@ DEPENDENCIES:
   - React-logger (from `../node_modules/react-native/ReactCommon/logger`)
   - React-Mapbuffer (from `../node_modules/react-native/ReactCommon`)
   - React-microtasksnativemodule (from `../node_modules/react-native/ReactCommon/react/nativemodule/microtasks`)
+  - "react-native-document-picker (from `../node_modules/@react-native-documents/picker`)"
   - react-native-image-picker (from `../node_modules/react-native-image-picker`)
   - React-NativeModulesApple (from `../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`)
   - React-oscompat (from `../node_modules/react-native/ReactCommon/oscompat`)
@@ -2543,6 +2572,8 @@ EXTERNAL SOURCES:
     :path: "../node_modules/react-native/ReactCommon"
   React-microtasksnativemodule:
     :path: "../node_modules/react-native/ReactCommon/react/nativemodule/microtasks"
+  react-native-document-picker:
+    :path: "../node_modules/@react-native-documents/picker"
   react-native-image-picker:
     :path: "../node_modules/react-native-image-picker"
   React-NativeModulesApple:
@@ -2612,7 +2643,7 @@ EXTERNAL SOURCES:
 
 SPEC CHECKSUMS:
   boost: 7e761d76ca2ce687f7cc98e698152abd03a18f90
-  Cactus: 45d4b8148a963a719617a71b06add8bb2ef8721c
+  Cactus: 2949301f1229677c0bbeba6856d3c78e5798aace
   DoubleConversion: cb417026b2400c8f53ae97020b2be961b59470cb
   fast_float: b32c788ed9c6a8c584d114d0047beda9664e7cc6
   FBLazyVector: b8f1312d48447cca7b4abc21ed155db14742bd03
@@ -2653,6 +2684,7 @@ SPEC CHECKSUMS:
   React-logger: d27dd2000f520bf891d24f6e141cde34df41f0ee
   React-Mapbuffer: 0746ffab5ac0f49b7c9347338e3d0c1d9dd634c8
   React-microtasksnativemodule: b0fb3f97372df39bda3e657536039f1af227cc29
+  react-native-document-picker: 63639c144fbdc4bf7b12d31a3827ae2bfbaf7ad4
   react-native-image-picker: 43e6cd4231e670030fe09b079d696fa5a634ccfc
   React-NativeModulesApple: 9ec9240159974c94886ebbe4caec18e3395f6aef
   React-oscompat: b12c633e9c00f1f99467b1e0e0b8038895dae436

example/package.json

@@ -11,6 +11,7 @@
   },
   "dependencies": {
     "@dr.pogodin/react-native-fs": "^2.36.1",
+    "@react-native-documents/picker": "^11.0.0",
     "react": "19.1.0",
     "react-native": "0.81.1",
     "react-native-image-picker": "^8.2.1",

example/src/App.tsx

@@ -11,7 +11,7 @@ import CompletionScreen from './CompletionScreen';
 import VisionScreen from './VisionScreen';
 import ToolCallingScreen from './ToolCallingScreen';
 import RAGScreen from './RAGScreen';
-import EmbeddingScreen from './EmbeddingScreen';
+import STTScreen from './STTScreen';
 import ChatScreen from './ChatScreen';
 import PerformanceScreen from './PerformanceScreen';
 
@@ -21,7 +21,7 @@ type Screen =
   | 'Vision'
   | 'ToolCalling'
   | 'RAG'
-  | 'Embedding'
+  | 'STT'
   | 'Chat'
   | 'Performance';
 
@@ -48,8 +48,8 @@ const App = () => {
     setSelectedScreen('RAG');
   };
 
-  const handleGoToEmbedding = () => {
-    setSelectedScreen('Embedding');
+  const handleGoToSTT = () => {
+    setSelectedScreen('STT');
   };
 
   const handleGoToChat = () => {
@@ -70,8 +70,8 @@ const App = () => {
         return <ToolCallingScreen />;
       case 'RAG':
         return <RAGScreen />;
-      case 'Embedding':
-        return <EmbeddingScreen />;
+      case 'STT':
+        return <STTScreen />;
       case 'Chat':
         return <ChatScreen />;
       case 'Performance':
@@ -106,7 +106,7 @@ const App = () => {
           >
             <Text style={styles.menuButtonTitle}>Completion</Text>
             <Text style={styles.menuButtonDescription}>
-              Generate text with streaming
+              Text generation and embeddings
             </Text>
           </TouchableOpacity>
 
@@ -115,7 +115,9 @@ const App = () => {
             onPress={handleGoToVision}
           >
             <Text style={styles.menuButtonTitle}>Vision</Text>
-            <Text style={styles.menuButtonDescription}>Analyze images</Text>
+            <Text style={styles.menuButtonDescription}>
+              Image analysis and embeddings
+            </Text>
           </TouchableOpacity>
 
           <TouchableOpacity
@@ -133,12 +135,11 @@ const App = () => {
             </Text>
           </TouchableOpacity>
 
-          <TouchableOpacity
-            style={styles.menuButton}
-            onPress={handleGoToEmbedding}
-          >
-            <Text style={styles.menuButtonTitle}>Embedding</Text>
-            <Text style={styles.menuButtonDescription}>Text to vectors</Text>
+          <TouchableOpacity style={styles.menuButton} onPress={handleGoToSTT}>
+            <Text style={styles.menuButtonTitle}>Speech-to-Text</Text>
+            <Text style={styles.menuButtonDescription}>
+              Audio transcription and embeddings
+            </Text>
           </TouchableOpacity>
 
           <TouchableOpacity style={styles.menuButton} onPress={handleGoToChat}>