[no merge] Testing put get bytes

examples/serial/cpp/CMakeLists.txt

@@ -75,6 +75,7 @@ list(APPEND EXECUTABLES
     smartredis_dataset
     smartredis_model
     smartredis_mnist
+    smartredis_put_get_bytes
 )
 
 # Build the examples

examples/serial/cpp/smartredis_put_get_bytes.cpp

@@ -0,0 +1,62 @@
+/*
+ * BSD 2-Clause License
+ *
+ * Copyright (c) 2021-2024, Hewlett Packard Enterprise
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright notice, this
+ *    list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright notice,
+ *    this list of conditions and the following disclaimer in the documentation
+ *    and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "client.h"
+#include <vector>
+#include <string>
+
+int main(int argc, char* argv[]) {
+
+  // Initialize some byte data
+  size_t n_bytes = 255;
+  std::vector<char> input_bytes(n_bytes);
+  for(size_t i = 0; i < n_bytes; i++) {
+    input_bytes[i] = i;
+  }
+
+  SmartRedis::Client client("client_test_put_get_bytes");
+
+  std::string key = "put_get_bytes_test";
+
+  client.put_bytes(key, input_bytes.data(), n_bytes);
+
+  std::vector<char> output_bytes(n_bytes, 0);
+
+  client.unpack_bytes(key, output_bytes.data(), n_bytes);
+
+  for(int i = 0; i < n_bytes; i++) {
+    if (output_bytes[i] != input_bytes[i]) {
+      std::cout<<"Byte "<<i<<" does not match."<<std::endl;
+      throw std::exception();
+    }
+  }
+
+  std::cout<<"Put bytes test complete"<<std::endl;
+
+  return 0;
+}

include/client.h

@@ -224,6 +224,34 @@ class Client : public SRObject
                         const SRMemoryLayout mem_layout);
 
         /*!
+        *   \brief Puts a raw byte string into the database
+        *   \details The provided byte string is placed into the database
+        *            without any type associated with the bytes.
+        *   \param name The name for referencing the bytes in the database
+        *   \param data The data
+        *   \param n_bytes The number of bytes in the provided data
+        *   \throw SmartRedis::Exception if put bytes command fails
+        */
+        void put_bytes(const std::string& name,
+                       const void* bytes,
+                       const size_t n_bytes);
+
+        /*!
+        *   \brief Retrieve bytes from the database and place into memory
+        *          provided by the caller
+        *   \details The key used to locate the stored bytes
+        *            may be formed by applying a prefix to the supplied
+        *            name. See set_data_source()
+        *            and use_tensor_ensemble_prefix() for more details.
+        *   \param name The name for referencing the bytes in the database
+        *   \param data A buffer into which to place byte data
+        *   \param n_bytes The number of bytes in the provided data
+        *   \throw SmartRedis::Exception if unpack bytes command fails
+        */
+        void unpack_bytes(const std::string& name,
+                          void* data,
+                          const size_t n_bytes);
+        /*!
         *   \brief Retrieve the tensor data, dimensions, and type for the
         *          provided tensor key. This function will allocate and retain
         *          management of the memory for the tensor data.
@@ -1502,6 +1530,19 @@ class Client : public SRObject
         */
         TensorBase* _get_tensorbase_obj(const std::string& name);
 
+        /*!
+        *   \brief This function retrieves bytes stored in the database
+        *          and attaches them to the provided void* pointer.
+        *          The provided n_bytes is modified (by reference) to 
+        *          inform the caller the number of bytes associated 
+        *          with the pointer.  The caller is responsible
+        *          for freeing the memory with free.
+        *   \param name  The name used to reference the bytes
+        */
+        void _get_bytes_no_mem_handling(const std::string& name,
+                                        void*& data, 
+                                        size_t& n_bytes);
+
         /*!
         *   \brief The name of the hash field used to confirm that the
         *          DataSet placement operation was successfully completed.

include/pyclient.h

@@ -100,6 +100,16 @@ class PyClient : public PySRObject
                         std::string& type,
                         py::array data);
 
+        /*!
+        *   \brief Put bytes into the database
+        *   \param name The name to associate with the bytes
+        *              in the database
+        *   \param data The bytes to store
+        *   \throw RuntimeException for all client errors
+        */
+        void put_bytes(std::string& name,
+                       py::object data);
+
         /*!
         *   \brief  Retrieve a tensor from the database.
         *   \details The memory of the data pointer used
@@ -117,6 +127,13 @@ class PyClient : public PySRObject
         */
         py::array get_tensor(const std::string& name);
 
+        /*!
+        *   \brief  Retrieve bytes from the database.
+        *   \param name The name used to reference the bytes
+        *   \throw RuntimeException for all client errors
+        */
+        py::bytes get_bytes(const std::string& name);
+
         /*!
         *   \brief delete a tensor stored in the database
         *   \param name The name of tensor to delete

include/redis.h

@@ -211,6 +211,26 @@ class Redis : public RedisServer
         */
         virtual CommandReply put_tensor(TensorBase& tensor);
 
+        /*!
+        *   \brief Put bytes on the server
+        *   \param tensor The bytes to put on the server
+        *   \returns The CommandReply from the put bytes
+        *            command execution
+        *   \throw SmartRedis::Exception if bytes storage fails
+        */
+        virtual CommandReply put_bytes(const std::string& key, 
+                                       const void* bytes,
+                                       const size_t n_bytes);
+
+        /*!
+        *   \brief Get bytes from the server
+        *   \param key The name of the bytes to retrieve
+        *   \returns The CommandReply from the get bytes server
+        *            command execution
+        *   \throw SmartRedis::Exception if bytes retrieval fails
+        */
+        virtual CommandReply get_bytes(const std::string& key);
+
         /*!
         *   \brief Get a Tensor from the server
         *   \param key The name of the tensor to retrieve

include/rediscluster.h

@@ -221,6 +221,26 @@ class RedisCluster : public RedisServer
         */
         virtual CommandReply put_tensor(TensorBase& tensor);
 
+        /*!
+        *   \brief Put bytes on the server
+        *   \param tensor The bytes to put on the server
+        *   \returns The CommandReply from the put bytes
+        *            command execution
+        *   \throw SmartRedis::Exception if bytes storage fails
+        */
+        virtual CommandReply put_bytes(const std::string& key, 
+                                       const void* bytes,
+                                       const size_t n_bytes);
+
+        /*!
+        *   \brief Get bytes from the server
+        *   \param key The name of the bytes to retrieve
+        *   \returns The CommandReply from the get bytes server
+        *            command execution
+        *   \throw SmartRedis::Exception if bytes retrieval fails
+        */
+        virtual CommandReply get_bytes(const std::string& key);
+
         /*!
         *   \brief Get a Tensor from the server
         *   \param key The name of the tensor to retrieve

include/redisserver.h

@@ -201,6 +201,26 @@ class RedisServer {
         */
         virtual CommandReply put_tensor(TensorBase& tensor) = 0;
 
+        /*!
+        *   \brief Put bytes on the server
+        *   \param tensor The bytes to put on the server
+        *   \returns The CommandReply from the put bytes
+        *            command execution
+        *   \throw SmartRedis::Exception if bytes storage fails
+        */
+        virtual CommandReply put_bytes(const std::string& key, 
+                                       const void* bytes,
+                                       const size_t n_bytes) = 0;
+
+        /*!
+        *   \brief Get bytes from the server
+        *   \param key The name of the bytes to retrieve
+        *   \returns The CommandReply from the get bytes server
+        *            command execution
+        *   \throw SmartRedis::Exception if bytes retrieval fails
+        */
+        virtual CommandReply get_bytes(const std::string& key) = 0;
+
         /*!
         *   \brief Get a Tensor from the server
         *   \param key The name of the tensor to retrieve

src/cpp/client.cpp

@@ -348,6 +348,39 @@ void Client::put_tensor(const std::string& name,
         throw SRRuntimeException("put_tensor failed");
 }
 
+// Put bytes into the database
+void Client::put_bytes(const std::string& name,
+                       const void* bytes,
+                       const size_t n_bytes)
+{
+    // Track calls to this API function
+    LOG_API_FUNCTION();
+
+    std::string key = _build_tensor_key(name, false);
+
+    // Send the tensor
+    CommandReply reply = _redis_server->put_bytes(key, bytes, n_bytes);
+
+    if (reply.has_error())
+        throw SRRuntimeException("put_bytes failed");
+}
+
+// Get byte data and fill an already allocated array memory space that
+// has the specified size. This method is the most memory 
+// efficient way to retrieve tensor data.
+void Client::unpack_bytes(const std::string& name,
+                          void* data,
+                          const size_t n_bytes)
+{
+    // Track calls to this API function
+    LOG_API_FUNCTION();
+
+    std::string get_key = _build_tensor_key(name, true);
+    CommandReply reply = _redis_server->get_bytes(get_key);
+
+    std::memcpy(data, reply.str(), reply.str_len());
+}
+
 // Get the tensor data, dimensions, and type for the provided tensor name.
 // This function will allocate and retain management of the memory for the
 // tensor data.
@@ -2226,6 +2259,25 @@ TensorBase* Client::_get_tensorbase_obj(const std::string& name)
     return ptr;
 }
 
+// Get the bytes stored in the database and allocate memory via malloc 
+// to hold it (deep copy)
+void Client::_get_bytes_no_mem_handling(const std::string& name,
+                                        void*& data, 
+                                        size_t& n_bytes)
+{
+    std::string get_key = _build_tensor_key(name, true);
+    CommandReply reply = _redis_server->get_bytes(get_key);
+
+    // TODO We don't have a way with CommandReply to transfer ownership of a str() reply
+    // to an outside pointer.  For now we do a naive memcopy, 
+    // but we really shouldn't have to do that
+    // We could set the internal str reply value to NULL and the redisreply destructor
+    // which calls free on it will likely do a no-op, but this isn't proven.
+    n_bytes = reply.str_len();
+    data = malloc(n_bytes);
+    std::memcpy(data, (void*)(reply.str()), n_bytes);
+}
+
 // Determine datset name from aggregation list entry
 std::string Client::_get_dataset_name_from_list_entry(
     const std::string& dataset_key)

src/cpp/redis.cpp

@@ -185,6 +185,31 @@ CommandReply Redis::put_tensor(TensorBase& tensor)
     return run(cmd);
 }
 
+// Put bytes on the server
+CommandReply Redis::put_bytes(const std::string& key,
+                              const void* bytes,
+                              const size_t n_bytes)
+{
+    // Build the command
+    SingleKeyCommand cmd;
+    cmd << "SET" << Keyfield(key) 
+        << std::string_view((char*)bytes, n_bytes);
+
+    // Run it
+    return run(cmd);
+}
+
+// Get bytes from the server
+CommandReply Redis::get_bytes(const std::string& key)
+{
+    // Build the command
+    SingleKeyCommand cmd;
+    cmd << "GET" << Keyfield(key);
+
+    // Run it
+    return run(cmd);
+}
+
 // Get a Tensor from the server
 CommandReply Redis::get_tensor(const std::string& key)
 {

src/cpp/rediscluster.cpp

@@ -377,6 +377,31 @@ CommandReply RedisCluster::put_tensor(TensorBase& tensor)
     return run(cmd);
 }
 
+// Put bytes on the server
+CommandReply RedisCluster::put_bytes(const std::string& key,
+                                     const void* bytes,
+                                     const size_t n_bytes)
+{
+    // Build the command
+    SingleKeyCommand cmd;
+    cmd << "SET" << Keyfield(key) 
+        << std::string_view((char*)bytes, n_bytes);
+
+    // Run it
+    return run(cmd);
+}
+
+// Get bytes from the server
+CommandReply RedisCluster::get_bytes(const std::string& key)
+{
+    // Build the command
+    SingleKeyCommand cmd;
+    cmd << "GET" << Keyfield(key);
+
+    // Run it
+    return run(cmd);
+}
+
 // Get a Tensor from the server
 CommandReply RedisCluster::get_tensor(const std::string& key)
 {

src/python/bindings/bind.cpp

@@ -62,6 +62,8 @@ PYBIND11_MODULE(smartredisPy, m) {
         .def(py::init<const std::string&>())
         .def(py::init<PyConfigOptions&, const std::string&>())
         .CLIENT_METHOD(put_tensor)
+        .CLIENT_METHOD(put_bytes)
+        .CLIENT_METHOD(get_bytes)
         .CLIENT_METHOD(get_tensor)
         .CLIENT_METHOD(delete_tensor)
         .CLIENT_METHOD(copy_tensor)