From f2bb5fa73b612e80cc899765fad5bd30ad98bc20 Mon Sep 17 00:00:00 2001
From: Harsha Vardhan Simhadri <harshasi@microsoft.com>
Date: Sat, 10 Sep 2022 21:10:27 -0400
Subject: [PATCH 1/2] added code to count balance of hash bins in build_lsh

---
 include/lsh.h       |  6 ++++++
 src/lsh.cpp         | 28 ++++++++++++++++++++++++++--
 tests/build_lsh.cpp |  1 +
 3 files changed, 33 insertions(+), 2 deletions(-)

diff --git a/include/lsh.h b/include/lsh.h
index ac69c27..c9fddd8 100644
--- a/include/lsh.h
+++ b/include/lsh.h
@@ -30,12 +30,17 @@ namespace grann {
     void write_to_file(std::ofstream &out);
     void read_from_file(std::ifstream &in);
 
+    void print_balance();
+
    protected:
     _u32 vector_dim;  // dimension of points stored/each hp vector
     _u32 table_size;  // number of hyperplanes
     // float **random_hps;
     std::vector<std::vector<float>>     random_hps;
     std::map<size_t, std::vector<_u32>> hashed_vectors;
+
+    std::vector<size_t> count_plus, count_minus;
+
   };
 
   template<typename T>
@@ -53,6 +58,7 @@ namespace grann {
     _u32 search(const T *query, _u32 res_count, const Parameters &search_params,
                 _u32 *indices, float *distances, QueryStats *stats = nullptr,
 								std::vector<label> search_filters = std::vector<label>());
+    void print_balance();
 
    protected:
     _u32                   num_tables;
diff --git a/src/lsh.cpp b/src/lsh.cpp
index 44e1bf8..4a47cd9 100644
--- a/src/lsh.cpp
+++ b/src/lsh.cpp
@@ -19,6 +19,11 @@ namespace grann {
 
     vector_dim = vector_d;
     table_size = table_s;
+
+    for (_u32 i=0; i<table_s; ++i) {
+      count_minus.push_back(0);
+      count_plus.push_back(0);
+    }
   }
 
   HashTable::~HashTable() {
@@ -55,14 +60,24 @@ namespace grann {
         dot_p += x;
       }
 
-      if (dot_p > 0)
+      if (dot_p > 0) {
         input_bits[i] = 1;
-      else
+        count_plus[i]++; 
+      }
+      else {
         input_bits[i] = 0;
+        count_minus[i]++;
+      }
     }
     return input_bits;
   }
 
+  void HashTable::print_balance() {
+    for (_u32 i=0; i < table_size; ++i) {
+      std::cout << count_plus[i] << "|" << count_minus[i] << "   ";
+    }
+  }
+
   void HashTable::add_vector(bitstring vector_hash, _u32 vector_id) {
     hashed_vectors[(size_t) vector_hash.to_ulong()].push_back(vector_id);
   }
@@ -146,6 +161,15 @@ namespace grann {
       }
     }
   }
+  
+  template<typename T>
+  void LSHIndex<T>::print_balance() {
+    for (auto &table : tables) {
+      std::cout << std::endl;    
+      table.print_balance();
+    }
+    std::cout << std::endl;
+  }
 
   template<typename T>
   _u32 LSHIndex<T>::search(const T *query, _u32 res_count,
diff --git a/tests/build_lsh.cpp b/tests/build_lsh.cpp
index af8be49..5250c37 100644
--- a/tests/build_lsh.cpp
+++ b/tests/build_lsh.cpp
@@ -14,6 +14,7 @@ int build_lsh_index(const std::string &data_path, const std::string &labels_file
 
   grann::LSHIndex<T> lsh(m, data_path.c_str(), idmap, labels_file);
   lsh.build(params);
+  lsh.print_balance();
   lsh.save(save_path.c_str());
 
   return 0;

From 18f7d3ade0f45706a504c3171daa904539dee509 Mon Sep 17 00:00:00 2001
From: Harsha Vardhan Simhadri <harshasi@microsoft.com>
Date: Sat, 10 Sep 2022 15:43:55 -0400
Subject: [PATCH 2/2] bias compute added

---
 include/lsh.h |  7 ++++--
 src/lsh.cpp   | 63 ++++++++++++++++++++++++++++++++++++++++++---------
 2 files changed, 57 insertions(+), 13 deletions(-)

diff --git a/include/lsh.h b/include/lsh.h
index c9fddd8..6bdb494 100644
--- a/include/lsh.h
+++ b/include/lsh.h
@@ -17,7 +17,8 @@ namespace grann {
     HashTable(_u32 table_size, _u32 vector_dim);
     ~HashTable();
 
-    void generate_hps();
+    template<typename T>
+    void generate_hps(const T* sample_data, const size_t ndata);
 
     std::vector<_u32> get_bucket(bitstring bucket_id);
 
@@ -25,7 +26,8 @@ namespace grann {
     bitstring get_hash(const T *input_vector);
 
     void add_vector(bitstring vector_hash, _u32 vector_id);
-    void add_hp(std::vector<float> hp);
+    void add_hp(std::vector<float>& hp);
+    void set_bias(std::vector<float>& bias);
 
     void write_to_file(std::ofstream &out);
     void read_from_file(std::ifstream &in);
@@ -41,6 +43,7 @@ namespace grann {
 
     std::vector<size_t> count_plus, count_minus;
 
+    std::vector<float>     bias;
   };
 
   template<typename T>
diff --git a/src/lsh.cpp b/src/lsh.cpp
index 4a47cd9..e7ae284 100644
--- a/src/lsh.cpp
+++ b/src/lsh.cpp
@@ -29,7 +29,8 @@ namespace grann {
   HashTable::~HashTable() {
   }
 
-  void HashTable::generate_hps() {
+  template<typename T>
+  void HashTable::generate_hps(const T* sample_data, const size_t nsample) {
     std::random_device              r;
     std::default_random_engine      rng{r()};
     std::normal_distribution<float> gaussian_dist;
@@ -41,6 +42,19 @@ namespace grann {
         random_hp.push_back(add);
       }
       add_hp(random_hp);
+      
+      std::vector<T> hash_values;
+      hash_values.reserve(nsample);
+#pragma omp parallel for
+      for (size_t i=0; i<nsample; ++i) {
+        float dot_p=0.0;
+        for (size_t j = 0; j < vector_dim; j++) {
+          dot_p += random_hp[j] * sample_data[i*vector_dim + j];
+        }
+        hash_values.push_back(dot_p);
+      }
+      std::sort(hash_values.begin(), hash_values.end());
+      bias.push_back(hash_values[hash_values.size()/2]);
     }
   }
 
@@ -50,26 +64,25 @@ namespace grann {
 
   template<typename T>
   bitstring HashTable::get_hash(const T *input_vector) {
-    bitstring input_bits;
+    bitstring bits;
     for (size_t i = 0; i < table_size; i++) {
       // float dot_p = std::inner_product(random_hps[i].begin(),
       // random_hps[i].end(), input_vector, 0.0);
       float dot_p = 0.0;
       for (size_t j = 0; j < vector_dim; j++) {
-        float x = random_hps[i][j] * input_vector[j];
-        dot_p += x;
+        dot_p += random_hps[i][j] * input_vector[j];
       }
 
-      if (dot_p > 0) {
-        input_bits[i] = 1;
+      if (dot_p > bias[i]) {
+        bits[i] = 1;
         count_plus[i]++; 
       }
       else {
-        input_bits[i] = 0;
+        bits[i] = 0;
         count_minus[i]++;
       }
     }
-    return input_bits;
+    return bits;
   }
 
   void HashTable::print_balance() {
@@ -82,10 +95,14 @@ namespace grann {
     hashed_vectors[(size_t) vector_hash.to_ulong()].push_back(vector_id);
   }
 
-  void HashTable::add_hp(std::vector<float> hp) {
+  void HashTable::add_hp(std::vector<float>& hp) {
     random_hps.push_back(hp);
   }
 
+  void HashTable::set_bias(std::vector<float>& bias) {
+    this->bias = bias;
+  }
+
   void HashTable::write_to_file(std::ofstream &out) {
     // 0. write null terminator to denote start of hashtable
     out.put('@');
@@ -101,7 +118,14 @@ namespace grann {
 
     out.put('%');
 
-    // 2. write map
+    // 2. write hyperplane vectors
+    for (const auto &b : bias) {
+      out.write(reinterpret_cast<const char *>(&b), sizeof(float));
+    }
+
+    out.put('%');
+
+    // 3. write map
     _u32 num_buckets = hashed_vectors.size();
     out.write(reinterpret_cast<const char *>(&num_buckets), sizeof(_u32));
     for (const auto &bucket : hashed_vectors) {
@@ -149,7 +173,7 @@ namespace grann {
     // 1. generate hyperplanes for the table
     for (size_t i = 0; i < num_tables; i++) {
       HashTable table = HashTable(table_size, this->_aligned_dim);
-      table.generate_hps();
+      table.generate_hps(this->_data, this->_num_points);
       tables.push_back(table);
     }
 
@@ -298,6 +322,23 @@ namespace grann {
         exit(1);
       }
 
+      std::vector<float> bias;
+      for (size_t j = 0; j < f_table_size; j++) {
+          float next_bias_element;
+          in.read(reinterpret_cast<char *>(&next_bias_element), sizeof(float));
+          bias.push_back(next_bias_element);
+      }
+      curr_table.set_bias(bias);
+
+      in.get(mid);
+      if ((mid) != '%') {
+        printf("%d\n", mid);
+        perror(
+            "Mistake in file formation, missing null terminator at middle. "
+            "Exiting...");
+        exit(1);
+      }
+
       // 3. get map
       _u32 f_num_buckets;
       in.read(reinterpret_cast<char *>(&f_num_buckets), sizeof(_u32));