Add FULL_SCAN selection mode to least request LB (#31507)

Add FULL_SCAN mode to least request load balancer.

By default, the least request load balancer returns the host with the fewest
active requests from a set of N randomly selected hosts.

This introduces a new "full scan" selection method that returns the host with
the fewest number of active requests from all hosts. If multiple hosts are
tied for "least", one of the tied hosts is randomly chosen.

Added selection_method option to the least request load balancer. If set to 
FULL_SCAN, Envoy will select the host with the fewest active requests from
the entire host set rather than choice_count random choices.

Risk Level: low, existing code path unchanged
Testing: unit tests add
Docs Changes: protobuf docs
Release Notes: added
Signed-off-by: Jared Kirschner <jkirschner@hashicorp.com>
Signed-off-by: Leonardo da Mata <ldamata@spotify.com>
Co-authored-by: Leonardo da Mata <barroca@gmail.com>

Author: jkirschner-hashicorp

api/envoy/extensions/load_balancing_policies/least_request/v3/BUILD

@@ -6,6 +6,7 @@ licenses(["notice"])  # Apache 2
 
 api_proto_package(
     deps = [
+        "//envoy/annotations:pkg",
         "//envoy/config/core/v3:pkg",
         "//envoy/extensions/load_balancing_policies/common/v3:pkg",
         "@com_github_cncf_xds//udpa/annotations:pkg",

api/envoy/extensions/load_balancing_policies/least_request/v3/least_request.proto

@@ -7,6 +7,7 @@ import "envoy/extensions/load_balancing_policies/common/v3/common.proto";
 
 import "google/protobuf/wrappers.proto";
 
+import "envoy/annotations/deprecation.proto";
 import "udpa/annotations/status.proto";
 import "validate/validate.proto";
 
@@ -22,10 +23,34 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // This configuration allows the built-in LEAST_REQUEST LB policy to be configured via the LB policy
 // extension point. See the :ref:`load balancing architecture overview
 // <arch_overview_load_balancing_types>` for more information.
-// [#next-free-field: 6]
+// [#next-free-field: 7]
 message LeastRequest {
+  // Available methods for selecting the host set from which to return the host with the
+  // fewest active requests.
+  enum SelectionMethod {
+    // Return host with fewest requests from a set of ``choice_count`` randomly selected hosts.
+    // Best selection method for most scenarios.
+    N_CHOICES = 0;
+
+    // Return host with fewest requests from all hosts.
+    // Useful in some niche use cases involving low request rates and one of:
+    // (example 1) low request limits on workloads, or (example 2) few hosts.
+    //
+    // Example 1: Consider a workload type that can only accept one connection at a time.
+    // If such workloads are deployed across many hosts, only a small percentage of those
+    // workloads have zero connections at any given time, and the rate of new connections is low,
+    // the ``FULL_SCAN`` method is more likely to select a suitable host than ``N_CHOICES``.
+    //
+    // Example 2: Consider a workload type that is only deployed on 2 hosts. With default settings,
+    // the ``N_CHOICES`` method will return the host with more active requests 25% of the time.
+    // If the request rate is sufficiently low, the behavior of always selecting the host with least
+    // requests as of the last metrics refresh may be preferable.
+    FULL_SCAN = 1;
+  }
+
   // The number of random healthy hosts from which the host with the fewest active requests will
   // be chosen. Defaults to 2 so that we perform two-choice selection if the field is not set.
+  // Only applies to the ``N_CHOICES`` selection method.
   google.protobuf.UInt32Value choice_count = 1 [(validate.rules).uint32 = {gte: 2}];
 
   // The following formula is used to calculate the dynamic weights when hosts have different load
@@ -61,8 +86,12 @@ message LeastRequest {
   common.v3.LocalityLbConfig locality_lb_config = 4;
 
   // [#not-implemented-hide:]
-  // Configuration for performing full scan on the list of hosts.
-  // If this configuration is set, when selecting the host a full scan on the list hosts will be
-  // used to select the one with least requests instead of using random choices.
-  google.protobuf.BoolValue enable_full_scan = 5;
+  // Unused. Replaced by the `selection_method` enum for better extensibility.
+  google.protobuf.BoolValue enable_full_scan = 5
+      [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"];
+
+  // Method for selecting the host set from which to return the host with the fewest active requests.
+  //
+  // Defaults to ``N_CHOICES``.
+  SelectionMethod selection_method = 6 [(validate.rules).enum = {defined_only: true}];
 }

changelogs/current.yaml

@@ -122,5 +122,12 @@ new_features:
     Added :ref:`uri_template<envoy_v3_api_field_config.rbac.v3.Permission.uri_template>` which uses existing
     :ref:`UriTemplateMatchConfig<envoy_v3_api_msg_extensions.path.match.uri_template.v3.UriTemplateMatchConfig>`
     to allow use of glob patterns for URI path matching in RBAC.
+- area: upstream
+  change: |
+    Added :ref:`selection_method <envoy_v3_api_msg_extensions.load_balancing_policies.least_request.v3.LeastRequest>`
+    option to the least request load balancer. If set to ``FULL_SCAN``,
+    Envoy will select the host with the fewest active requests from the entire host set rather than
+    :ref:`choice_count <envoy_v3_api_msg_extensions.load_balancing_policies.least_request.v3.LeastRequest>`
+    random choices.
 
 deprecated:

source/common/upstream/load_balancer_impl.cc

@@ -1298,19 +1298,80 @@ HostConstSharedPtr LeastRequestLoadBalancer::unweightedHostPick(const HostVector
                                                                 const HostsSource&) {
   HostSharedPtr candidate_host = nullptr;
 
+  switch (selection_method_) {
+  case envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest::FULL_SCAN:
+    candidate_host = unweightedHostPickFullScan(hosts_to_use);
+    break;
+  case envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest::N_CHOICES:
+    candidate_host = unweightedHostPickNChoices(hosts_to_use);
+    break;
+  default:
+    IS_ENVOY_BUG("unknown selection method specified for least request load balancer");
+  }
+
+  return candidate_host;
+}
+
+HostSharedPtr LeastRequestLoadBalancer::unweightedHostPickFullScan(const HostVector& hosts_to_use) {
+  HostSharedPtr candidate_host = nullptr;
+
+  size_t num_hosts_known_tied_for_least = 0;
+
+  const size_t num_hosts = hosts_to_use.size();
+
+  for (size_t i = 0; i < num_hosts; ++i) {
+    const HostSharedPtr& sampled_host = hosts_to_use[i];
+
+    if (candidate_host == nullptr) {
+      // Make a first choice to start the comparisons.
+      num_hosts_known_tied_for_least = 1;
+      candidate_host = sampled_host;
+      continue;
+    }
+
+    const auto candidate_active_rq = candidate_host->stats().rq_active_.value();
+    const auto sampled_active_rq = sampled_host->stats().rq_active_.value();
+
+    if (sampled_active_rq < candidate_active_rq) {
+      // Reset the count of known tied hosts.
+      num_hosts_known_tied_for_least = 1;
+      candidate_host = sampled_host;
+    } else if (sampled_active_rq == candidate_active_rq) {
+      ++num_hosts_known_tied_for_least;
+
+      // Use reservoir sampling to select 1 unique sample from the total number of hosts N
+      // that will tie for least requests after processing the full hosts array.
+      //
+      // Upon each new tie encountered, replace candidate_host with sampled_host
+      // with probability (1 / num_hosts_known_tied_for_least percent).
+      // The end result is that each tied host has an equal 1 / N chance of being the
+      // candidate_host returned by this function.
+      const size_t random_tied_host_index = random_.random() % num_hosts_known_tied_for_least;
+      if (random_tied_host_index == 0) {
+        candidate_host = sampled_host;
+      }
+    }
+  }
+
+  return candidate_host;
+}
+
+HostSharedPtr LeastRequestLoadBalancer::unweightedHostPickNChoices(const HostVector& hosts_to_use) {
+  HostSharedPtr candidate_host = nullptr;
+
   for (uint32_t choice_idx = 0; choice_idx < choice_count_; ++choice_idx) {
     const int rand_idx = random_.random() % hosts_to_use.size();
     const HostSharedPtr& sampled_host = hosts_to_use[rand_idx];
 
     if (candidate_host == nullptr) {
-
       // Make a first choice to start the comparisons.
       candidate_host = sampled_host;
       continue;
     }
 
     const auto candidate_active_rq = candidate_host->stats().rq_active_.value();
     const auto sampled_active_rq = sampled_host->stats().rq_active_.value();
+
     if (sampled_active_rq < candidate_active_rq) {
       candidate_host = sampled_host;
     }

source/common/upstream/load_balancer_impl.h

@@ -710,7 +710,8 @@ class LeastRequestLoadBalancer : public EdfLoadBalancerBase {
             least_request_config.has_active_request_bias()
                 ? absl::optional<Runtime::Double>(
                       {least_request_config.active_request_bias(), runtime})
-                : absl::nullopt) {
+                : absl::nullopt),
+        selection_method_(least_request_config.selection_method()) {
     initialize();
   }
 
@@ -737,6 +738,8 @@ class LeastRequestLoadBalancer : public EdfLoadBalancerBase {
                                         const HostsSource& source) override;
   HostConstSharedPtr unweightedHostPick(const HostVector& hosts_to_use,
                                         const HostsSource& source) override;
+  HostSharedPtr unweightedHostPickFullScan(const HostVector& hosts_to_use);
+  HostSharedPtr unweightedHostPickNChoices(const HostVector& hosts_to_use);
 
   const uint32_t choice_count_;
 
@@ -746,6 +749,8 @@ class LeastRequestLoadBalancer : public EdfLoadBalancerBase {
   double active_request_bias_{};
 
   const absl::optional<Runtime::Double> active_request_bias_runtime_;
+  const envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest::SelectionMethod
+      selection_method_{};
 };
 
 /**

test/common/upstream/BUILD

@@ -268,6 +268,7 @@ envoy_cc_test(
     srcs = ["load_balancer_impl_test.cc"],
     deps = [
         ":utility_lib",
+        "//source/common/common:random_generator_lib",
         "//source/common/network:utility_lib",
         "//source/common/upstream:load_balancer_lib",
         "//source/common/upstream:upstream_includes",

test/common/upstream/load_balancer_impl_test.cc

@@ -11,6 +11,7 @@
 #include "envoy/config/core/v3/base.pb.h"
 #include "envoy/config/core/v3/health_check.pb.h"
 
+#include "source/common/common/random_generator.h"
 #include "source/common/network/utility.h"
 #include "source/common/upstream/load_balancer_impl.h"
 #include "source/common/upstream/upstream_impl.h"
@@ -2880,6 +2881,96 @@ TEST_P(LeastRequestLoadBalancerTest, PNC) {
   EXPECT_EQ(hostSet().healthy_hosts_[3], lb_5.chooseHost(nullptr));
 }
 
+TEST_P(LeastRequestLoadBalancerTest, DefaultSelectionMethod) {
+  envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest lr_lb_config;
+  EXPECT_EQ(lr_lb_config.selection_method(),
+            envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest::N_CHOICES);
+}
+
+TEST_P(LeastRequestLoadBalancerTest, FullScanOneHostWithLeastRequests) {
+  hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80", simTime()),
+                              makeTestHost(info_, "tcp://127.0.0.1:81", simTime()),
+                              makeTestHost(info_, "tcp://127.0.0.1:82", simTime()),
+                              makeTestHost(info_, "tcp://127.0.0.1:83", simTime()),
+                              makeTestHost(info_, "tcp://127.0.0.1:84", simTime())};
+  hostSet().hosts_ = hostSet().healthy_hosts_;
+  hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant.
+
+  hostSet().healthy_hosts_[0]->stats().rq_active_.set(4);
+  hostSet().healthy_hosts_[1]->stats().rq_active_.set(3);
+  hostSet().healthy_hosts_[2]->stats().rq_active_.set(2);
+  hostSet().healthy_hosts_[3]->stats().rq_active_.set(1);
+  hostSet().healthy_hosts_[4]->stats().rq_active_.set(5);
+
+  envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest lr_lb_config;
+
+  // Enable FULL_SCAN on hosts.
+  lr_lb_config.set_selection_method(
+      envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest::FULL_SCAN);
+
+  LeastRequestLoadBalancer lb{priority_set_, nullptr, stats_,       runtime_,
+                              random_,       1,       lr_lb_config, simTime()};
+
+  // With FULL_SCAN we will always choose the host with least number of active requests.
+  EXPECT_EQ(hostSet().healthy_hosts_[3], lb.chooseHost(nullptr));
+}
+
+TEST_P(LeastRequestLoadBalancerTest, FullScanMultipleHostsWithLeastRequests) {
+  hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80", simTime()),
+                              makeTestHost(info_, "tcp://127.0.0.1:81", simTime()),
+                              makeTestHost(info_, "tcp://127.0.0.1:82", simTime()),
+                              makeTestHost(info_, "tcp://127.0.0.1:83", simTime()),
+                              makeTestHost(info_, "tcp://127.0.0.1:84", simTime())};
+  hostSet().hosts_ = hostSet().healthy_hosts_;
+  hostSet().runCallbacks({}, {}); // Trigger callbacks. The added/removed lists are not relevant.
+
+  hostSet().healthy_hosts_[0]->stats().rq_active_.set(3);
+  hostSet().healthy_hosts_[1]->stats().rq_active_.set(3);
+  hostSet().healthy_hosts_[2]->stats().rq_active_.set(1);
+  hostSet().healthy_hosts_[3]->stats().rq_active_.set(1);
+  hostSet().healthy_hosts_[4]->stats().rq_active_.set(1);
+
+  envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest lr_lb_config;
+
+  // Enable FULL_SCAN on hosts.
+  lr_lb_config.set_selection_method(
+      envoy::extensions::load_balancing_policies::least_request::v3::LeastRequest::FULL_SCAN);
+
+  auto random = Random::RandomGeneratorImpl();
+
+  LeastRequestLoadBalancer lb{priority_set_, nullptr, stats_,       runtime_,
+                              random,        1,       lr_lb_config, simTime()};
+
+  // Make 1 million selections. Then, check that the selection probability is
+  // approximately equal among the 3 hosts tied for least requests.
+  // Accept a +/-0.5% deviation from the expected selection probability (33.3..%).
+  size_t num_selections = 1000000;
+  size_t expected_approx_selections_per_tied_host = num_selections / 3;
+  size_t abs_error = 5000;
+
+  size_t host_2_counts = 0;
+  size_t host_3_counts = 0;
+  size_t host_4_counts = 0;
+
+  for (size_t i = 0; i < num_selections; ++i) {
+    auto selected_host = lb.chooseHost(nullptr);
+
+    if (selected_host == hostSet().healthy_hosts_[2]) {
+      ++host_2_counts;
+    } else if (selected_host == hostSet().healthy_hosts_[3]) {
+      ++host_3_counts;
+    } else if (selected_host == hostSet().healthy_hosts_[4]) {
+      ++host_4_counts;
+    } else {
+      FAIL() << "Must only select hosts with least requests";
+    }
+  }
+
+  EXPECT_NEAR(expected_approx_selections_per_tied_host, host_2_counts, abs_error);
+  EXPECT_NEAR(expected_approx_selections_per_tied_host, host_3_counts, abs_error);
+  EXPECT_NEAR(expected_approx_selections_per_tied_host, host_4_counts, abs_error);
+}
+
 TEST_P(LeastRequestLoadBalancerTest, WeightImbalance) {
   hostSet().healthy_hosts_ = {makeTestHost(info_, "tcp://127.0.0.1:80", simTime(), 1),
                               makeTestHost(info_, "tcp://127.0.0.1:81", simTime(), 2)};

tools/spelling/spelling_dictionary.txt

@@ -747,6 +747,7 @@ exe
 execlp
 exprfor
 expectable
+extensibility
 extrahelp
 faceplant
 facto