From 2c94831f05e891b630ad2048067ca018da563eb9 Mon Sep 17 00:00:00 2001
From: Mark <mark@helenandmark.org>
Date: Thu, 7 Apr 2016 22:05:12 +0200
Subject: [PATCH 01/20] Add a filter function to enable bindhost to be device
 name, instead of an IP address. This means that if the IP address is dynamic,
 the correct IP address will still be found The changes are made in config.c,
 so they happen only during config read (hence the configuration would need to
 be re-read on an IP address change).

---
 src/config.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/src/config.c b/src/config.c
index f1705a4..39cb6f4 100644
--- a/src/config.c
+++ b/src/config.c
@@ -22,6 +22,7 @@
  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
+#include <ifaddrs.h>
 
 #include "includes.h"
 #include "mlvpn.h"
@@ -36,6 +37,11 @@ extern struct mlvpn_filters_s mlvpn_filters;
 extern struct tuntap_s tuntap;
 extern struct mlvpn_reorder_buffer *reorder_buffer;
 
+char *ip_from_if(char *ifname);
+// we'll declair this here, so that any device name used instead of an IP
+// address gets translated before we go anywhere else...
+
+
 /* Config file reading / re-read.
  * config_file_fd: fd opened in priv_open_config
  * first_time: set to 0 for re-read, or 1 for initial configuration
@@ -311,6 +317,10 @@ mlvpn_config(int config_file_fd, int first_time)
                         config, lastSection, "remoteport", &dstport, NULL,
                         "No remote port specified.\n", 1);
                 }
+
+                bindaddr=ip_from_if(bindaddr);
+                
+                
                 _conf_set_uint_from_conf(
                     config, lastSection, "bandwidth_upload", &bwlimit, 0,
                     NULL, 0);
@@ -485,3 +495,43 @@ mlvpn_config(int config_file_fd, int first_time)
     log_warnx("config", "parse error");
     return 1;
 }
+
+
+/* This is a filter function, it takes an name, if the name turns out to be an
+ * interface, it translates it to it's IP address,
+ * the resulting filtered name is returned (whether it has matched an interface
+ * or not */
+char *ip_from_if(char *ifname) 
+{
+  
+    struct ifaddrs *ifaddr, *ifa;
+    int s;
+    char host[NI_MAXHOST];
+    
+    if (getifaddrs(&ifaddr) == -1) 
+    {
+      log_warn(NULL, "unable to collect ifaddrs");
+      return ifname;
+    }
+
+
+    for (ifa = ifaddr; ifa != NULL; ifa = ifa->ifa_next) 
+    {
+        if (ifa->ifa_addr == NULL)
+            continue;  
+
+        s=getnameinfo(ifa->ifa_addr,sizeof(struct sockaddr_in),host, NI_MAXHOST, NULL, 0, NI_NUMERICHOST);
+
+        if((strcmp(ifa->ifa_name,ifname)==0)&&(ifa->ifa_addr->sa_family==AF_INET))
+        {
+            if (s == 0)
+            {
+              if (ifname) free(ifname);
+              ifname = strdup(host);
+            }
+        }
+    }
+
+    freeifaddrs(ifaddr);
+    return ifname;
+}

From 42b3f4c262bad62fc82ad413376a59812d37d66f Mon Sep 17 00:00:00 2001
From: Mark Burton <mark@greensocs.com>
Date: Wed, 14 Dec 2016 08:58:23 +0100
Subject: [PATCH 02/20] Change in wrr.c to choose the tunnel more 'optimally'
 (and consider the current weights, so be more dynamic).

---
 src/wrr.c | 37 ++++++++++++++++++++-----------------
 1 file changed, 20 insertions(+), 17 deletions(-)

diff --git a/src/wrr.c b/src/wrr.c
index 8e0cb6e..6b241f4 100644
--- a/src/wrr.c
+++ b/src/wrr.c
@@ -17,9 +17,9 @@ static struct mlvpn_wrr wrr = {
 
 static int wrr_min_index()
 {
-    double min = 100.0;
     int min_index = 0;
     int i;
+    double min = wrr.tunval[0];
 
     for(i = 0; i < wrr.len; i++)
     {
@@ -60,21 +60,24 @@ int mlvpn_rtun_wrr_reset(struct rtunhead *head, int use_fallbacks)
 mlvpn_tunnel_t *
 mlvpn_rtun_wrr_choose()
 {
-    int i;
-    int idx;
-
-    if (wrr.len == 0)
-        return NULL;
-
-    idx = wrr_min_index();
-    if (idx < 0)
-        fatalx("Programming error: wrr_min_index < 0!");
-
-    for(i = 0; i < wrr.len; i++)
-    {
-        if (wrr.tunval[i] > 0)
-            wrr.tunval[i] -= 1;
+  int idx = wrr_min_index();
+  
+  double total=0;
+  for (int i = 0; i< wrr.len; i++) {
+    total+= wrr.tunnel[i]->weight;
+  }
+  
+  if (wrr.tunval[idx]<=0 || wrr.tunval[idx] > 10000) {
+    for (int i = 0; i< wrr.len; i++) {
+      if (wrr.tunnel[i]->weight) {
+        wrr.tunval[i]=total / wrr.tunnel[i]->weight;
+      } else {
+        wrr.tunval[i]=wrr.len; // handle initial setup fairly
+      }      
     }
-    wrr.tunval[idx] = (double) 100.0 / wrr.tunnel[idx]->weight;
-    return wrr.tunnel[idx];
+  } else {
+    wrr.tunval[idx]+=total / wrr.tunnel[idx]->weight;
+  }
+  
+  return wrr.tunnel[idx];
 }

From 8168407ea43400a322e5672252afcff4151e09ee Mon Sep 17 00:00:00 2001
From: Mark Burton <mark@greensocs.com>
Date: Wed, 14 Dec 2016 09:07:13 +0100
Subject: [PATCH 03/20] Use non-blocking flags for the tuntap file descriptor
 (for tuntap_linux only, it may need to be done for other platforms,  but they
 may differ in behaviour)

---
 src/tuntap_linux.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/tuntap_linux.c b/src/tuntap_linux.c
index f46314a..917ed4c 100644
--- a/src/tuntap_linux.c
+++ b/src/tuntap_linux.c
@@ -16,15 +16,16 @@ mlvpn_tuntap_read(struct tuntap_s *tuntap)
 {
     ssize_t ret;
     u_char data[DEFAULT_MTU];
+
     ret = read(tuntap->fd, &data, tuntap->maxmtu);
+      
+    if (ret<0 && (errno==EAGAIN || errno==EWOULDBLOCK)) {
+      return -1;
+    }
+    
     if (ret < 0) {
-        if (errno != EAGAIN && errno != EWOULDBLOCK) {
-            /* read error on tuntap is not recoverable. We must die. */
-            fatal("tuntap", "unrecoverable read error");
-        } else {
-            /* false reading from libev read would block, we can't read */
-            return 0;
-        }
+        /* read error on tuntap is not recoverable. We must die. */
+        fatal("tuntap", "unrecoverable read error");
     } else if (ret == 0) { /* End of file */
         fatalx("tuntap device closed");
     } else if (ret > tuntap->maxmtu)  {
@@ -119,6 +120,9 @@ root_tuntap_open(int tuntapmode, char *devname, int mtu)
             return -1;
         }
 
+        int flags = fcntl(fd, F_GETFL, 0);
+        fcntl(fd, F_SETFL, flags | O_NONBLOCK);
+
         /* set tun MTU */
         if ((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
             warn("socket creation failed");

From add22deebfe3887bd582ad4db36519665a3ed032 Mon Sep 17 00:00:00 2001
From: Mark Burton <mark@greensocs.com>
Date: Wed, 14 Dec 2016 09:15:25 +0100
Subject: [PATCH 04/20] Add the 'weight' of tunnels to the control output so
 that it can be monitored.

---
 src/control.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/control.c b/src/control.c
index 7f53098..d30d3ff 100644
--- a/src/control.c
+++ b/src/control.c
@@ -70,7 +70,8 @@ void mlvpn_control_write_status(struct mlvpn_control *ctrl);
     "   \"loss\": %u,\n" \
     "   \"disconnects\": %u,\n" \
     "   \"last_packet\": %u,\n" \
-    "   \"timeout\": %u\n" \
+    "   \"timeout\": %u,\n" \
+    "   \"weight\": %.3f\n" \
     "}%s\n"
 #define JSON_STATUS_ERROR_UNKNOWN_COMMAND "{\"error\": 'unknown command'}\n"
 
@@ -431,6 +432,7 @@ void mlvpn_control_write_status(struct mlvpn_control *ctrl)
                        t->disconnects,
                        (uint32_t)t->last_activity,
                        (uint32_t)t->timeout,
+                       t->weight,
                        (LIST_NEXT(t, entries) ? "," : "")
                       );
         mlvpn_control_write(ctrl, buf, ret);

From 566b4ce2da873843c9595f9c61beb604a43084f5 Mon Sep 17 00:00:00 2001
From: Mark Burton <mark@greensocs.com>
Date: Wed, 14 Dec 2016 09:21:08 +0100
Subject: [PATCH 05/20] Better calculate 'losses' for tunnels, based on a
 huristic that if a tunnel is advancing the sequence number - it's     (Anyway
 this is an improvement on what was there before)

---
 src/mlvpn.c | 44 +++++++++++++++++++++++++++++---------------
 1 file changed, 29 insertions(+), 15 deletions(-)

diff --git a/src/mlvpn.c b/src/mlvpn.c
index 7a2a332..dfb59cb 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -265,30 +265,44 @@ mlvpn_rtun_reorder_drain(uint32_t reorder)
 static void
 mlvpn_loss_update(mlvpn_tunnel_t *tun, uint64_t seq)
 {
-    if (seq > tun->seq_last + 64) {
-        /* consider a connection reset. */
-        tun->seq_vect = (uint64_t) -1;
-        tun->seq_last = seq;
-    } else if (seq > tun->seq_last) {
-        /* new sequence number -- recent message arrive */
-        tun->seq_vect <<= seq - tun->seq_last;
-        tun->seq_vect |= 1;
-        tun->seq_last = seq;
-    } else if (seq >= tun->seq_last - 63) {
-        tun->seq_vect |= (1 << (tun->seq_last - seq));
+// If a tunnel moves forward, leaving a 'hole' - then we GUESS the hole is for
+// the other tunnel, and if it's not filled in, will be a loss marked for the
+// other tunnel.
+  
+  mlvpn_tunnel_t *t;
+  LIST_FOREACH(t, &rtuns, entries) {
+    
+    if (seq > t->seq_last + 64) {
+      /* consider a connection reset. */
+      t->seq_vect = (uint64_t) -1;
+      t->seq_last = seq;
+    } else if (seq > t->seq_last) {
+      /* new sequence number -- recent message arrive */
+      t->seq_vect <<= seq - t->seq_last;
+      if (t==tun) {
+        t->seq_vect |= ~((long long)-1<<(seq - t->seq_last));
+        // If I move it forward, I claim all the other holes are somebody elses
+        // problem to fill in...., so not my error !
+      } else {
+        t->seq_vect |= 1;
+      }
+      t->seq_last = seq;
+    } else if (seq >= t->seq_last - 63) {
+      t->seq_vect |= (1 << (t->seq_last - seq));
     }
+  } 
 }
 
 int
 mlvpn_loss_ratio(mlvpn_tunnel_t *tun)
 {
     int loss = 0;
-    int i;
+    unsigned int i;
     /* Count zeroes */
     for (i = 0; i < 64; i++) {
-        if (! (1 & (tun->seq_vect >> i))) {
-            loss++;
-        }
+      if ( (1 & (tun->seq_vect >> i)) == 0 ) {
+        loss++;
+      }
     }
     return loss * 100 / 64;
 }

From 3260cde0520e9cdeaefd41121bbbfbca74adc866 Mon Sep 17 00:00:00 2001
From: Mark Burton <mark@greensocs.com>
Date: Wed, 14 Dec 2016 10:48:55 +0100
Subject: [PATCH 06/20] Add an 'average' srtt to ease calculations Add a
 mechanism to set tunnel weights based on the srtt (averaged) Call this
 mechanism if the tunnels dont have their bandwidth set.

---
 src/mlvpn.c | 88 ++++++++++++++++++++++++++++++++++++++---------------
 src/mlvpn.h |  1 +
 2 files changed, 64 insertions(+), 25 deletions(-)

diff --git a/src/mlvpn.c b/src/mlvpn.c
index dfb59cb..f7a781f 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -514,6 +514,12 @@ mlvpn_protocol_read(
                 tun->rttvar = (1 - beta) * tun->rttvar + (beta * fabs(tun->srtt - R));
                 tun->srtt = (1 - alpha) * tun->srtt + (alpha * R);
             }
+            if (tun->srtt_av==0) {
+                tun->srtt_av=tun->srtt;
+            } else {
+                tun->srtt_av=((tun->srtt_av * 999)+tun->srtt) / 1000;
+            }
+            mlvpn_rtun_recalc_weight();
         }
         log_debug("rtt", "%ums srtt %ums loss ratio: %d",
             (unsigned int)R, (unsigned int)tun->srtt, mlvpn_loss_ratio(tun));
@@ -680,6 +686,7 @@ mlvpn_rtun_new(const char *name,
     new->saved_timestamp = -1;
     new->saved_timestamp_received_at = 0;
     new->srtt = 1000;
+    new->srtt_av = 0;
     new->rttvar = 500;
     new->rtt_hit = 0;
     new->seq_last = 0;
@@ -743,40 +750,71 @@ mlvpn_rtun_drop(mlvpn_tunnel_t *t)
     update_process_title();
 }
 
-/* Based on tunnel bandwidth, compute a "weight" value
- * to balance correctly the round robin rtun_choose.
- */
+
 static void
-mlvpn_rtun_recalc_weight()
+mlvpn_rtun_recalc_weight_srtt()
 {
     mlvpn_tunnel_t *t;
-    uint32_t bandwidth_total = 0;
-    int warned = 0;
-    /* If the bandwidth limit is not set on all interfaces, then
-     * it's impossible to balance correctly! */
+    double totalsrtt=0;
+
     LIST_FOREACH(t, &rtuns, entries)
     {
-        if (t->bandwidth == 0)
-            warned++;
-        bandwidth_total += t->bandwidth;
+      totalsrtt+=t->srtt_av;
     }
-    if (warned && bandwidth_total > 0) {
-        log_warnx("config", "you must set the bandwidth on every tunnel");
+    double totalf=0;
+      
+    LIST_FOREACH(t, &rtuns, entries)
+    {
+      if (t->srtt_av > 0)  {
+        totalf += totalsrtt / t->srtt_av;
+      }
     }
-    if (warned == 0)
+    
+    LIST_FOREACH(t, &rtuns, entries)
     {
-        LIST_FOREACH(t, &rtuns, entries)
-        {
-            /* useless, but we want to be sure not to divide by 0 ! */
-            if (t->bandwidth > 0 && bandwidth_total > 0)
-            {
-                t->weight = (((double)t->bandwidth /
-                              (double)bandwidth_total) * 100.0);
-                log_debug("wrr", "%s weight = %f (%u %u)", t->name, t->weight,
-                    t->bandwidth, bandwidth_total);
-            }
-        }
+      double st=t->srtt_av;
+      if (st > 0)  {
+        // should be 1 / (t->srtt / totalsrtt)
+        // e.g. (1 / (srtt / totalsrtt)) * (100 / totalf)
+        t->weight=((totalsrtt * 100) / (st * totalf));
+        if (t->weight < 1) t->weight=1;
+        if (t->weight > 100) t->weight=100;
+        log_debug("wrr", "%s weight = %f%%", t->name, t->weight);
+      } 
+    }
+}
+
+/* Based on tunnel bandwidth, compute a "weight" value
+ * to balance correctly the round robin rtun_choose.
+ */
+static void
+mlvpn_rtun_recalc_weight()
+{
+  mlvpn_tunnel_t *t;
+  int unset=0;
+  uint32_t bandwidth_total = 0;
+  
+  LIST_FOREACH(t, &rtuns, entries)
+  {
+    if (t->bandwidth == 0)
+      unset++;
+    bandwidth_total += t->bandwidth;
+  }
+  if (unset) {
+    return mlvpn_rtun_recalc_weight_srtt();
+  } else {
+    LIST_FOREACH(t, &rtuns, entries)
+    {
+      /* useless, but we want to be sure not to divide by 0 ! */
+      if (t->bandwidth > 0 && bandwidth_total > 0)
+      {
+        t->weight = (((double)t->bandwidth /
+                          (double)bandwidth_total) * 100.0);
+        log_debug("wrr", "%s weight = %f (%u %u)", t->name, t->weight,
+                  t->bandwidth, bandwidth_total);
+      }
     }
+  }   
 }
 
 static int
diff --git a/src/mlvpn.h b/src/mlvpn.h
index 0026dfa..b74f434 100644
--- a/src/mlvpn.h
+++ b/src/mlvpn.h
@@ -155,6 +155,7 @@ typedef struct mlvpn_tunnel_s
     uint64_t seq_vect;
     int rtt_hit;
     double srtt;
+    double srtt_av;
     double rttvar;
     double weight;        /* For weight round robin */
     uint32_t flow_id;

From 7ff5866439db05efc0703b7109c529279a635af9 Mon Sep 17 00:00:00 2001
From: Mark <mark@helenandmark.org>
Date: Fri, 16 Dec 2016 12:45:50 +0100
Subject: [PATCH 07/20] Check for valid timestamp when calculating SRTT (and
 revert back to using DEFAULT_MTU for the tuntap read)

---
 src/mlvpn.c        | 7 ++++++-
 src/tuntap_linux.c | 4 ++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/mlvpn.c b/src/mlvpn.c
index f7a781f..d797a17 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -555,12 +555,17 @@ mlvpn_rtun_send(mlvpn_tunnel_t *tun, circular_buffer_t *pktbuf)
     proto.reorder = pkt->reorder;
 
     /* we have a recent received timestamp */
-    if (now64 - tun->saved_timestamp_received_at < 1000 ) {
+    if (tun->saved_timestamp != -1) {
+      if (now64 - tun->saved_timestamp_received_at < 1000 ) {
         /* send "corrected" timestamp advanced by how long we held it */
         /* Cast to uint16_t there intentional */
         proto.timestamp_reply = tun->saved_timestamp + (now64 - tun->saved_timestamp_received_at);
         tun->saved_timestamp = -1;
         tun->saved_timestamp_received_at = 0;
+      } else {
+        proto.timestamp_reply = -1;
+        log_debug("rtt","(%s) No timestamp added, time too long! (%lu > 1000)",tun->name, tun->saved_timestamp + (now64 - tun->saved_timestamp_received_at ));
+      }
     } else {
         proto.timestamp_reply = -1;
     }
diff --git a/src/tuntap_linux.c b/src/tuntap_linux.c
index 917ed4c..bea2d43 100644
--- a/src/tuntap_linux.c
+++ b/src/tuntap_linux.c
@@ -17,7 +17,7 @@ mlvpn_tuntap_read(struct tuntap_s *tuntap)
     ssize_t ret;
     u_char data[DEFAULT_MTU];
 
-    ret = read(tuntap->fd, &data, tuntap->maxmtu);
+    ret = read(tuntap->fd, &data, DEFAULT_MTU);
       
     if (ret<0 && (errno==EAGAIN || errno==EWOULDBLOCK)) {
       return -1;
@@ -72,7 +72,7 @@ mlvpn_tuntap_alloc(struct tuntap_s *tuntap)
     int fd;
 
     if ((fd = priv_open_tun(tuntap->type,
-           tuntap->devname, tuntap->maxmtu)) <= 0 )
+                            tuntap->devname, tuntap->maxmtu)) <= 0 )
         fatalx("failed to open /dev/net/tun read/write");
     tuntap->fd = fd;
     return fd;

From ec83014e38a42674258f601911f22ffc113d0677 Mon Sep 17 00:00:00 2001
From: Mark <mark at helenandmark dot org>
Date: Wed, 16 May 2018 12:17:55 +0200
Subject: [PATCH 08/20] Mechanism to allocate bandwidth based on a quota
 system, priority, bandwidth and finally srtt.

---
 src/config.c         |  14 ++++-
 src/mlvpn.c          | 139 ++++++++++++++++++++++++++++++++++---------
 src/mlvpn.h          |   8 ++-
 src/tuntap_generic.c |   2 +-
 src/tuntap_linux.c   |   4 +-
 src/wrr.c            |  14 ++++-
 6 files changed, 143 insertions(+), 38 deletions(-)

diff --git a/src/config.c b/src/config.c
index 0745ebe..7dc6122 100644
--- a/src/config.c
+++ b/src/config.c
@@ -279,6 +279,7 @@ mlvpn_config(int config_file_fd, int first_time)
                 char *dstaddr;
                 char *dstport;
                 uint32_t bwlimit = 0;
+                uint32_t quota = 0;
                 uint32_t timeout = 30;
                 uint32_t loss_tolerence;
                 int create_tunnel = 1;
@@ -324,6 +325,9 @@ mlvpn_config(int config_file_fd, int first_time)
                 _conf_set_uint_from_conf(
                     config, lastSection, "bandwidth_upload", &bwlimit, 0,
                     NULL, 0);
+                _conf_set_uint_from_conf(
+                    config, lastSection, "quota", &quota, 0,
+                    NULL, 0);
                 _conf_set_uint_from_conf(
                     config, lastSection, "timeout", &timeout, default_timeout,
                     NULL, 0);
@@ -382,10 +386,16 @@ mlvpn_config(int config_file_fd, int first_time)
                         }
                         if (tmptun->bandwidth != bwlimit)
                         {
-                        log_info("config", "%s bandwidth changed from %d to %d",
+                          log_info("config", "%s bandwidth changed from %d to %d",
                                 tmptun->name, tmptun->bandwidth, bwlimit);
                             tmptun->bandwidth = bwlimit;
                         }
+                        if (tmptun->quota != quota)
+                        {
+                          log_info("config", "%s quota changed from %d to %d",
+                                tmptun->name, tmptun->quota, quota);
+                            tmptun->quota = quota;
+                        }
                         if (tmptun->loss_tolerence != loss_tolerence)
                         {
                             log_info("config", "%s loss tolerence changed from %d%% to %d%%",
@@ -403,7 +413,7 @@ mlvpn_config(int config_file_fd, int first_time)
                     mlvpn_rtun_new(
                         lastSection, bindaddr, bindport, bindfib, dstaddr, dstport,
                         default_server_mode, timeout, fallback_only,
-                        bwlimit, loss_tolerence);
+                        bwlimit, loss_tolerence, quota);
                 }
                 if (bindaddr)
                     free(bindaddr);
diff --git a/src/mlvpn.c b/src/mlvpn.c
index f7a781f..86ba0fd 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -87,6 +87,9 @@ char *process_title = NULL;
 int logdebug = 0;
 
 static uint64_t data_seq = 0;
+ev_tstamp lastsent=0;
+uint64_t bandwidthdata=0;
+double bandwidth=0;
 
 struct mlvpn_status_s mlvpn_status = {
     .start_time = 0,
@@ -280,7 +283,7 @@ mlvpn_loss_update(mlvpn_tunnel_t *tun, uint64_t seq)
       /* new sequence number -- recent message arrive */
       t->seq_vect <<= seq - t->seq_last;
       if (t==tun) {
-        t->seq_vect |= ~((long long)-1<<(seq - t->seq_last));
+        t->seq_vect |= ~((uint64_t)-1<<(seq - t->seq_last));
         // If I move it forward, I claim all the other holes are somebody elses
         // problem to fill in...., so not my error !
       } else {
@@ -304,7 +307,7 @@ mlvpn_loss_ratio(mlvpn_tunnel_t *tun)
         loss++;
       }
     }
-    return loss * 100 / 64;
+    return (loss * 100) / 64;
 }
 
 static int
@@ -377,6 +380,9 @@ mlvpn_rtun_read(EV_P_ ev_io *w, int revents)
 
         tun->recvbytes += len;
         tun->recvpackets += 1;
+        if (tun->quota) {
+          tun->permitted -= len;
+        }
 
         if (! tun->addrinfo)
             fatalx("tun->addrinfo is NULL!");
@@ -462,6 +468,7 @@ mlvpn_protocol_read(
         log_warnx("protocol", "%s invalid packet size: %d", tun->name, rlen);
         goto fail;
     }
+
     proto.seq = be64toh(proto.seq);
     proto.timestamp = be16toh(proto.timestamp);
     proto.timestamp_reply = be16toh(proto.timestamp_reply);
@@ -514,12 +521,6 @@ mlvpn_protocol_read(
                 tun->rttvar = (1 - beta) * tun->rttvar + (beta * fabs(tun->srtt - R));
                 tun->srtt = (1 - alpha) * tun->srtt + (alpha * R);
             }
-            if (tun->srtt_av==0) {
-                tun->srtt_av=tun->srtt;
-            } else {
-                tun->srtt_av=((tun->srtt_av * 999)+tun->srtt) / 1000;
-            }
-            mlvpn_rtun_recalc_weight();
         }
         log_debug("rtt", "%ums srtt %ums loss ratio: %d",
             (unsigned int)R, (unsigned int)tun->srtt, mlvpn_loss_ratio(tun));
@@ -538,8 +539,8 @@ mlvpn_rtun_send(mlvpn_tunnel_t *tun, circular_buffer_t *pktbuf)
     mlvpn_proto_t proto;
     uint64_t now64 = mlvpn_timestamp64(ev_now(EV_DEFAULT_UC));
     memset(&proto, 0, sizeof(proto));
-
     mlvpn_pkt_t *pkt = mlvpn_pktbuffer_read(pktbuf);
+
     pkt->reorder = 1;
     if (pkt->type == MLVPN_PKT_DATA && pkt->reorder) {
         proto.data_seq = data_seq++;
@@ -555,15 +556,21 @@ mlvpn_rtun_send(mlvpn_tunnel_t *tun, circular_buffer_t *pktbuf)
     proto.reorder = pkt->reorder;
 
     /* we have a recent received timestamp */
-    if (now64 - tun->saved_timestamp_received_at < 1000 ) {
+    if (tun->saved_timestamp != -1) {
+      if (now64 - tun->saved_timestamp_received_at < 1000 ) {
         /* send "corrected" timestamp advanced by how long we held it */
         /* Cast to uint16_t there intentional */
         proto.timestamp_reply = tun->saved_timestamp + (now64 - tun->saved_timestamp_received_at);
         tun->saved_timestamp = -1;
         tun->saved_timestamp_received_at = 0;
-    } else {
+      } else {
         proto.timestamp_reply = -1;
+        log_debug("rtt","(%s) No timestamp added, time too long! (%lu > 1000)",tun->name, tun->saved_timestamp + (now64 - tun->saved_timestamp_received_at ));
+      }
+    } else {
+      proto.timestamp_reply = -1;
     }
+
     proto.timestamp = mlvpn_timestamp16(now64);
 #ifdef ENABLE_CRYPTO
     if (mlvpn_options.cleartext_data && pkt->type == MLVPN_PKT_DATA) {
@@ -610,6 +617,10 @@ mlvpn_rtun_send(mlvpn_tunnel_t *tun, circular_buffer_t *pktbuf)
     } else {
         tun->sentpackets++;
         tun->sentbytes += ret;
+        if (tun->quota) {
+          tun->permitted -= ret;
+        }
+
         if (wlen != ret)
         {
             log_warnx("net", "%s write error %d/%u",
@@ -646,7 +657,7 @@ mlvpn_rtun_new(const char *name,
                const char *destaddr, const char *destport,
                int server_mode, uint32_t timeout,
                int fallback_only, uint32_t bandwidth,
-               uint32_t loss_tolerence)
+               uint32_t loss_tolerence, uint32_t quota)
 {
     mlvpn_tunnel_t *new;
 
@@ -681,12 +692,13 @@ mlvpn_rtun_new(const char *name,
     new->sentpackets = 0;
     new->sentbytes = 0;
     new->recvbytes = 0;
+    new->permitted = 0;
+    new->quota = quota;
     new->seq = 0;
     new->expected_receiver_seq = 0;
     new->saved_timestamp = -1;
     new->saved_timestamp_received_at = 0;
     new->srtt = 1000;
-    new->srtt_av = 0;
     new->rttvar = 500;
     new->rtt_hit = 0;
     new->seq_last = 0;
@@ -759,26 +771,25 @@ mlvpn_rtun_recalc_weight_srtt()
 
     LIST_FOREACH(t, &rtuns, entries)
     {
-      totalsrtt+=t->srtt_av;
+      totalsrtt+=t->srtt;
     }
     double totalf=0;
       
     LIST_FOREACH(t, &rtuns, entries)
     {
-      if (t->srtt_av > 0)  {
-        totalf += totalsrtt / t->srtt_av;
+      if (t->srtt > 0)  {
+        totalf += totalsrtt / t->srtt;
       }
     }
-    
     LIST_FOREACH(t, &rtuns, entries)
     {
-      double st=t->srtt_av;
+      double st=t->srtt;
       if (st > 0)  {
         // should be 1 / (t->srtt / totalsrtt)
         // e.g. (1 / (srtt / totalsrtt)) * (100 / totalf)
-        t->weight=((totalsrtt * 100) / (st * totalf));
-        if (t->weight < 1) t->weight=1;
-        if (t->weight > 100) t->weight=100;
+        mlvpn_rtun_set_weight(t, ((totalsrtt * 100) / (st * totalf)));
+        if (t->weight < 1) mlvpn_rtun_set_weight(t,1);
+        if (t->weight > 100) mlvpn_rtun_set_weight(t,100);
         log_debug("wrr", "%s weight = %f%%", t->name, t->weight);
       } 
     }
@@ -788,12 +799,12 @@ mlvpn_rtun_recalc_weight_srtt()
  * to balance correctly the round robin rtun_choose.
  */
 static void
-mlvpn_rtun_recalc_weight()
+mlvpn_rtun_recalc_weight_bw()
 {
   mlvpn_tunnel_t *t;
   int unset=0;
   uint32_t bandwidth_total = 0;
-  
+
   LIST_FOREACH(t, &rtuns, entries)
   {
     if (t->bandwidth == 0)
@@ -808,8 +819,8 @@ mlvpn_rtun_recalc_weight()
       /* useless, but we want to be sure not to divide by 0 ! */
       if (t->bandwidth > 0 && bandwidth_total > 0)
       {
-        t->weight = (((double)t->bandwidth /
-                          (double)bandwidth_total) * 100.0);
+        mlvpn_rtun_set_weight(t, (((double)t->bandwidth /
+                                   (double)bandwidth_total) * 100.0));
         log_debug("wrr", "%s weight = %f (%u %u)", t->name, t->weight,
                   t->bandwidth, bandwidth_total);
       }
@@ -817,6 +828,49 @@ mlvpn_rtun_recalc_weight()
   }   
 }
 
+
+/* Based on tunnel bandwidth, with priority compute a "weight" value
+ * to balance correctly the round robin rtun_choose.
+ */
+static void
+mlvpn_rtun_recalc_weight_prio()
+{
+  if (bandwidth<=0) {
+    mlvpn_rtun_recalc_weight_bw();
+  }
+  mlvpn_tunnel_t *t;
+  double bw;
+  bw=bandwidth*1.5;
+  double bwavailable=0;
+  LIST_FOREACH(t, &rtuns, entries) {
+    if (bw>0) {
+      bwavailable+=t->bandwidth;
+      bw-=t->bandwidth;
+    }
+  }
+  if (bwavailable<=0) {
+    mlvpn_rtun_recalc_weight_bw();
+  }
+  bw=bandwidth*1.5;
+  LIST_FOREACH(t, &rtuns, entries) {
+    if (bw>0) {
+      mlvpn_rtun_set_weight(t, (t->bandwidth*100) / bwavailable);
+      bw-=t->bandwidth;
+    } else {
+      mlvpn_rtun_set_weight(t, 0);
+    }
+  }
+}
+
+
+
+static void
+mlvpn_rtun_recalc_weight()
+{
+  mlvpn_rtun_recalc_weight_prio();
+}
+
+
 static int
 mlvpn_rtun_bind(mlvpn_tunnel_t *t)
 {
@@ -1179,12 +1233,39 @@ mlvpn_rtun_tick_connect(mlvpn_tunnel_t *t)
     }
 }
 
+void mlvpn_calc_bandwidth(uint32_t len)
+{
+  ev_tstamp now=ev_now(EV_A);
+  if (lastsent==0) lastsent=now;
+  ev_tstamp diff=now - lastsent;
+  bandwidthdata+=len;
+  if (diff>3.0) {
+    lastsent=now;
+    bandwidth=((((double)bandwidthdata*8) / diff))/1000; // kbits/sec
+//    printf("%10.1f %lu %5.2f\n",bandwidth, bandwidthdata, diff);
+    bandwidthdata=0;
+
+    // what we can do here is add any bandwidth allocation
+    // The allocation should be per second.
+    // permittedis in bytes.
+    mlvpn_tunnel_t *t;
+    LIST_FOREACH(t, &rtuns, entries) {
+      // permitted is in BYTES per second.
+      if (t->quota) {
+        t->permitted+=((t->quota * diff)*1000.0)/8.0; // listed in kbps
+      }
+    }
+    mlvpn_rtun_recalc_weight();
+  }
+}
+
 mlvpn_tunnel_t *
-mlvpn_rtun_choose()
+mlvpn_rtun_choose(uint32_t len)
 {
-    mlvpn_tunnel_t *tun;
-    tun = mlvpn_rtun_wrr_choose();
-    return tun;
+  mlvpn_calc_bandwidth(len);
+  mlvpn_tunnel_t *tun;
+  tun = mlvpn_rtun_wrr_choose(len);
+  return tun;
 }
 
 static void
diff --git a/src/mlvpn.h b/src/mlvpn.h
index b74f434..e58f1e6 100644
--- a/src/mlvpn.h
+++ b/src/mlvpn.h
@@ -155,7 +155,6 @@ typedef struct mlvpn_tunnel_s
     uint64_t seq_vect;
     int rtt_hit;
     double srtt;
-    double srtt_av;
     double rttvar;
     double weight;        /* For weight round robin */
     uint32_t flow_id;
@@ -163,6 +162,8 @@ typedef struct mlvpn_tunnel_s
     uint64_t recvpackets; /* 64bit packets recv counter */
     uint64_t sentbytes;   /* 64bit bytes sent counter */
     uint64_t recvbytes;   /* 64bit bytes recv counter */
+    int64_t permitted;  /* how many bytes we can send */
+    double quota; /* how many bytes per second we can send */
     uint32_t timeout;     /* configured timeout in seconds */
     uint32_t bandwidth;   /* bandwidth in bytes per second */
     circular_buffer_t *sbuf;    /* send buffer */
@@ -192,14 +193,15 @@ int mlvpn_sock_set_nonblocking(int fd);
 
 int mlvpn_loss_ratio(mlvpn_tunnel_t *tun);
 int mlvpn_rtun_wrr_reset(struct rtunhead *head, int use_fallbacks);
+void mlvpn_rtun_set_weight(mlvpn_tunnel_t *t, double weight);
 mlvpn_tunnel_t *mlvpn_rtun_wrr_choose();
-mlvpn_tunnel_t *mlvpn_rtun_choose();
+mlvpn_tunnel_t *mlvpn_rtun_choose(uint32_t len);
 mlvpn_tunnel_t *mlvpn_rtun_new(const char *name,
     const char *bindaddr, const char *bindport, uint32_t bindfib,
     const char *destaddr, const char *destport,
     int server_mode, uint32_t timeout,
     int fallback_only, uint32_t bandwidth,
-    uint32_t loss_tolerence);
+    uint32_t loss_tolerence, uint32_t quota);
 void mlvpn_rtun_drop(mlvpn_tunnel_t *t);
 void mlvpn_rtun_status_down(mlvpn_tunnel_t *t);
 #ifdef HAVE_FILTERS
diff --git a/src/tuntap_generic.c b/src/tuntap_generic.c
index 1141c35..c2f72ff 100644
--- a/src/tuntap_generic.c
+++ b/src/tuntap_generic.c
@@ -15,7 +15,7 @@ mlvpn_tuntap_generic_read(u_char *data, uint32_t len)
     }
 #endif
     if (!rtun) {
-        rtun = mlvpn_rtun_choose();
+        rtun = mlvpn_rtun_choose(len);
         /* Not connected to anyone. read and discard packet. */
         if (! rtun) 
             return len;
diff --git a/src/tuntap_linux.c b/src/tuntap_linux.c
index 917ed4c..bea2d43 100644
--- a/src/tuntap_linux.c
+++ b/src/tuntap_linux.c
@@ -17,7 +17,7 @@ mlvpn_tuntap_read(struct tuntap_s *tuntap)
     ssize_t ret;
     u_char data[DEFAULT_MTU];
 
-    ret = read(tuntap->fd, &data, tuntap->maxmtu);
+    ret = read(tuntap->fd, &data, DEFAULT_MTU);
       
     if (ret<0 && (errno==EAGAIN || errno==EWOULDBLOCK)) {
       return -1;
@@ -72,7 +72,7 @@ mlvpn_tuntap_alloc(struct tuntap_s *tuntap)
     int fd;
 
     if ((fd = priv_open_tun(tuntap->type,
-           tuntap->devname, tuntap->maxmtu)) <= 0 )
+                            tuntap->devname, tuntap->maxmtu)) <= 0 )
         fatalx("failed to open /dev/net/tun read/write");
     tuntap->fd = fd;
     return fd;
diff --git a/src/wrr.c b/src/wrr.c
index 6b241f4..f5d0113 100644
--- a/src/wrr.c
+++ b/src/wrr.c
@@ -23,7 +23,8 @@ static int wrr_min_index()
 
     for(i = 0; i < wrr.len; i++)
     {
-        if (wrr.tunval[i] < min)
+      if ((wrr.tunval[i] < min) &&
+          (wrr.tunnel[i]->quota==0 || wrr.tunnel[i]->permitted>0))
         {
             min = wrr.tunval[i];
             min_index = i;
@@ -57,9 +58,20 @@ int mlvpn_rtun_wrr_reset(struct rtunhead *head, int use_fallbacks)
     return 0;
 }
 
+void mlvpn_rtun_set_weight(mlvpn_tunnel_t *t, double weight)
+{
+  if (t->weight!=weight) {
+    t->weight=weight;
+    for (int i = 0; i< wrr.len; i++) {
+      wrr.tunval[i] = 0.0;
+    }
+  }
+}
+
 mlvpn_tunnel_t *
 mlvpn_rtun_wrr_choose()
 {
+
   int idx = wrr_min_index();
   
   double total=0;

From c09a98612f1a771ae579ef50bab4d3c23eb8b865 Mon Sep 17 00:00:00 2001
From: Mark <mark at helenandmark dot org>
Date: Wed, 16 May 2018 12:34:13 +0200
Subject: [PATCH 09/20] fix for float/int error

---
 src/mlvpn.c | 2 +-
 src/mlvpn.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mlvpn.c b/src/mlvpn.c
index dcbd8fe..6d8ce90 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -1253,7 +1253,7 @@ void mlvpn_calc_bandwidth(uint32_t len)
     LIST_FOREACH(t, &rtuns, entries) {
       // permitted is in BYTES per second.
       if (t->quota) {
-        t->permitted+=((t->quota * diff)*1000.0)/8.0; // listed in kbps
+        t->permitted+=(((double)t->quota * diff)*1000.0)/8.0; // listed in kbps
       }
     }
     mlvpn_rtun_recalc_weight();
diff --git a/src/mlvpn.h b/src/mlvpn.h
index e58f1e6..d1490a5 100644
--- a/src/mlvpn.h
+++ b/src/mlvpn.h
@@ -163,7 +163,7 @@ typedef struct mlvpn_tunnel_s
     uint64_t sentbytes;   /* 64bit bytes sent counter */
     uint64_t recvbytes;   /* 64bit bytes recv counter */
     int64_t permitted;  /* how many bytes we can send */
-    double quota; /* how many bytes per second we can send */
+    uint32_t quota; /* how many bytes per second we can send */
     uint32_t timeout;     /* configured timeout in seconds */
     uint32_t bandwidth;   /* bandwidth in bytes per second */
     circular_buffer_t *sbuf;    /* send buffer */

From 0953686af5c7973804d35d329526bc167fce809f Mon Sep 17 00:00:00 2001
From: Mark <mark at helenandmark dot org>
Date: Sat, 19 May 2018 11:36:47 +0200
Subject: [PATCH 10/20] added a way to preset the permitted value, and also
 fixed up the weight thing to be better, and make better choices

---
 src/control.c |  3 ++-
 src/mlvpn.c   | 44 ++++++++++++++++++++++++++------------------
 2 files changed, 28 insertions(+), 19 deletions(-)

diff --git a/src/control.c b/src/control.c
index d30d3ff..254e2e2 100644
--- a/src/control.c
+++ b/src/control.c
@@ -428,7 +428,8 @@ void mlvpn_control_write_status(struct mlvpn_control *ctrl)
                        t->recvbytes,
                        0,
                        (uint32_t)t->srtt,
-                       mlvpn_loss_ratio(t),
+                       (uint32_t)t->permitted,
+//                       mlvpn_loss_ratio(t),
                        t->disconnects,
                        (uint32_t)t->last_activity,
                        (uint32_t)t->timeout,
diff --git a/src/mlvpn.c b/src/mlvpn.c
index 6d8ce90..7e2a287 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -90,6 +90,7 @@ static uint64_t data_seq = 0;
 ev_tstamp lastsent=0;
 uint64_t bandwidthdata=0;
 double bandwidth=0;
+uint64_t permitted_preset=0;
 
 struct mlvpn_status_s mlvpn_status = {
     .start_time = 0,
@@ -129,7 +130,7 @@ struct mlvpn_filters_s mlvpn_filters = {
 struct mlvpn_reorder_buffer *reorder_buffer;
 freebuffer_t *freebuf;
 
-static char *optstr = "c:n:u:hvVD:";
+static char *optstr = "c:n:u:hvVD:p:";
 static struct option long_options[] = {
     {"config",        required_argument, 0, 'c' },
     {"debug",         no_argument,       0, 2   },
@@ -141,6 +142,7 @@ static struct option long_options[] = {
     {"quiet",         no_argument,       0, 'q' },
     {"version",       no_argument,       0, 'V' },
     {"yes-run-as-root",no_argument,      0, 3   },
+    {"permitted",     required_argument, 0, 'p' },
     {0,               0,                 0, 0 }
 };
 
@@ -184,6 +186,7 @@ usage(char **argv)
             " -v --verbose          increase verbosity\n"
             " -q --quiet            decrease verbosity\n"
             " -V, --version         output version information and exit\n"
+            " -p, --permitted       Set all tunnels with a quota to this value of permitted bytes\n"
             "\n"
             "For more details see mlvpn(1) and mlvpn.conf(5).\n", argv[0]);
     exit(2);
@@ -693,7 +696,11 @@ mlvpn_rtun_new(const char *name,
     new->sentpackets = 0;
     new->sentbytes = 0;
     new->recvbytes = 0;
-    new->permitted = 0;
+    if (quota) {
+      new->permitted = permitted_preset;
+    } else {
+      new->permitted = 0;
+    }
     new->quota = quota;
     new->seq = 0;
     new->expected_receiver_seq = 0;
@@ -837,29 +844,26 @@ static void
 mlvpn_rtun_recalc_weight_prio()
 {
   if (bandwidth<=0) {
-    mlvpn_rtun_recalc_weight_bw();
+    return mlvpn_rtun_recalc_weight_bw();
   }
   mlvpn_tunnel_t *t;
   double bw;
-  bw=bandwidth*1.5;
-  double bwavailable=0;
-  LIST_FOREACH(t, &rtuns, entries) {
-    if (bw>0) {
-      bwavailable+=t->bandwidth;
-      bw-=t->bandwidth;
-    }
-  }
-  if (bwavailable<=0) {
-    mlvpn_rtun_recalc_weight_bw();
-  }
-  bw=bandwidth*1.5;
+  double bwneeded=bandwidth*1.5;
+  bw=bwneeded;
   LIST_FOREACH(t, &rtuns, entries) {
-    if (bw>0) {
-      mlvpn_rtun_set_weight(t, (t->bandwidth*100) / bwavailable);
+    if (bw>0 && (t->permitted > (t->bandwidth*3)) && (t->status >= MLVPN_AUTHOK)) {
+      if (t->bandwidth > bw) {
+        mlvpn_rtun_set_weight(t, (bw*100) / bwneeded);
+      } else {
+        mlvpn_rtun_set_weight(t, (t->bandwidth*100) / bwneeded);
+      }
       bw-=t->bandwidth;
     } else {
       mlvpn_rtun_set_weight(t, 0);
-    }
+    }    
+  }
+  if (bw==bwneeded) {
+    return mlvpn_rtun_recalc_weight_bw();
   }
 }
 
@@ -1567,6 +1571,10 @@ main(int argc, char **argv)
         case 'q': /* --quiet */
             mlvpn_options.verbose--;
             break;
+        case 'p': /* preset the current 'permitted' values (for all tunnels with
+                   * a quota, which is a pritty rubish plan*/
+            permitted_preset=atoll(optarg);
+            break;
         case 'h': /* --help */
         default:
             usage(argv);

From 1f731548401117e0b75ffcc4ece35ea34ce991bd Mon Sep 17 00:00:00 2001
From: Mark <mark at helenandmark dot org>
Date: Sat, 19 May 2018 11:46:45 +0200
Subject: [PATCH 11/20] fixed bug in chooser

---
 src/mlvpn.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mlvpn.c b/src/mlvpn.c
index 7e2a287..afd8ec2 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -851,7 +851,7 @@ mlvpn_rtun_recalc_weight_prio()
   double bwneeded=bandwidth*1.5;
   bw=bwneeded;
   LIST_FOREACH(t, &rtuns, entries) {
-    if (bw>0 && (t->permitted > (t->bandwidth*3)) && (t->status >= MLVPN_AUTHOK)) {
+    if (bw>0 && (t->quota==0 || t->permitted > (t->bandwidth*3)) && (t->status >= MLVPN_AUTHOK)) {
       if (t->bandwidth > bw) {
         mlvpn_rtun_set_weight(t, (bw*100) / bwneeded);
       } else {

From 71bd0c16192eb163b1a18bfcd3dfbbd43e0c8a45 Mon Sep 17 00:00:00 2001
From: Mark <mark at helenandmark dot org>
Date: Sat, 19 May 2018 21:52:51 +0200
Subject: [PATCH 12/20] share non-quote tunnels

---
 src/mlvpn.c | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/mlvpn.c b/src/mlvpn.c
index afd8ec2..73a5906 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -847,20 +847,25 @@ mlvpn_rtun_recalc_weight_prio()
     return mlvpn_rtun_recalc_weight_bw();
   }
   mlvpn_tunnel_t *t;
-  double bw;
   double bwneeded=bandwidth*1.5;
-  bw=bwneeded;
+  double bw=bwneeded;
   LIST_FOREACH(t, &rtuns, entries) {
+    if ((t->quota == 0) && (t->status >= MLVPN_AUTHOK)) {
+      mlvpn_rtun_set_weight(t, (t->bandwidth*80) / bwneeded);
+      bw-=(t->bandwidth*0.8);
+    } else {
     if (bw>0 && (t->quota==0 || t->permitted > (t->bandwidth*3)) && (t->status >= MLVPN_AUTHOK)) {
-      if (t->bandwidth > bw) {
+      if (t->bandwidth*0.8 > bw) {
         mlvpn_rtun_set_weight(t, (bw*100) / bwneeded);
       } else {
-        mlvpn_rtun_set_weight(t, (t->bandwidth*100) / bwneeded);
+        mlvpn_rtun_set_weight(t, (t->bandwidth*80) / bwneeded);
       }
-      bw-=t->bandwidth;
+      bw-=(t->bandwidth*0.8);
     } else {
       mlvpn_rtun_set_weight(t, 0);
-    }    
+    }
+    }
+    
   }
   if (bw==bwneeded) {
     return mlvpn_rtun_recalc_weight_bw();

From 33baf54e4dab9d2ba61fb981a57142fb12d7746e Mon Sep 17 00:00:00 2001
From: Mark <mark at helenandmark dot org>
Date: Mon, 21 May 2018 10:55:04 +0200
Subject: [PATCH 13/20] start of new reorder mechanism, which should not rely
 on srtt timeouts

---
 src/mlvpn.c   |   3 +-
 src/reorder.c | 159 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 161 insertions(+), 1 deletion(-)

diff --git a/src/mlvpn.c b/src/mlvpn.c
index 73a5906..d65452d 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -232,6 +232,7 @@ static void
 mlvpn_rtun_reorder_drain_timeout(EV_P_ ev_timer *w, int revents)
 {
     log_debug("reorder", "reorder timeout. Packet loss?");
+    printf("Reorder timeout\n");
     mlvpn_rtun_reorder_drain(0);
     if (freebuf->used == 0) {
         ev_timer_stop(EV_A_ w);
@@ -847,7 +848,7 @@ mlvpn_rtun_recalc_weight_prio()
     return mlvpn_rtun_recalc_weight_bw();
   }
   mlvpn_tunnel_t *t;
-  double bwneeded=bandwidth*1.5;
+  double bwneeded=bandwidth*2;
   double bw=bwneeded;
   LIST_FOREACH(t, &rtuns, entries) {
     if ((t->quota == 0) && (t->status >= MLVPN_AUTHOK)) {
diff --git a/src/reorder.c b/src/reorder.c
index adf3164..c515e0a 100644
--- a/src/reorder.c
+++ b/src/reorder.c
@@ -39,6 +39,8 @@
 #include "reorder.h"
 #include "log.h"
 
+#define MARK
+
 /* A generic circular buffer */
 struct cir_buffer {
     unsigned int size;   /**< Number of pkts that can be stored */
@@ -48,6 +50,15 @@ struct cir_buffer {
     mlvpn_pkt_t **pkts;
 };
 
+#ifdef MARK
+struct pktlist 
+{
+  mlvpn_pkt_t *pkt;
+  struct pktlist **prev;
+  struct pktlist *next;
+};
+#endif
+
 /* The reorder buffer data structure itself */
 struct mlvpn_reorder_buffer {
     uint64_t min_seqn;  /**< Lowest seq. number that can be in the buffer */
@@ -55,8 +66,155 @@ struct mlvpn_reorder_buffer {
     struct cir_buffer ready_buf; /**< temp buffer for dequeued pkts */
     struct cir_buffer order_buf; /**< buffer used to reorder pkts */
     int is_initialized;
+  
+#ifdef MARK
+  struct pktlist *pool;
+  struct pktlist *list;
+  struct pktlist **tail;
+  int list_size;
+  int max_size;
+#endif
 };
 
+
+#ifdef MARK
+
+struct mlvpn_reorder_buffer *
+mlvpn_reorder_init(struct mlvpn_reorder_buffer *b, unsigned int bufsize,
+        unsigned int size)
+{
+  b->max_size=size;
+  b->pool=NULL;
+  b->list=NULL;
+  b->tail=&b->list;
+  b->list_size=0;
+
+  return b;
+}
+struct mlvpn_reorder_buffer*
+mlvpn_reorder_create(unsigned int size)
+{
+  struct mlvpn_reorder_buffer *b = malloc(sizeof(struct mlvpn_reorder_buffer));
+  mlvpn_reorder_init(b, 0, size);
+  return b;
+}
+void
+mlvpn_reorder_reset(struct mlvpn_reorder_buffer *b)
+{
+  *b->tail=b->pool;
+  b->pool=b->list;
+  b->list=NULL;
+  b->tail=&b->list;
+  b->list_size=0;
+}
+void mlvpn_reorder_free(struct mlvpn_reorder_buffer *b)
+{
+  struct pktlist *l,*n;
+  for (l=b->list;l;l=n) {n=l->next;free(l);}
+  for (l=b->pool;l;l=n) {n=l->next;free(l);}
+  free(b);
+}
+
+int
+mlvpn_reorder_insert(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t *pkt)
+{
+    struct pktlist *p;
+    if (b->pool) {
+      p=b->pool;
+      b->pool=b->pool->next;
+    } else {
+      p=malloc(sizeof (struct pktlist));
+    }
+    p->pkt=pkt;
+    
+    if (!b->is_initialized) {
+        b->min_seqn = pkt->seq;
+        b->is_initialized = 1;
+        log_debug("reorder", "initial sequence: %"PRIu64"", pkt->seq);
+    }
+
+    
+    /*
+     * calculate the offset from the head pointer we need to go.
+     * The subtraction takes care of the sequence number wrapping.
+     * For example (using 16-bit for brevity):
+     *  min_seqn  = 0xFFFD
+     *  pkt_seq   = 0x0010
+     *  offset    = 0x0010 - 0xFFFD = 0x13
+     * Then we cast to a signed int, if the subtraction ends up in a large
+     * number, that will be seen as negative when casted....
+     */
+    struct pktlist *l;
+    for (l=b->list;l && ((int64_t)(pkt->seq - l->pkt->seq)>0);l=l->next);
+    if (!l) {
+      p->prev=b->tail;
+      p->next=NULL;
+      *b->tail=p;
+      b->tail=&p->next;
+    } else {
+      p->prev=l->prev;
+      *l->prev=p;
+      p->next=l;
+      l->prev=&p->next;
+    }
+    b->list_size++;
+
+    if (b->list && ((int64_t)(b->min_seqn - b->list->pkt->seq) > 0)) {
+      printf("got old (insert) %d\n",(int)(b->min_seqn - b->list->pkt->seq));
+      b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
+    }
+
+    return 0;
+}
+
+      
+unsigned int
+mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
+        unsigned max_pkts)
+{
+
+  unsigned int drain_cnt = 0;
+
+//  offset = pkt->seq - b->min_seqn;
+
+
+  if (b->list && ((int64_t)(b->list->pkt->seq - b->min_seqn) > b->max_size)) {
+    printf("Skipping %d\n",(int)(b->list->pkt->seq - b->min_seqn));
+    b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
+  }
+  
+  if (b->list && ((int64_t)(b->min_seqn - b->list->pkt->seq) > 0)) {
+    printf("got old (drain) %d\n",(int)(b->min_seqn - b->list->pkt->seq));
+    b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
+  }
+  
+  while (b->list && ((int64_t)(b->min_seqn - b->list->pkt->seq)>=0) && (drain_cnt < max_pkts)) {
+    pkts[drain_cnt++]=b->list->pkt;
+    struct pktlist *l=b->list;
+    b->list=l->next;
+    if (b->list) b->list->prev=&b->list;
+    else b->tail=&b->list;
+    l->next=b->pool;
+    b->pool=l;
+    b->list_size--;
+
+    b->min_seqn=l->pkt->seq + 1;
+
+// do we need to do this again here?
+/*    if (b->list && ((int64_t)(b->list->pkt->seq - b->min_seqn) > b->max_size)) {
+      printf("Skipping %d\n",(int)(b->list->pkt->seq - b->min_seqn));
+      b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
+      }*/
+
+  }
+  return drain_cnt;
+}
+  
+#else   
+
+
+// OLD CODE....
+
 struct mlvpn_reorder_buffer *
 mlvpn_reorder_init(struct mlvpn_reorder_buffer *b, unsigned int bufsize,
         unsigned int size)
@@ -256,3 +414,4 @@ mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
     }
     return drain_cnt;
 }
+#endif

From 91b958659c1c4a0b464fe01d9f4a8509ad7f27d6 Mon Sep 17 00:00:00 2001
From: Mark <mark at helenandmark dot org>
Date: Mon, 21 May 2018 11:00:57 +0200
Subject: [PATCH 14/20] bug

---
 src/reorder.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/reorder.c b/src/reorder.c
index c515e0a..f2198aa 100644
--- a/src/reorder.c
+++ b/src/reorder.c
@@ -88,7 +88,8 @@ mlvpn_reorder_init(struct mlvpn_reorder_buffer *b, unsigned int bufsize,
   b->list=NULL;
   b->tail=&b->list;
   b->list_size=0;
-
+  b->is_initialized = 0;
+  
   return b;
 }
 struct mlvpn_reorder_buffer*

From 319b56ce5286cd044146f2f640fd4ee6ebb29900 Mon Sep 17 00:00:00 2001
From: Mark <mark@helenandmark.org>
Date: Mon, 21 May 2018 17:52:51 +0200
Subject: [PATCH 15/20] ibetter reorder

---
 src/privsep.c |  2 +-
 src/reorder.c | 39 +++++++++++++++++++++++++++++++++------
 2 files changed, 34 insertions(+), 7 deletions(-)

diff --git a/src/privsep.c b/src/privsep.c
index 8005909..9907d06 100644
--- a/src/privsep.c
+++ b/src/privsep.c
@@ -159,7 +159,7 @@ priv_init(char *argv[], char *username)
             if (is_root && pw)
             {
                 if (chroot(pw->pw_dir) != 0)
-                    err(1, "unable to chroot");
+                    err(1, "unable to chroot...");
             }
 
             /* May be usefull to chose chdir directory ? */
diff --git a/src/reorder.c b/src/reorder.c
index f2198aa..c47062e 100644
--- a/src/reorder.c
+++ b/src/reorder.c
@@ -51,6 +51,7 @@ struct cir_buffer {
 };
 
 #ifdef MARK
+#include "mlvpn.h"
 struct pktlist 
 {
   mlvpn_pkt_t *pkt;
@@ -134,6 +135,7 @@ mlvpn_reorder_insert(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t *pkt)
         log_debug("reorder", "initial sequence: %"PRIu64"", pkt->seq);
     }
 
+//    printf("Insert %lu\n",pkt->seq);
     
     /*
      * calculate the offset from the head pointer we need to go.
@@ -160,7 +162,7 @@ mlvpn_reorder_insert(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t *pkt)
     }
     b->list_size++;
 
-    if (b->list && ((int64_t)(b->min_seqn - b->list->pkt->seq) > 0)) {
+    if (b->list && ((int64_t)(b->min_seqn - b->list->pkt->seq) > b->max_size)) {
       printf("got old (insert) %d\n",(int)(b->min_seqn - b->list->pkt->seq));
       b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
     }
@@ -178,9 +180,33 @@ mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
 
 //  offset = pkt->seq - b->min_seqn;
 
+//  printf("min_seqn %lu, top %lu\n",b->min_seqn, b->list->pkt->seq);
 
+  uint64_t old=0,new=0;
+  mlvpn_tunnel_t *t;
+  old=rtuns.lh_first->seq;
+  new=rtuns.lh_first->seq;
+  LIST_FOREACH(t, &rtuns, entries) {
+    if ((int64_t)(t->seq - old) > 0) old=t->seq;
+    if ((int64_t)(new - t->seq) > 0) new=t->seq;
+  }
+//  No point being behind the oldest tunnel !
+  if (b->list && ((int64_t)(old - b->min_seqn) > 0)) {
+    printf("Skipping (no old tunnels)  %d\n",(int)(old - b->min_seqn));
+    b->min_seqn=b->list->pkt->seq;
+  }
+  
+
+//    no point being older than the newest thing in - max_size
+  if (b->list && ((int64_t)(new - b->min_seqn) > b->max_size)) {
+    printf("Skipping (newest is too far ahead) %d\n",(int)(old - b->min_seqn));
+    b->min_seqn=b->list->pkt->seq;
+  }
+  
+
+  
   if (b->list && ((int64_t)(b->list->pkt->seq - b->min_seqn) > b->max_size)) {
-    printf("Skipping %d\n",(int)(b->list->pkt->seq - b->min_seqn));
+    printf("Skipping (whole) %d\n",(int)(b->list->pkt->seq - b->min_seqn));
     b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
   }
   
@@ -189,7 +215,7 @@ mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
     b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
   }
   
-  while (b->list && ((int64_t)(b->min_seqn - b->list->pkt->seq)>=0) && (drain_cnt < max_pkts)) {
+  while (b->list && ((b->list_size>(b->max_size*2)) || ((int64_t)(b->min_seqn - b->list->pkt->seq)>=0)) && (drain_cnt < max_pkts)) {
     pkts[drain_cnt++]=b->list->pkt;
     struct pktlist *l=b->list;
     b->list=l->next;
@@ -202,12 +228,13 @@ mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
     b->min_seqn=l->pkt->seq + 1;
 
 // do we need to do this again here?
-/*    if (b->list && ((int64_t)(b->list->pkt->seq - b->min_seqn) > b->max_size)) {
-      printf("Skipping %d\n",(int)(b->list->pkt->seq - b->min_seqn));
+    if (b->list && ((int64_t)(b->list->pkt->seq - b->min_seqn) > b->max_size)) {
+      printf("Skipping (whole - b) %d\n",(int)(b->list->pkt->seq - b->min_seqn));
       b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
-      }*/
+    }
 
   }
+//  printf("Draining %d\n",drain_cnt);
   return drain_cnt;
 }
   

From 453f6a66d5881de9fe79c9426e288fa67a0fe627 Mon Sep 17 00:00:00 2001
From: Mark <mark@helenandmark.org>
Date: Tue, 22 May 2018 11:55:12 +0200
Subject: [PATCH 16/20] improve reorder, andmake it adjust automatically

---
 src/mlvpn.c   |   2 +-
 src/reorder.c | 104 ++++++++++++++++++++++++++++++++------------------
 2 files changed, 67 insertions(+), 39 deletions(-)

diff --git a/src/mlvpn.c b/src/mlvpn.c
index d65452d..e6a065c 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -233,7 +233,7 @@ mlvpn_rtun_reorder_drain_timeout(EV_P_ ev_timer *w, int revents)
 {
     log_debug("reorder", "reorder timeout. Packet loss?");
     printf("Reorder timeout\n");
-    mlvpn_rtun_reorder_drain(0);
+    mlvpn_rtun_reorder_drain(1);
     if (freebuf->used == 0) {
         ev_timer_stop(EV_A_ w);
     }
diff --git a/src/reorder.c b/src/reorder.c
index c47062e..98f473d 100644
--- a/src/reorder.c
+++ b/src/reorder.c
@@ -51,11 +51,11 @@ struct cir_buffer {
 };
 
 #ifdef MARK
-#include "mlvpn.h"
+//#include "mlvpn.h"
 struct pktlist 
 {
   mlvpn_pkt_t *pkt;
-  struct pktlist **prev;
+  struct pktlist *last;
   struct pktlist *next;
 };
 #endif
@@ -71,8 +71,9 @@ struct mlvpn_reorder_buffer {
 #ifdef MARK
   struct pktlist *pool;
   struct pktlist *list;
-  struct pktlist **tail;
+  struct pktlist *tail;
   int list_size;
+  int list_size_av;
   int max_size;
 #endif
 };
@@ -84,11 +85,12 @@ struct mlvpn_reorder_buffer *
 mlvpn_reorder_init(struct mlvpn_reorder_buffer *b, unsigned int bufsize,
         unsigned int size)
 {
-  b->max_size=size;
+  b->max_size=10;
   b->pool=NULL;
   b->list=NULL;
-  b->tail=&b->list;
+  b->tail=NULL;
   b->list_size=0;
+  b->list_size_av=size;
   b->is_initialized = 0;
   
   return b;
@@ -103,10 +105,12 @@ mlvpn_reorder_create(unsigned int size)
 void
 mlvpn_reorder_reset(struct mlvpn_reorder_buffer *b)
 {
-  *b->tail=b->pool;
-  b->pool=b->list;
+  if (b->tail) {
+    b->tail->next=b->pool;
+    b->pool=b->list;
+  }
   b->list=NULL;
-  b->tail=&b->list;
+  b->tail=NULL;
   b->list_size=0;
 }
 void mlvpn_reorder_free(struct mlvpn_reorder_buffer *b)
@@ -135,7 +139,6 @@ mlvpn_reorder_insert(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t *pkt)
         log_debug("reorder", "initial sequence: %"PRIu64"", pkt->seq);
     }
 
-//    printf("Insert %lu\n",pkt->seq);
     
     /*
      * calculate the offset from the head pointer we need to go.
@@ -148,23 +151,35 @@ mlvpn_reorder_insert(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t *pkt)
      * number, that will be seen as negative when casted....
      */
     struct pktlist *l;
-    for (l=b->list;l && ((int64_t)(pkt->seq - l->pkt->seq)>0);l=l->next);
+    for (l=b->list;l && ((int64_t)(pkt->seq - l->pkt->seq)<=0);l=l->next){}
     if (!l) {
-      p->prev=b->tail;
-      p->next=NULL;
-      *b->tail=p;
-      b->tail=&p->next;
+      if (b->tail) {
+        p->last=b->tail;
+        b->tail->next=p;
+        b->tail=p;
+        p->next=NULL;
+      } else {
+        p->last=NULL;
+        p->next=NULL;
+        b->list=p;
+        b->tail=p;
+      }
     } else {
-      p->prev=l->prev;
-      *l->prev=p;
+      p->last=l->last;
+      if (p->last) {
+        p->last->next=p;
+      } else {
+        b->list=p;
+      }
       p->next=l;
-      l->prev=&p->next;
+      l->last=p;
     }
+    
     b->list_size++;
+//    printf("Insert    %lu  list size %d min %lu\n",pkt->seq, b->list_size, b->min_seqn);
 
-    if (b->list && ((int64_t)(b->min_seqn - b->list->pkt->seq) > b->max_size)) {
-      printf("got old (insert) %d\n",(int)(b->min_seqn - b->list->pkt->seq));
-      b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
+    if (b->tail && ((int64_t)(b->min_seqn - b->tail->pkt->seq) > 0)) {
+//      printf("got old (insert) consider increasing buffer by %d\n",(int)(b->min_seqn - b->tail->pkt->seq));
     }
 
     return 0;
@@ -182,7 +197,7 @@ mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
 
 //  printf("min_seqn %lu, top %lu\n",b->min_seqn, b->list->pkt->seq);
 
-  uint64_t old=0,new=0;
+/*  uint64_t old=0,new=0;
   mlvpn_tunnel_t *t;
   old=rtuns.lh_first->seq;
   new=rtuns.lh_first->seq;
@@ -202,39 +217,52 @@ mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
     printf("Skipping (newest is too far ahead) %d\n",(int)(old - b->min_seqn));
     b->min_seqn=b->list->pkt->seq;
   }
-  
+*/
 
-  
+/*  
   if (b->list && ((int64_t)(b->list->pkt->seq - b->min_seqn) > b->max_size)) {
     printf("Skipping (whole) %d\n",(int)(b->list->pkt->seq - b->min_seqn));
     b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
-  }
-  
-  if (b->list && ((int64_t)(b->min_seqn - b->list->pkt->seq) > 0)) {
+  } else
+*/
+/*  if (b->list && ((int64_t)(b->min_seqn - b->list->pkt->seq) > 0)) {
     printf("got old (drain) %d\n",(int)(b->min_seqn - b->list->pkt->seq));
     b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
   }
+*/
   
-  while (b->list && ((b->list_size>(b->max_size*2)) || ((int64_t)(b->min_seqn - b->list->pkt->seq)>=0)) && (drain_cnt < max_pkts)) {
-    pkts[drain_cnt++]=b->list->pkt;
-    struct pktlist *l=b->list;
-    b->list=l->next;
-    if (b->list) b->list->prev=&b->list;
-    else b->tail=&b->list;
+  while (b->tail && ((b->list_size>((b->list_size_av*2))) || ((int64_t)(b->min_seqn - b->tail->pkt->seq)>=0)) && (drain_cnt < max_pkts)) {
+    struct pktlist *l=b->tail;
+    pkts[drain_cnt++]=l->pkt;
+    if (l->last) {
+      b->tail=l->last;
+      b->tail->next=NULL;
+    } else {
+      b->list=NULL;
+      b->tail=NULL;
+    }
     l->next=b->pool;
     b->pool=l;
     b->list_size--;
 
-    b->min_seqn=l->pkt->seq + 1;
+    b->min_seqn=l->pkt->seq+1;
 
 // do we need to do this again here?
-    if (b->list && ((int64_t)(b->list->pkt->seq - b->min_seqn) > b->max_size)) {
-      printf("Skipping (whole - b) %d\n",(int)(b->list->pkt->seq - b->min_seqn));
+/*    if (b->list && ((int64_t)(b->list->pkt->seq - b->min_seqn) > b->max_size)) {
+      printf("Skipping (hole - b) %d\n",(int)(b->list->pkt->seq - b->min_seqn));
       b->min_seqn=b->list->pkt->seq; // we have a whole, skip over !!!
-    }
-
+      }
+*/
+    
   }
-//  printf("Draining %d\n",drain_cnt);
+  if (drain_cnt > 1) {
+    b->list_size_av = ((b->list_size_av*9) + (b->list_size + drain_cnt) + 5)/10;
+//    printf("%d\n",b->list_size_av);
+  }
+  
+//  if (b->tail) b->min_seqn++;
+  
+//  printf("Drain %d %lu  list size %d min %lu\n",drain_cnt, pkts[0]->seq, b->list_size, b->min_seqn);
   return drain_cnt;
 }
   

From 81b130babb2cc27c77ad553956787438664a3abd Mon Sep 17 00:00:00 2001
From: Mark <mark@helenandmark.org>
Date: Tue, 22 May 2018 13:33:01 +0200
Subject: [PATCH 17/20] minor

---
 src/mlvpn.c   |  1 +
 src/reorder.c | 12 +++++++++---
 src/reorder.h |  6 ++++++
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/src/mlvpn.c b/src/mlvpn.c
index e6a065c..c65383b 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -233,6 +233,7 @@ mlvpn_rtun_reorder_drain_timeout(EV_P_ ev_timer *w, int revents)
 {
     log_debug("reorder", "reorder timeout. Packet loss?");
     printf("Reorder timeout\n");
+    mlvpn_reorder_skip(reorder_buffer);
     mlvpn_rtun_reorder_drain(1);
     if (freebuf->used == 0) {
         ev_timer_stop(EV_A_ w);
diff --git a/src/reorder.c b/src/reorder.c
index 98f473d..a22f4f8 100644
--- a/src/reorder.c
+++ b/src/reorder.c
@@ -90,7 +90,7 @@ mlvpn_reorder_init(struct mlvpn_reorder_buffer *b, unsigned int bufsize,
   b->list=NULL;
   b->tail=NULL;
   b->list_size=0;
-  b->list_size_av=size;
+  b->list_size_av=10;
   b->is_initialized = 0;
   
   return b;
@@ -178,13 +178,19 @@ mlvpn_reorder_insert(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t *pkt)
     b->list_size++;
 //    printf("Insert    %lu  list size %d min %lu\n",pkt->seq, b->list_size, b->min_seqn);
 
-    if (b->tail && ((int64_t)(b->min_seqn - b->tail->pkt->seq) > 0)) {
+//    if (b->tail && ((int64_t)(b->min_seqn - b->tail->pkt->seq) > 0)) {
 //      printf("got old (insert) consider increasing buffer by %d\n",(int)(b->min_seqn - b->tail->pkt->seq));
-    }
+//    }
 
     return 0;
 }
 
+void mlvpn_reorder_skip(struct mlvpn_reorder_buffer *b)
+{
+    b->min_seqn=b->tail->pkt->seq; // Jump over any hole !!!!
+}
+
+  
       
 unsigned int
 mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
diff --git a/src/reorder.h b/src/reorder.h
index 8e048fd..c5a5ecf 100644
--- a/src/reorder.h
+++ b/src/reorder.h
@@ -150,4 +150,10 @@ unsigned int
 mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
         unsigned max_pkts);
 
+
+/* skip over any holes in the buffer to try and force more data to be available
+ * to drain */
+void mlvpn_reorder_skip(struct mlvpn_reorder_buffer *b);
+
+
 #endif /* MLVPN_REORDER_H */

From 6d9ab70cd0c19fa0c4ecf8a655caaa73777ebe7a Mon Sep 17 00:00:00 2001
From: Mark <mark@helenandmark.org>
Date: Mon, 28 May 2018 21:37:14 +0200
Subject: [PATCH 18/20] limit the list order length for reordering

---
 src/control.c |  2 +-
 src/mlvpn.c   |  6 +++---
 src/reorder.c | 32 +++++++++++++++++++++++++++-----
 3 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/src/control.c b/src/control.c
index 254e2e2..a368acf 100644
--- a/src/control.c
+++ b/src/control.c
@@ -428,7 +428,7 @@ void mlvpn_control_write_status(struct mlvpn_control *ctrl)
                        t->recvbytes,
                        0,
                        (uint32_t)t->srtt,
-                       (uint32_t)t->permitted,
+                       (uint32_t)t->permitted/1000000,
 //                       mlvpn_loss_ratio(t),
                        t->disconnects,
                        (uint32_t)t->last_activity,
diff --git a/src/mlvpn.c b/src/mlvpn.c
index c65383b..666e7f9 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -232,9 +232,9 @@ static void
 mlvpn_rtun_reorder_drain_timeout(EV_P_ ev_timer *w, int revents)
 {
     log_debug("reorder", "reorder timeout. Packet loss?");
-    printf("Reorder timeout\n");
+//    printf("Reorder timeout\n");
     mlvpn_reorder_skip(reorder_buffer);
-    mlvpn_rtun_reorder_drain(1);
+    mlvpn_rtun_reorder_drain(1);  // MARK = 1, old = 0
     if (freebuf->used == 0) {
         ev_timer_stop(EV_A_ w);
     }
@@ -849,7 +849,7 @@ mlvpn_rtun_recalc_weight_prio()
     return mlvpn_rtun_recalc_weight_bw();
   }
   mlvpn_tunnel_t *t;
-  double bwneeded=bandwidth*2;
+  double bwneeded=bandwidth*1.5;
   double bw=bwneeded;
   LIST_FOREACH(t, &rtuns, entries) {
     if ((t->quota == 0) && (t->status >= MLVPN_AUTHOK)) {
diff --git a/src/reorder.c b/src/reorder.c
index a22f4f8..18bd44c 100644
--- a/src/reorder.c
+++ b/src/reorder.c
@@ -51,7 +51,7 @@ struct cir_buffer {
 };
 
 #ifdef MARK
-//#include "mlvpn.h"
+
 struct pktlist 
 {
   mlvpn_pkt_t *pkt;
@@ -178,16 +178,19 @@ mlvpn_reorder_insert(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t *pkt)
     b->list_size++;
 //    printf("Insert    %lu  list size %d min %lu\n",pkt->seq, b->list_size, b->min_seqn);
 
-//    if (b->tail && ((int64_t)(b->min_seqn - b->tail->pkt->seq) > 0)) {
-//      printf("got old (insert) consider increasing buffer by %d\n",(int)(b->min_seqn - b->tail->pkt->seq));
-//    }
+    if (b->tail && ((int64_t)(b->min_seqn - b->tail->pkt->seq) > 0)) {
+      log_debug("reorder", "got old (insert) consider increasing buffer (%d behind)\n",(int)(b->min_seqn - b->tail->pkt->seq));
+    }
 
     return 0;
 }
 
 void mlvpn_reorder_skip(struct mlvpn_reorder_buffer *b)
 {
+  printf("number in list %u list_size_av %u min %lu tail %lu\n",b->list_size, b->list_size_av, b->min_seqn, b->tail->pkt->seq);
+  if (b->tail) {
     b->min_seqn=b->tail->pkt->seq; // Jump over any hole !!!!
+  }
 }
 
   
@@ -262,8 +265,24 @@ mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
     
   }
   if (drain_cnt > 1) {
+    int last=b->list_size_av;
     b->list_size_av = ((b->list_size_av*9) + (b->list_size + drain_cnt) + 5)/10;
-//    printf("%d\n",b->list_size_av);
+    if (b->list_size_av > 64) {
+      b->list_size_av = 64;
+      if (b->list_size_av != last ) {
+        log_info("reorder", "List size reached limit (64)\n");
+      }
+    } 
+    if (b->list_size_av < 4) {
+      b->list_size_av = 4;
+      if (b->list_size_av != last ) {
+        log_debug("reorder", "List size reached limit (4)\n");
+      }
+    } 
+//    if (b->list_size_av != last ) {
+//      log_debug("reorder", "Changed in list_size_av %d\n",b->list_size_av);
+//      printf("reorder:  Changed in list_size_av %d\n",b->list_size_av);
+//    }
   }
   
 //  if (b->tail) b->min_seqn++;
@@ -276,6 +295,9 @@ mlvpn_reorder_drain(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t **pkts,
 
 
 // OLD CODE....
+void mlvpn_reorder_skip(struct mlvpn_reorder_buffer *b)
+{
+}
 
 struct mlvpn_reorder_buffer *
 mlvpn_reorder_init(struct mlvpn_reorder_buffer *b, unsigned int bufsize,

From 65531b9b3e32f7edb2a496b76f933bf8587cf06c Mon Sep 17 00:00:00 2001
From: Mark <mark at helenandmark dot org>
Date: Mon, 28 May 2018 21:39:08 +0200
Subject: [PATCH 19/20] add permitted data to netdata

---
 src/control.c | 5 +++--
 src/mlvpn.c   | 6 +++---
 src/reorder.c | 2 +-
 3 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/control.c b/src/control.c
index 254e2e2..beeabe1 100644
--- a/src/control.c
+++ b/src/control.c
@@ -68,6 +68,7 @@ void mlvpn_control_write_status(struct mlvpn_control *ctrl);
     "   \"bandwidth\": %u,\n" \
     "   \"srtt\": %u,\n" \
     "   \"loss\": %u,\n" \
+    "   \"permitted\": %u,\n" \
     "   \"disconnects\": %u,\n" \
     "   \"last_packet\": %u,\n" \
     "   \"timeout\": %u,\n" \
@@ -428,8 +429,8 @@ void mlvpn_control_write_status(struct mlvpn_control *ctrl)
                        t->recvbytes,
                        0,
                        (uint32_t)t->srtt,
-                       (uint32_t)t->permitted,
-//                       mlvpn_loss_ratio(t),
+                       mlvpn_loss_ratio(t),
+                       (uint32_t)t->permitted/1000000,
                        t->disconnects,
                        (uint32_t)t->last_activity,
                        (uint32_t)t->timeout,
diff --git a/src/mlvpn.c b/src/mlvpn.c
index e6a065c..dd962f8 100644
--- a/src/mlvpn.c
+++ b/src/mlvpn.c
@@ -232,8 +232,8 @@ static void
 mlvpn_rtun_reorder_drain_timeout(EV_P_ ev_timer *w, int revents)
 {
     log_debug("reorder", "reorder timeout. Packet loss?");
-    printf("Reorder timeout\n");
-    mlvpn_rtun_reorder_drain(1);
+//    printf("Reorder timeout\n");
+    mlvpn_rtun_reorder_drain(0);
     if (freebuf->used == 0) {
         ev_timer_stop(EV_A_ w);
     }
@@ -848,7 +848,7 @@ mlvpn_rtun_recalc_weight_prio()
     return mlvpn_rtun_recalc_weight_bw();
   }
   mlvpn_tunnel_t *t;
-  double bwneeded=bandwidth*2;
+  double bwneeded=bandwidth*1.5;
   double bw=bwneeded;
   LIST_FOREACH(t, &rtuns, entries) {
     if ((t->quota == 0) && (t->status >= MLVPN_AUTHOK)) {
diff --git a/src/reorder.c b/src/reorder.c
index 98f473d..8209553 100644
--- a/src/reorder.c
+++ b/src/reorder.c
@@ -39,7 +39,7 @@
 #include "reorder.h"
 #include "log.h"
 
-#define MARK
+//#define MARK
 
 /* A generic circular buffer */
 struct cir_buffer {

From 418de3f6f63312af4dbe8c1ad5dd209aeb3b7a45 Mon Sep 17 00:00:00 2001
From: Mark <mark at helenandmark dot org>
Date: Tue, 29 May 2018 08:54:01 +0200
Subject: [PATCH 20/20] fix wrap round issue in netdata

---
 src/control.c | 2 +-
 src/reorder.c | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/control.c b/src/control.c
index beeabe1..d05cf90 100644
--- a/src/control.c
+++ b/src/control.c
@@ -430,7 +430,7 @@ void mlvpn_control_write_status(struct mlvpn_control *ctrl)
                        0,
                        (uint32_t)t->srtt,
                        mlvpn_loss_ratio(t),
-                       (uint32_t)t->permitted/1000000,
+                       (uint32_t)(t->permitted/1000000),
                        t->disconnects,
                        (uint32_t)t->last_activity,
                        (uint32_t)t->timeout,
diff --git a/src/reorder.c b/src/reorder.c
index 03022b3..2fedb75 100644
--- a/src/reorder.c
+++ b/src/reorder.c
@@ -39,7 +39,7 @@
 #include "reorder.h"
 #include "log.h"
 
-//#define MARK
+#define MARK
 
 /* A generic circular buffer */
 struct cir_buffer {
@@ -187,7 +187,7 @@ mlvpn_reorder_insert(struct mlvpn_reorder_buffer *b, mlvpn_pkt_t *pkt)
 
 void mlvpn_reorder_skip(struct mlvpn_reorder_buffer *b)
 {
-  printf("number in list %u list_size_av %u min %lu tail %lu\n",b->list_size, b->list_size_av, b->min_seqn, b->tail->pkt->seq);
+//  printf("number in list %u list_size_av %u min %lu tail %lu\n",b->list_size, b->list_size_av, b->min_seqn, b->tail->pkt->seq);
   if (b->tail) {
     b->min_seqn=b->tail->pkt->seq; // Jump over any hole !!!!
   }