code cleanup (#41)

* code cleanup

* address comments

Author: zhukexin1228

bankofanthos_prototype/README.md

@@ -1,6 +1,14 @@
 # Eval Driver Prototype on Bank Of Anthos
 Eval driver runs diff testing for bank of anthos, it takes two versions of Bank Of Anthos and sampled requests as input, display response diff and database diff as output.
 
+For the same set of requests, we run 4 different trials:
+1. Send all requests to v1 as Control
+2. Send all requests to v2 as Experimental_1
+3. Send half request to v1, half request to v2 as Experimental_2
+4. Send half request to v2, half request to v1  as Experimental_3
+
+While we do diffs at the end, we compare Control and Experimental_*.
+
 ## Prereq
 - Install [Service Weaver](https://serviceweaver.dev/docs.html#what-is-service-weaver) locally
 - Install [Docker](https://www.docker.com/get-started) locally

bankofanthos_prototype/eval_driver/benchmark/README.md

@@ -1,7 +1,7 @@
 # Benchmark For R+/R- Database performance
 ## Prerequsite
 - Install [Dolt](https://docs.dolthub.com/introduction/installation)
-- Install [pg2mysql tool](https://docs.dolthub.com/introduction/installation) to convert postgres database to mysql database
+- Install [pg2mysql tool](https://docs.dolthub.com/introduction/installation) to convert postgres database to mysql database for Dolt setup
 - Install [Postgres](https://www.postgresql.org/download/)
 
 ## Setup
@@ -10,36 +10,52 @@ Run script to create `benchmark` database with two tables users(username, passwo
 ./init.sh $row_num$
 ``` 
 
-## Cloning
+You can create multiple databases with different name by using ths script.
+
+## Instruction
+```code
+go run . --dbUrlLists dbUrl1 --dbUrlLists dbUrl2 --dbUrlLIsts dbUrl3 ...
+``` 
+Pass multiple database urls to the command for benchmark.
+
+All the stats will be dumped into a json file. Run the plot.py file to plot the benchmark stats.
+
+```code
+python plot.py
+``` 
+
+## Benchmarks
+We benchmark clone one entire database, read/write performance and diffing two tables.
+### Cloning
 Create a table with size X
-### Baseline 1: Postgres
+#### Baseline 1: Postgres
 Dump the whole database and then restore the database
-### Baseline 2: Dolt
+#### Baseline 2: Dolt
 Create a new branch for the database
-### R+/R-
+#### R+/R-
 Create a new branch for the database
 
-## Read/Write Performance
+### Read/Write Performance
 Write Y row into Table X With and Without Primary Key\
 Read queries:
 - 	SELECT * FROM TABLE WHERE username = 'aaaa';
 - 	SELECT * FROM TABLE WHERE LENGTH(password) = 8 AND username LIKE 'a%';
 -	SELECT COUNT(*) FROM TABLE;
 -   SELECT * FROM TABLE
-### Baseline 1: Postgres
+#### Baseline 1: Postgres
 Execute Read/Write oprations on table
-### Baseline 2: Dolt
+#### Baseline 2: Dolt
 Execute Read/Write oprations on new branch
-### R+/R-
+#### R+/R-
 Execute Read/Write oprations on new branch
 
-## Diffing
+### Diffing
 Diff table size X with Y modified rows
-### Baseline 1: Postgres
+#### Baseline 1: Postgres
 Compare two table with `SELECT * FROM TABLE A EXCEPT ALL SELECT * FROM TABLE B;` and `SELECT * FROM TABLE B EXCEPT ALL SELECT * FROM TABLE A;`
 
-### Baseline 2: Dolt
+#### Baseline 2: Dolt
 Diff two branches
 
-### R+/R-
+#### R+/R-
 Diff two branches

bankofanthos_prototype/eval_driver/benchmark/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"flag"
+	"fmt"
 	"log"
 	"os"
 	"path/filepath"
@@ -11,9 +12,26 @@ import (
 
 const dumpDir = "dump"
 
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+	return strings.Join(*i, ", ")
+}
+
+func (i *arrayFlags) Set(value string) error {
+	if i == nil {
+		return fmt.Errorf("array flag is null")
+	}
+	*i = append(*i, value)
+	return nil
+}
+
 func main() {
 	var debug bool
+	var dbs arrayFlags
+
 	flag.BoolVar(&debug, "debug", false, "Debug / analyze R+/R- implementation")
+	flag.Var(&dbs, "dbUrlLists", "Database url lists for benchmark to run")
 	flag.Parse()
 
 	if _, err := os.Stat(dumpDir); os.IsNotExist(err) {
@@ -22,8 +40,6 @@ func main() {
 		}
 	}
 
-	dbs := []string{"postgresql://postgres:postgres@localhost:5433/benchmark_1mb", "postgresql://postgres:postgres@localhost:5433/benchmark_20mb", "postgresql://postgres:postgres@localhost:5433/benchmark_100mb"}
-
 	metricsStats := map[string]map[string]*metrics{} // {Database: {table:metrics, table_with_primary_key:metrics}}
 
 	for _, dbUrl := range dbs {

bankofanthos_prototype/eval_driver/benchmark/plot.py

@@ -3,8 +3,7 @@
 import matplotlib.pyplot as plt
 import numpy as np
 
-
-database = ['benchmark_1mb','benchmark_20mb','benchmark_100mb']
+database = []
 types = ['RPlusRMinus','Postgres', 'Dolt']
 colors = ['blue', 'green', 'red']
 table = 'users'
@@ -137,6 +136,9 @@ def plot(x_values, y_values, yerr_values, colors,line_names,title_name):
 
 def main():
     data = readFromFile("dump/metrics.json")
+    global database
+    database = data.keys()
+    
     plotStats(data)
 
 if __name__ == "__main__":

bankofanthos_prototype/eval_driver/main.go

@@ -36,7 +36,7 @@ func runTrail(ctx context.Context, trail *utility.Trail, branchers map[string]*d
 	var prodServices []*utility.ProdService
 	if trail.IsControl() {
 		prodServices = []*utility.ProdService{v1ProdService}
-	} else if trail.IsConaryOnly() {
+	} else if trail.IsCanaryOnly() {
 		prodServices = []*utility.ProdService{v2ProdService}
 	} else {
 		prodServices = []*utility.ProdService{v1ProdService, v2ProdService}
@@ -90,7 +90,7 @@ func main() {
 	var deleteBranches, inlineDiff, respDiff bool
 	flag.StringVar(&configFile, "configFile", "config.toml", "Config file for eval")
 	flag.BoolVar(&deleteBranches, "deleteBranches", true, "Delete branches at the end of eval run, only set false for investigation purpose")
-	flag.BoolVar(&inlineDiff, "inlineDiff", false, "Whether to use inline diff or not")
+	flag.BoolVar(&inlineDiff, "inlineDiff", false, "Whether to use inline diff or side by side diff")
 	flag.BoolVar(&respDiff, "respDiff", true, "Whether to show response diff or not")
 	flag.Parse()
 

bankofanthos_prototype/eval_driver/utility/trail.go

@@ -67,6 +67,6 @@ func (t *Trail) IsControl() bool {
 	return t.Name == "Control"
 }
 
-func (t *Trail) IsConaryOnly() bool {
+func (t *Trail) IsCanaryOnly() bool {
 	return t.Name == "E_C"
 }