diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2b054a6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# downloads +binaries + +#ci +.ci/ + +# coverage +coverage.out +coverage.txt \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index bfb9f3b..f9f028c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ service: - docker env: - - GO111MODULE=on GOPROXY=https://proxy.golang.org + - GO111MODULE=on GOPROXY=https://proxy.golang.org REG_TOKEN=$GITHUB_TOKEN matrix: fast_finish: true diff --git a/README.md b/README.md index 85fcca8..f0f0f16 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Usage: data-retrieval utils regression tester. -This tool executes several versions of data-retrieval utils and compares query times and resource usage. There should be at least two versions specified as arguments in the following way: +This tool executes several versions of data-retrieval utils and compares query times and resource usage. There should be at least one version specified as an argument in the following way: * v0.12.1 - release name from github (https://github.com/src-d/your-data-retrieval-util/releases). The binary will be downloaded. * latest - latest release from github. The binary will be downloaded. diff --git a/cmd/regression-retrieval/main.go b/cmd/regression-retrieval/main.go index 284d3d1..9244ee2 100644 --- a/cmd/regression-retrieval/main.go +++ b/cmd/regression-retrieval/main.go @@ -5,6 +5,7 @@ import ( "github.com/src-d/regression-retrieval/test" _ "github.com/src-d/regression-retrieval/test/gitcollector" + _ "github.com/src-d/regression-retrieval/test/metadata-retrieval" "github.com/jessevdk/go-flags" "github.com/src-d/regression-core" @@ -13,7 +14,7 @@ import ( var description = `data-retrieval utils regression tester. -This tool executes several versions of data-retrieval utils and compares query times and resource usage. There should be at least two versions specified as arguments in the following way: +This tool executes several versions of data-retrieval utils and compares query times and resource usage. There should be at least one version specified as an argument in the following way: * v0.12.1 - release name from github (https://github.com/src-d/your-data-retrieval-util/releases). The binary will be downloaded. * latest - latest release from github. The binary will be downloaded. @@ -26,6 +27,7 @@ This tool executes several versions of data-retrieval utils and compares query t The repositories and downloaded/built binaries are cached by default in "repos" and "binaries" repositories from the current directory. ` +// Options CLI options type Options struct { regression.Config diff --git a/go.mod b/go.mod index bfa99df..a3461da 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/mattn/go-colorable v0.1.4 // indirect github.com/mattn/go-isatty v0.0.9 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/client_golang v1.1.0 github.com/src-d/regression-core v0.0.0-20191003075028-b476aeec74d4 github.com/stretchr/testify v1.3.0 diff --git a/test/gitcollector/gitcollector.go b/test/gitcollector/gitcollector.go index 18beef9..f7b4368 100644 --- a/test/gitcollector/gitcollector.go +++ b/test/gitcollector/gitcollector.go @@ -2,7 +2,6 @@ package gitcollector import ( "fmt" - "io/ioutil" "os" "strings" "text/tabwriter" @@ -13,11 +12,10 @@ import ( "github.com/src-d/regression-core" "gopkg.in/src-d/go-log.v1" - "gopkg.in/yaml.v2" ) type ( - gitCollectorResults map[string][]*Result + gitCollectorResults map[string][]*test.Result versionResults map[string]gitCollectorResults // Test holds the information about a gitcollector test @@ -38,12 +36,6 @@ func init() { test.Register(Kind, NewTest) } -// Result is a wrapper around regression.Result that additionally contains organizations that were processed -type Result struct { - *regression.Result - Organizations string -} - // NewTest creates a new Test struct func NewTest(config regression.Config) (test.Test, error) { l, err := (&log.LoggerFactory{Level: log.InfoLevel}).New(log.Fields{}) @@ -105,14 +97,14 @@ func (t *Test) RunLoad() error { } rf := gitCollector.ExtraFile("regression.yml") - if organizations, err := loadOrganizationsYaml(rf); err != nil { + if organizations, err := test.LoadOrganizationsYaml(rf); err != nil { return err } else { t.organizations = organizations } for _, orgs := range t.organizations { - results[version][orgs] = make([]*Result, times) + results[version][orgs] = make([]*test.Result, times) for i := 0; i < times; i++ { l.New(log.Fields{ "orgs": orgs, @@ -137,7 +129,7 @@ func (t *Test) RunLoad() error { func (t *Test) runLoadTest( gitCollector *regression.Binary, orgs string, -) (*Result, error) { +) (*test.Result, error) { t.log.Infof("Executing gitcollector test") command := NewCommand(gitCollector.Path, orgs) @@ -167,7 +159,7 @@ func (t *Test) runLoadTest( Memory: rusage.Maxrss * 1024, } - r := &Result{ + r := &test.Result{ Result: result, Organizations: orgs, } @@ -273,8 +265,8 @@ func (t *Test) GetResults() bool { queryA := a[orgs][0] queryB := b[orgs][0] - queryA.Result = average(a[orgs]) - queryB.Result = average(b[orgs]) + queryA.Result = test.Average(a[orgs]) + queryB.Result = test.Average(b[orgs]) c := queryA.Result.ComparePrint(queryB.Result, 10.0) if !c { ok = false @@ -292,7 +284,7 @@ func (t *Test) GetResults() bool { func (t *Test) SaveLatestCSV() { version := t.config.Versions[len(t.config.Versions)-1] for _, orgs := range t.organizations { - res := average(t.results[version][orgs]) + res := test.Average(t.results[version][orgs]) if err := res.SaveAllCSV(fmt.Sprintf("plot_%s_", strings.Replace(orgs, ",", "_", -1))); err != nil { panic(err) } @@ -304,38 +296,10 @@ func (t *Test) StoreLatestToPrometheus(promConfig regression.PromConfig, ciConfi version := t.config.Versions[len(t.config.Versions)-1] cli := prometheus.NewPromClient(Kind, promConfig) for _, orgs := range t.organizations { - res := average(t.results[version][orgs]) + res := test.Average(t.results[version][orgs]) if err := cli.Dump(res, version, orgs, ciConfig.Branch, ciConfig.Commit); err != nil { return err } } return nil } - -func average(pr []*Result) *regression.Result { - if len(pr) == 0 { - return nil - } - - results := make([]*regression.Result, 0, len(pr)) - for _, r := range pr { - results = append(results, r.Result) - } - - return regression.Average(results) -} - -func loadOrganizationsYaml(file string) ([]string, error) { - text, err := ioutil.ReadFile(file) - if err != nil { - return nil, err - } - - var res []string - err = yaml.Unmarshal(text, &res) - if err != nil { - return nil, err - } - - return res, nil -} diff --git a/test/metadata-retrieval/command.go b/test/metadata-retrieval/command.go new file mode 100644 index 0000000..42c390d --- /dev/null +++ b/test/metadata-retrieval/command.go @@ -0,0 +1,51 @@ +package metadataretrieval + +import ( + "os" + "os/exec" + "syscall" +) + +// Command wraps a metadata-retrieval server instance. +type Command struct { + cmd *exec.Cmd + binary string + orgs string + dir string +} + +// NewCommand creates a new metadata-retrieval command struct. +func NewCommand(binary, orgs string) *Command { + return &Command{ + cmd: new(exec.Cmd), + binary: binary, + orgs: orgs, + } +} + +// Run runs metadata-retrieval util to discover and download organizations +func (c *Command) Run(envs map[string]string) error { + c.cmd = exec.Command( + c.binary, + "ghsync", + "--version", "0", + "--orgs", c.orgs, + "--no-forks", + ) + c.cmd.Stdout = os.Stdout + c.cmd.Stderr = os.Stderr + c.cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } + for k, v := range envs { + c.cmd.Env = append(c.cmd.Env, k+"="+v) + } + + return c.cmd.Run() +} + +// Rusage returns usage counters +func (c *Command) Rusage() *syscall.Rusage { + rusage, _ := c.cmd.ProcessState.SysUsage().(*syscall.Rusage) + return rusage +} diff --git a/test/metadata-retrieval/metadataretrieval.go b/test/metadata-retrieval/metadataretrieval.go new file mode 100644 index 0000000..56642f7 --- /dev/null +++ b/test/metadata-retrieval/metadataretrieval.go @@ -0,0 +1,309 @@ +package metadataretrieval + +import ( + "fmt" + "os" + "strings" + "text/tabwriter" + "time" + + "github.com/src-d/regression-retrieval/prometheus" + "github.com/src-d/regression-retrieval/test" + + "github.com/src-d/regression-core" + "gopkg.in/src-d/go-log.v1" +) + +type ( + metadataRetrievalResults map[string][]*test.Result + versionResults map[string]metadataRetrievalResults + + // Test holds the information about a metadata-retrieval test + Test struct { + config regression.Config + metadataRetrieval map[string]*regression.Binary + // organizations is array of lists of coma-separated organizations + organizations []string + results versionResults + log log.Logger + } +) + +// TODO(kyrcha): refactor in order to make the code DRY (gitcollector.go and metadataretrieval.go) + +// Kind is an identifier of util type to be tested, used in factory test constructor +const Kind = "metadata-retrieval" + +func init() { + test.Register(Kind, NewTest) +} + +// Result is a wrapper around regression.Result that additionally contains organizations that were processed +type Result struct { + *regression.Result + Organizations string +} + +// NewTest creates a new Test struct +func NewTest(config regression.Config) (test.Test, error) { + l, err := (&log.LoggerFactory{Level: log.InfoLevel}).New(log.Fields{}) + if err != nil { + return nil, err + } + + return &Test{ + config: config, + log: l, + }, nil +} + +// Prepare downloads and builds required metadata-retrieval versions +func (t *Test) Prepare() error { + return t.prepareMetadataRetrieval() +} + +func (t *Test) prepareMetadataRetrieval() error { + t.log.Infof("Preparing metadata-retrieval binaries") + releases := regression.NewReleases("src-d", "metadata-retrieval", t.config.GitHubToken) + + t.metadataRetrieval = make(map[string]*regression.Binary, len(t.config.Versions)) + for _, version := range t.config.Versions { + b := NewMetadataRetrieval(t.config, version, releases) + err := b.Download() + if err != nil { + return err + } + + t.metadataRetrieval[version] = b + } + + return nil +} + +// RunLoad executes the tests +func (t *Test) RunLoad() error { + results := make(versionResults) + + for _, version := range t.config.Versions { + _, ok := results[version] + if !ok { + results[version] = make(metadataRetrievalResults) + } + + metadataRetrieval, ok := t.metadataRetrieval[version] + if !ok { + panic("metadataRetrieval not initialized. Was Prepare called?") + } + + l := t.log.New(log.Fields{"version": version}) + + l.Infof("Running version tests") + + times := t.config.Repeat + if times < 1 { + times = 1 + } + + // TODO(kyrcha): add a regression.yml file to metadata-retrieval + t.organizations = []string{"git-fixtures"} + + for _, orgs := range t.organizations { + results[version][orgs] = make([]*test.Result, times) + for i := 0; i < times; i++ { + l.New(log.Fields{ + "orgs": orgs, + }).Infof("Running query") + + result, err := t.runLoadTest(metadataRetrieval, orgs) + results[version][orgs][i] = result + + if err != nil { + return err + } + } + } + } + + t.results = results + + return nil +} + +// runLoadTest runs metadata-retrieval download command and saves execution time + memory usage +func (t *Test) runLoadTest( + metadataRetrieval *regression.Binary, + orgs string, +) (*test.Result, error) { + t.log.Infof("Executing metadata-retrieval test") + + command := NewCommand(metadataRetrieval.Path, orgs) + start := time.Now() + if err := command.Run(map[string]string{ + "LOG_LEVEL": "debug", + "GITHUB_TOKENS": t.config.GitHubToken, + }); err != nil { + t.log.With(log.Fields{ + "orgs": orgs, + "metadata-retrieval": metadataRetrieval.Path, + }).Errorf(err, "Could not execute metadata-retrieval") + return nil, err + } + wall := time.Since(start) + rusage := command.Rusage() + + t.log.With(log.Fields{ + "wall": wall, + "memory": rusage.Maxrss, + }).Infof("Finished queries") + + result := ®ression.Result{ + Wtime: wall, + Stime: time.Duration(rusage.Stime.Nano()), + Utime: time.Duration(rusage.Utime.Nano()), + Memory: rusage.Maxrss * 1024, + } + + r := &test.Result{ + Result: result, + Organizations: orgs, + } + + return r, nil +} + +// PrintTabbedResults prints table with results to stdout +// Example: +// Org | remote:add-regression-config +// bblfsh,git-fixtures | 1m12.223594627s +func (t *Test) PrintTabbedResults() { + w := tabwriter.NewWriter(os.Stdout, 0, 0, 0, ' ', tabwriter.TabIndent|tabwriter.Debug) + fmt.Fprint(w, "\x1b[1;33m Org \x1b[0m") + versions := t.config.Versions + for _, v := range versions { + fmt.Fprintf(w, "\t\x1b[1;33m %s \x1b[0m", v) + } + fmt.Fprintf(w, "\n") + + for _, orgs := range t.organizations { + fmt.Fprintf(w, "\x1b[1;37m %s \x1b[0m", orgs) + var ( + mini int + min time.Duration + maxi int + max time.Duration + results []string + ) + for i, v := range versions { + if r, found := t.results[v][orgs]; !found { + results = append(results, "--") + } else { + t := r[0].Wtime + for _, ri := range r[1:] { + if ri.Wtime < min { + t = ri.Wtime + } + } + + if min == 0 { + min = t + } + + if max == 0 { + max = t + } + + if t < min { + min = t + mini = i + } + + if t > max { + max = t + maxi = i + } + + results = append(results, t.String()) + } + } + + for i, r := range results { + fmt.Fprint(w, "\t") + if i == mini { + fmt.Fprintf(w, "\x1b[1;32m %s \x1b[0m", r) + } else if i == maxi { + fmt.Fprintf(w, "\x1b[1;31m %s \x1b[0m", r) + } else { + fmt.Fprintf(w, "\x1b[1;37m %s \x1b[0m", r) + } + } + fmt.Fprintf(w, "\n") + } + w.Flush() + fmt.Println() +} + +// GetResults prints test results and returns if the tests passed +func (t *Test) GetResults() bool { + if len(t.config.Versions) < 1 { + panic("there should be at least one version") + } + + versions := t.config.Versions + ok := true + for i, version := range versions[0 : len(versions)-1] { + fmt.Printf("%s - %s ####\n", version, versions[i+1]) + a := t.results[versions[i]] + b := t.results[versions[i+1]] + + for _, orgs := range t.organizations { + fmt.Printf("## Organizations {%s} ##\n", orgs) + if _, found := a[orgs]; !found { + fmt.Printf("# Skip - organizations {%s} not found for version: %s\n", orgs, versions[i]) + continue + } + if _, found := b[orgs]; !found { + fmt.Printf("# Skip - organizations {%s} not found for version: %s\n", orgs, versions[i+1]) + continue + } + + queryA := a[orgs][0] + queryB := b[orgs][0] + + queryA.Result = test.Average(a[orgs]) + queryB.Result = test.Average(b[orgs]) + c := queryA.Result.ComparePrint(queryB.Result, 10.0) + if !c { + ok = false + } + } + } + + return ok +} + +// SaveLatestCSV saves test results in a CSV files +// created file examples: +// - plot_org1_org2_org3_memory.csv +// - plot_org1_org2_org3_time.csv +func (t *Test) SaveLatestCSV() { + version := t.config.Versions[len(t.config.Versions)-1] + for _, orgs := range t.organizations { + res := test.Average(t.results[version][orgs]) + if err := res.SaveAllCSV(fmt.Sprintf("plot_%s_", strings.Replace(orgs, ",", "_", -1))); err != nil { + panic(err) + } + } +} + +// StoreLatestToPrometheus stores latest version results to prometheus pushgateway +func (t *Test) StoreLatestToPrometheus(promConfig regression.PromConfig, ciConfig regression.CIConfig) error { + version := t.config.Versions[len(t.config.Versions)-1] + cli := prometheus.NewPromClient(Kind, promConfig) + for _, orgs := range t.organizations { + res := test.Average(t.results[version][orgs]) + if err := cli.Dump(res, version, orgs, ciConfig.Branch, ciConfig.Commit); err != nil { + return err + } + } + return nil +} diff --git a/test/metadata-retrieval/tool.go b/test/metadata-retrieval/tool.go new file mode 100644 index 0000000..1becf20 --- /dev/null +++ b/test/metadata-retrieval/tool.go @@ -0,0 +1,33 @@ +package metadataretrieval + +import "github.com/src-d/regression-core" + +// NewToolMetadataRetrieval creates a Tool with metadata-retrieval parameters filled +func NewToolMetadataRetrieval() regression.Tool { + return regression.Tool{ + Name: "metadata-retrieval", + GitURL: "https://github.com/src-d/metadata-retrieval", + ProjectPath: "github.com/src-d/metadata-retrieval", + BinaryName: "cmd", + BuildSteps: []regression.BuildStep{ + { + Dir: "", + Command: "make", + Args: []string{"packages"}, + Env: []string{"GOPROXY=https://proxy.golang.org"}, + }, + }, + // ExtraFiles: []string{ + // "testdata/regression.yml", + // }, + } +} + +// NewMetadataRetrieval returns a Binary struct for metadata-retrieval Tool +func NewMetadataRetrieval( + config regression.Config, + version string, + releases *regression.Releases, +) *regression.Binary { + return regression.NewBinary(config, NewToolMetadataRetrieval(), version, releases) +} diff --git a/test/test.go b/test/test.go index 58f5e61..05e704c 100644 --- a/test/test.go +++ b/test/test.go @@ -1,13 +1,22 @@ package test import ( + "io/ioutil" + "github.com/src-d/regression-core" "gopkg.in/src-d/go-errors.v1" + "gopkg.in/yaml.v2" ) // Constructor is a type that represents function of default Test Constructor type Constructor func(config regression.Config) (Test, error) +// Result is a wrapper around regression.Result that additionally contains organizations that were processed +type Result struct { + *regression.Result + Organizations string +} + var ( // constructors is a map of all supported test constructors constructors = make(map[string]Constructor) @@ -54,3 +63,31 @@ func ValidateKind(kind string) (Constructor, error) { return c, nil } + +func Average(pr []*Result) *regression.Result { + if len(pr) == 0 { + return nil + } + + results := make([]*regression.Result, 0, len(pr)) + for _, r := range pr { + results = append(results, r.Result) + } + + return regression.Average(results) +} + +func LoadOrganizationsYaml(file string) ([]string, error) { + text, err := ioutil.ReadFile(file) + if err != nil { + return nil, err + } + + var res []string + err = yaml.Unmarshal(text, &res) + if err != nil { + return nil, err + } + + return res, nil +} diff --git a/test_test.go b/test_test.go index 44c0566..537aec2 100644 --- a/test_test.go +++ b/test_test.go @@ -1,11 +1,15 @@ package regression_retrieval import ( + "fmt" + "os" "testing" - "github.com/src-d/regression-core" "github.com/src-d/regression-retrieval/test" "github.com/src-d/regression-retrieval/test/gitcollector" + metadataretrieval "github.com/src-d/regression-retrieval/test/metadata-retrieval" + + "github.com/src-d/regression-core" "github.com/stretchr/testify/require" ) @@ -27,3 +31,25 @@ func TestGitCollector(t *testing.T) { gitCollectorTest.GetResults() } + +// TestMetadataRetrieval +// runs regression comparison for remote:master and the first release +// no errors occurred during tests execution +// GetResults returns true +func TestMetadataRetrieval(t *testing.T) { + config := regression.NewConfig() + config.BinaryCache = "binaries" + config.Versions = []string{"remote:master", "v0.1.0"} + config.Repeat = 1 + // No token, no access to the v4 API + config.GitHubToken = os.Getenv("REG_TOKEN") + + metadataRetrievalTest, err := test.NewTest(metadataretrieval.Kind, config) + fmt.Printf("%+v", config) + require.NoError(t, err) + + require.NoError(t, metadataRetrievalTest.Prepare()) + require.NoError(t, metadataRetrievalTest.RunLoad()) + + metadataRetrievalTest.GetResults() +}