Check for alternative seed url

go.sum

@@ -849,6 +849,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA
 golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
 golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
+golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
 golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
@@ -857,6 +858,7 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

k8s/kustomization.yaml

@@ -11,29 +11,3 @@ resources:
 
 patchesStrategicMerge:
   - deployment_log_level_patch.yaml
-
-patchesJson6902:
-  - target:
-      kind: Deployment
-      name: veidemann-harvester
-      version: v1
-      group: apps
-    patch: |
-      - op: replace
-        path: /spec/template/spec/containers/0/envFrom/0/secretRef/name
-        value: skaffold-veidemann-browser-controller-rethinkdb
-
-      - op: replace
-        path: /spec/template/spec/volumes/3/secret/secretName
-        value: skaffold-veidemann-browser-controller-cache-cert
-
-secretGenerator:
-  - name: skaffold-veidemann-browser-controller-rethinkdb
-    envs:
-      - .env
-  - name: skaffold-veidemann-browser-controller-cache-cert
-    type: Opaque
-    files:
-      - cert/cache-selfsigned.key
-      - cert/cache-selfsignedCA.crt
-      - cert/ec.param

pkg/database/codec.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	configV1 "github.com/nlnwa/veidemann-api/go/config/v1"
+	eventHandlerV1 "github.com/nlnwa/veidemann-api/go/eventhandler/v1"
 	frontierV1 "github.com/nlnwa/veidemann-api/go/frontier/v1"
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
@@ -63,6 +64,22 @@ var decodeCrawlExecutionStatus = func(encoded interface{}, value reflect.Value)
 	return nil
 }
 
+var decodeEventObject = func(encoded interface{}, value reflect.Value) error {
+	b, err := json.Marshal(encoded)
+	if err != nil {
+		return fmt.Errorf("error decoding EventObject: %w", err)
+	}
+
+	var eo eventHandlerV1.EventObject
+	err = protojson.Unmarshal(b, &eo)
+	if err != nil {
+		return fmt.Errorf("error decoding EventJobject: #{err}")
+	}
+
+	value.Set(reflect.ValueOf(eo))
+	return nil
+}
+
 var encodeProtoMessage = func(value interface{}) (i interface{}, err error) {
 	b, err := protojson.Marshal(value.(proto.Message))
 	if err != nil {
@@ -88,6 +105,11 @@ func init() {
 		encodeProtoMessage,
 		decodeCrawlExecutionStatus,
 	)
+	encoding.SetTypeEncoding(
+		reflect.TypeOf(&eventHandlerV1.EventObject{}),
+		encodeProtoMessage,
+		decodeEventObject,
+	)
 	encoding.SetTypeEncoding(
 		reflect.TypeOf(map[string]interface{}{}),
 		func(value interface{}) (i interface{}, err error) {

pkg/database/database.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	configV1 "github.com/nlnwa/veidemann-api/go/config/v1"
+	eventHandlerV1 "github.com/nlnwa/veidemann-api/go/eventhandler/v1"
 	"github.com/sirupsen/logrus"
 	r "gopkg.in/rethinkdb/rethinkdb-go.v6"
 	"time"
@@ -104,6 +105,24 @@ func (c *RethinkDbConnection) GetConfigObject(ctx context.Context, ref *configV1
 	return &result, nil
 }
 
+func (c *RethinkDbConnection) GetSeedByExecutionId(ctx context.Context, executionId string) (*configV1.ConfigObject, error) {
+	term := r.Table("executions").Get(executionId).Do(func(doc r.Term) interface{} {
+		return r.Table("config_seeds").Get(doc.Field("seedId"))
+	})
+	res, err := c.execRead(ctx, "get-seed-by-execution-id", &term)
+	if err != nil {
+		return nil, err
+	}
+	result := new(configV1.ConfigObject)
+	err = res.One(result)
+	if err != nil {
+		return nil, err
+	}
+
+	return result, nil
+}
+
+
 // GetConfigsForSelector fetches a list of config.ConfigObject's matching config.Kind and config.Label
 func (c *RethinkDbConnection) GetConfigsForSelector(ctx context.Context, kind configV1.Kind, label *configV1.Label) ([]*configV1.ConfigObject, error) {
 	term := r.Table("config").GetAllByIndex("label", r.Expr([]string{label.Key, label.Value})).
@@ -132,6 +151,11 @@ func (c *RethinkDbConnection) GetConfigsForSelector(ctx context.Context, kind co
 	return configObjects, nil
 }
 
+func (c *RethinkDbConnection) WriteEvent(ctx context.Context, eventObject *eventHandlerV1.EventObject) error {
+	term := r.Table("events").Insert(eventObject)
+	return c.execWrite(ctx, "write-event", &term)
+}
+
 // execRead executes the given read term with a timeout
 func (c *RethinkDbConnection) execRead(ctx context.Context, name string, term *r.Term) (*r.Cursor, error) {
 	q := func(ctx context.Context) (*r.Cursor, error) {

pkg/database/dbadapter.go

@@ -20,18 +20,23 @@ import (
 	"context"
 	"fmt"
 	configV1 "github.com/nlnwa/veidemann-api/go/config/v1"
+	eventHandlerV1 "github.com/nlnwa/veidemann-api/go/eventhandler/v1"
 	"strings"
 	"time"
 )
 
 type ConfigCache interface {
 	GetConfigObject(context.Context, *configV1.ConfigRef) (*configV1.ConfigObject, error)
 	GetScripts(context.Context, *configV1.BrowserConfig) ([]*configV1.ConfigObject, error)
+	WriteEvent(ctx context.Context, object *eventHandlerV1.EventObject) error
+	GetSeedByExecutionId(ctx context.Context, executionId string) (*configV1.ConfigObject, error)
 }
 
 type DbAdapter interface {
 	GetConfigObject(context.Context, *configV1.ConfigRef) (*configV1.ConfigObject, error)
 	GetConfigsForSelector(context.Context, configV1.Kind, *configV1.Label) ([]*configV1.ConfigObject, error)
+	WriteEvent(ctx context.Context, object *eventHandlerV1.EventObject) error
+	GetSeedByExecutionId(ctx context.Context, executionId string) (*configV1.ConfigObject, error)
 }
 
 type configCache struct {
@@ -104,3 +109,15 @@ func (cc *configCache) GetScripts(ctx context.Context, browserConfig *configV1.B
 	}
 	return scripts, nil
 }
+
+
+func (cc *configCache) GetSeedByExecutionId(ctx context.Context, executionId string) (*configV1.ConfigObject, error) {
+	return cc.db.GetSeedByExecutionId(ctx, executionId)
+}
+
+
+
+func (cc *configCache) WriteEvent(ctx context.Context, eventObject *eventHandlerV1.EventObject) error {
+	return cc.db.WriteEvent(ctx, eventObject)
+}
+

pkg/session/session.go

@@ -33,6 +33,7 @@ import (
 	"github.com/chromedp/chromedp/device"
 	"github.com/google/uuid"
 	configV1 "github.com/nlnwa/veidemann-api/go/config/v1"
+	eventHandlerV1 "github.com/nlnwa/veidemann-api/go/eventhandler/v1"
 	frontierV1 "github.com/nlnwa/veidemann-api/go/frontier/v1"
 	logV1 "github.com/nlnwa/veidemann-api/go/log/v1"
 	"github.com/nlnwa/veidemann-browser-controller/pkg/database"
@@ -430,6 +431,8 @@ func (sess *Session) Fetch(ctx context.Context, QUri *frontierV1.QueuedUri, craw
 		PageFetchTimeMs: fetchDuration.Milliseconds(),
 	}
 
+	sess.AlternativeSeed(ctx)
+
 	log.Debugf("Fetch done: %v", QUri.Uri)
 	return result, nil
 }
@@ -597,3 +600,62 @@ func (sess *Session) AbortFetch() {
 	sess.frameWg.Cancel()
 	sess.loadCancel()
 }
+
+func (sess *Session) AlternativeSeed(ctx context.Context) {
+	if sess.Requests.RootRequest().CrawlLog == nil || sess.Requests.RootRequest().RedirectParent == nil {
+		return
+	}
+	discoveryPath := sess.Requests.RootRequest().CrawlLog.DiscoveryPath
+	if discoveryPath != "R" && discoveryPath != "RR" {
+		return
+	}
+	data := []*eventHandlerV1.Data{
+		{
+			Key:   "Url",
+			Value: sess.Requests.InitialRequest().Url,
+		},
+
+		{
+			Key:   "Alternative Url",
+			Value: sess.Requests.RootRequest().CrawlLog.RequestedUri,
+		},
+
+		{
+			Key:   "Discovery path",
+			Value: discoveryPath,
+		},
+	}
+
+	seed, err := sess.configCache.GetSeedByExecutionId(ctx, sess.Requests.RootRequest().CrawlLog.ExecutionId)
+	if err != nil {
+		log.Errorf("Failed to get seed by execution id")
+	}
+
+	data = append(data, &eventHandlerV1.Data{
+		Key:   "SeedId",
+		Value: seed.Id,
+	})
+
+	event := &eventHandlerV1.EventObject{
+		Type:     "Alternative seed",
+		Source:   "veidemann-browser-controller",
+		State:    eventHandlerV1.EventObject_NEW,
+		Data:     data,
+		Severity: eventHandlerV1.EventObject_INFO,
+		Activity: []*eventHandlerV1.Activity{
+			{
+				ModifiedBy:   "veidemann-browser-controller",
+				ModifiedTime: timestamppb.Now(),
+				Description: []*eventHandlerV1.Activity_Change{
+					{
+						Type:   eventHandlerV1.Activity_CREATED,
+					},
+				},
+			},
+		},
+	}
+	if err := sess.configCache.WriteEvent(ctx, event); err != nil {
+		log.Errorf("Error writing alternative seed event: %v", err)
+	}
+}
+