From 451efcddd78ef0dfac4f73cc7ca6fe2a6e6debce Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 9 Sep 2025 16:25:09 -0700 Subject: [PATCH 01/12] update funcs --- grids/filters.go | 7 +-- grids/graph.go | 117 +++++++++++++++++++++------------------- grids/index.go | 14 +++-- gripql/python/setup.py | 3 +- test/processors_test.go | 18 +++---- 5 files changed, 87 insertions(+), 72 deletions(-) diff --git a/grids/filters.go b/grids/filters.go index b0e620ad..c4c4e32e 100644 --- a/grids/filters.go +++ b/grids/filters.go @@ -2,7 +2,8 @@ package grids import ( bFilters "github.com/bmeg/benchtop/filters" - "github.com/bmeg/benchtop/jsontable" + "github.com/bmeg/benchtop/jsontable/table" + "github.com/bmeg/benchtop/jsontable/tpath" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" "github.com/bytedance/sonic" @@ -70,9 +71,9 @@ func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { // Handle lookup based on input type switch v := val.(type) { case map[string]any: - lookupVal = jsontable.PathLookup(v, cond.Key) + lookupVal = tpath.PathLookup(v, cond.Key) case []byte: - pathArr, err := jsontable.ConvertJSONPathToArray(cond.Key) + pathArr, err := table.ConvertJSONPathToArray(cond.Key) if err != nil { log.Errorf("Error converting JSON path: %v", err) return false diff --git a/grids/graph.go b/grids/graph.go index f8dd2dec..8c4b12e9 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -11,7 +11,8 @@ import ( "sync/atomic" "github.com/bmeg/benchtop" - "github.com/bmeg/benchtop/jsontable" + jTable "github.com/bmeg/benchtop/jsontable/table" + "github.com/bmeg/benchtop/jsontable/tpath" "github.com/bmeg/benchtop/pebblebulk" "github.com/bmeg/grip/engine/core" "github.com/bmeg/grip/gdbi" @@ -54,7 +55,7 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) return fmt.Errorf("indexVertex: %s", err) } ggraph.jsonkv.Lock.Lock() - table = newTable.(*jsontable.JSONTable) + table = newTable.(*jTable.JSONTable) ggraph.jsonkv.Tables[vertexLabel] = table ggraph.jsonkv.Lock.Unlock() } @@ -69,19 +70,22 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) if err != nil { return fmt.Errorf("AddVertex Error %s", err) } - table.AddTableEntryInfo(tx, []byte(vertex.ID), *rowLoc) - _, ok = ggraph.jsonkv.PageCache.Set(vertex.ID, *rowLoc) + err = ggraph.jsonkv.AddTableEntryInfo(tx, []byte(vertex.ID), rowLoc) + if err != nil { + return fmt.Errorf("AddVertex Error %s", err) + } + + _, ok = ggraph.jsonkv.LocCache.Set(vertex.ID, rowLoc) if !ok { - ggraph.jsonkv.PageCache.Invalidate(vertex.ID) - ggraph.jsonkv.PageCache.Set(vertex.ID, *rowLoc) - //log.Debugln("Replaced vals: ", vertex.ID, oldVal, newVal) + ggraph.jsonkv.LocCache.Invalidate(vertex.ID) + ggraph.jsonkv.LocCache.Set(vertex.ID, rowLoc) } _, fieldsExist := ggraph.jsonkv.Fields[vertexLabel] if fieldsExist { for field := range ggraph.jsonkv.Fields[vertexLabel] { - if val := jsontable.PathLookup(vertex.Data, field); val != nil { + if val := tpath.PathLookup(vertex.Data, field); val != nil { err := tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) if err != nil { return err @@ -147,7 +151,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error return fmt.Errorf("indexEdge: jsonkv.New: %s", err) } ggraph.jsonkv.Lock.Lock() - table = newTable.(*jsontable.JSONTable) + table = newTable.(*jTable.JSONTable) ggraph.jsonkv.Tables[edgeLabel] = table ggraph.jsonkv.Lock.Unlock() } @@ -155,18 +159,21 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error if err != nil { return fmt.Errorf("indexEdge: table.AddRow: %s", err) } - table.AddTableEntryInfo(tx, []byte(edge.ID), *rowLoc) + err = ggraph.jsonkv.AddTableEntryInfo(tx, []byte(edge.ID), rowLoc) + if err != nil { + return fmt.Errorf("indexEdge: jsonkv.AddTableEntryInfo: %s", err) + } - _, ok = ggraph.jsonkv.PageCache.Set(edge.ID, *rowLoc) + _, ok = ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) if !ok { - ggraph.jsonkv.PageCache.Invalidate(edge.ID) - ggraph.jsonkv.PageCache.Set(edge.ID, *rowLoc) + ggraph.jsonkv.LocCache.Invalidate(edge.ID) + ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) } _, fieldsExist := ggraph.jsonkv.Fields[edgeLabel] if fieldsExist { for field := range ggraph.jsonkv.Fields[edgeLabel] { - if val := jsontable.PathLookup(edge.Data, field); val != nil { + if val := tpath.PathLookup(edge.Data, field); val != nil { err := tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) if err != nil { return err @@ -192,7 +199,7 @@ func (ggraph *Graph) Compiler() gdbi.Compiler { // AddVertex adds an edge to the graph, if it already exists // in the graph, it is replaced func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { - err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err := ggraph.jsonkv.Pkv.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, vert := range vertices { if err := insertVertex(tx, vert); err != nil { @@ -204,7 +211,7 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { return bulkErr.ErrorOrNil() }) - err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pkv.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, vert := range vertices { if err := ggraph.indexVertex(vert, tx); err != nil { @@ -222,8 +229,8 @@ func (ggraph *Graph) AddVertex(vertices []*gdbi.Vertex) error { // in the graph, it is replaced func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { var err error = nil - err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { - err = ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err = ggraph.jsonkv.Pkv.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for _, edge := range edges { err = insertEdge(tx, edge) if err != nil { @@ -239,7 +246,7 @@ func (ggraph *Graph) AddEdge(edges []*gdbi.Edge) error { if err != nil { return err } - err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pkv.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { var bulkErr *multierror.Error for _, edge := range edges { if err := ggraph.indexEdge(edge, tx); err != nil { @@ -264,7 +271,7 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { go func() { defer wg.Done() - err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err := ggraph.jsonkv.Pkv.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { for elem := range insertStream { if elem.Vertex != nil { if err := insertVertex(tx, elem.Vertex); err != nil { @@ -288,7 +295,7 @@ func (ggraph *Graph) BulkAdd(stream <-chan *gdbi.GraphElement) error { go func() { defer wg.Done() - err := ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err := ggraph.jsonkv.Pkv.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { if err := ggraph.jsonkv.BulkLoad(indexStream, tx); err != nil { return fmt.Errorf("jsonkv bulk load error: %v", err) } @@ -343,7 +350,7 @@ func (ggraph *Graph) DelVertex(id string) error { edgesToDelete := make(map[string]string) var bulkErr *multierror.Error - err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(skeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), skeyPrefix); it.Next() { skey := it.Key() eid, sid, did, label := SrcEdgeKeyParse(skey) @@ -398,20 +405,20 @@ func (ggraph *Graph) DelVertex(id string) error { } } - loc, err := ggraph.jsonkv.PageCache.Get(context.Background(), id, ggraph.jsonkv.PageLoader) + loc, err := ggraph.jsonkv.LocCache.Get(context.Background(), id) if err != nil { return err } - label := ggraph.jsonkv.LabelLookup[loc.Label] + label := ggraph.jsonkv.LabelLookup[loc.TableId] if label == "" { - bulkErr = multierror.Append(bulkErr, fmt.Errorf("Failed to lookup table label %d", loc.Label)) + bulkErr = multierror.Append(bulkErr, fmt.Errorf("Failed to lookup table label %d", loc.TableId)) } if err := ggraph.DeleteAnyRow(id, label, false); err != nil { bulkErr = multierror.Append(bulkErr, err) } - err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pkv.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { if err := tx.DeletePrefix(vid); err != nil { return err } @@ -434,7 +441,7 @@ func (ggraph *Graph) DelVertex(id string) error { func (ggraph *Graph) DelEdge(eid string) error { ekeyPrefix := EdgeKeyPrefix(eid) var ekey []byte - err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { ekey = it.Key() } @@ -454,7 +461,7 @@ func (ggraph *Graph) DelEdge(eid string) error { dkey := DstEdgeKey(sid, did, eid, lbl) var bulkErr *multierror.Error - err = ggraph.jsonkv.Pb.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { + err = ggraph.jsonkv.Pkv.BulkWrite(func(tx *pebblebulk.PebbleBulk) error { if err := tx.Delete(ekey, nil); err != nil { bulkErr = multierror.Append(bulkErr, err) } @@ -484,7 +491,7 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb o := make(chan *gdbi.Edge, 100) go func() { defer close(o) - ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { ePrefix := EdgeListPrefix() for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { select { @@ -495,7 +502,7 @@ func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdb eid, sid, did, label := EdgeKeyParse(it.Key()) e := &gdbi.Edge{ID: eid, Label: label, From: sid, To: did} if loadProp { - entry, err := ggraph.jsonkv.PageCache.Get(ctx, eid, ggraph.jsonkv.PageLoader) + entry, err := ggraph.jsonkv.LocCache.Get(ctx, eid) if err != nil { log.Errorf("GetEdgeList: PageCache.Get( error: %v", err) continue @@ -522,7 +529,7 @@ func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { ekeyPrefix := VertexKey(id) var byteLabel []byte = nil var err error = nil - err = ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err = ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { byteLabel, err = it.Value() } @@ -537,7 +544,7 @@ func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { Label: string(byteLabel), } if loadProp { - entry, err := ggraph.jsonkv.PageCache.Get(context.Background(), id, ggraph.jsonkv.PageLoader) + entry, err := ggraph.jsonkv.LocCache.Get(context.Background(), id) if err != nil { log.Errorf("GetVertex: PageCache.Get( error: %v", err) return nil @@ -564,7 +571,7 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element out := make(chan gdbi.ElementLookup, 100) go func() { defer close(out) - ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for id := range ids { if id.IsSignal() { out <- id @@ -580,7 +587,7 @@ func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.Element } v.Label = string(label) - entry, err := ggraph.jsonkv.PageCache.Get(ctx, id.ID, ggraph.jsonkv.PageLoader) + entry, err := ggraph.jsonkv.LocCache.Get(ctx, id.ID) if err != nil { log.Errorf("GetVertexChannel: PageCache.Get( error: %v", err) continue @@ -618,7 +625,7 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen lookupChan := make(chan lookup, 1000) go func() { defer close(lookupChan) - ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { lookupChan <- lookup{req: req} @@ -655,12 +662,12 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen o <- req.req } else { if req.key != "" { - entry, err := ggraph.jsonkv.PageCache.Get(ctx, req.key, ggraph.jsonkv.PageLoader) + entry, err := ggraph.jsonkv.LocCache.Get(ctx, req.key) if err != nil { log.Errorf("GetOutChannel: PageCache.Get( error: %v", err) continue } - vLabel, ok := ggraph.jsonkv.LabelLookup[entry.Label] + vLabel, ok := ggraph.jsonkv.LabelLookup[entry.TableId] if !ok { log.Errorf("GetOutChannel: Label not a string %s", vLabel) continue @@ -693,7 +700,7 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element o := make(chan gdbi.ElementLookup, 100) go func() { defer close(o) - ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { o <- req @@ -703,13 +710,13 @@ func (ggraph *Graph) GetInChannel(ctx context.Context, reqChan chan gdbi.Element for it.Seek(dkeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), dkeyPrefix); it.Next() { _, sid, _, label := DstEdgeKeyParse(it.Key()) if len(edgeLabels) == 0 || setcmp.ContainsString(edgeLabels, label) { - entry, err := ggraph.jsonkv.PageCache.Get(ctx, sid, ggraph.jsonkv.PageLoader) + entry, err := ggraph.jsonkv.LocCache.Get(ctx, sid) if err != nil { log.Errorf("GetInChannel: PageCache.Get( error: %v", err) continue } - vLabel, ok := ggraph.jsonkv.LabelLookup[entry.Label] + vLabel, ok := ggraph.jsonkv.LabelLookup[entry.TableId] if !ok { log.Errorf("GetInChannel Label lookup failed") continue @@ -749,7 +756,7 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El o := make(chan gdbi.ElementLookup, 100) go func() { defer close(o) - ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { o <- req @@ -766,7 +773,7 @@ func (ggraph *Graph) GetOutEdgeChannel(ctx context.Context, reqChan chan gdbi.El ID: eid, } if load { - entry, err := ggraph.jsonkv.PageCache.Get(ctx, e.ID, ggraph.jsonkv.PageLoader) + entry, err := ggraph.jsonkv.LocCache.Get(ctx, e.ID) if err != nil { log.Errorf("GetOutEdgeChannel: PageCache.Get( error: %v", err) continue @@ -803,7 +810,7 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele o := make(chan gdbi.ElementLookup, 100) go func() { defer close(o) - ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { o <- req @@ -820,7 +827,7 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele Label: label, } if load { - entry, err := ggraph.jsonkv.PageCache.Get(ctx, e.ID, ggraph.jsonkv.PageLoader) + entry, err := ggraph.jsonkv.LocCache.Get(ctx, e.ID) if err != nil { log.Errorf("GetInEdgeChannel: PageCache.Get( error: %v", err) continue @@ -859,7 +866,7 @@ func (ggraph *Graph) GetInEdgeChannel(ctx context.Context, reqChan chan gdbi.Ele func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { ekeyPrefix := EdgeKeyPrefix(id) var e *gdbi.Edge - err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for it.Seek(ekeyPrefix); it.Valid() && bytes.HasPrefix(it.Key(), ekeyPrefix); it.Next() { eid, src, dst, label := EdgeKeyParse(it.Key()) e = &gdbi.Edge{ @@ -869,7 +876,7 @@ func (ggraph *Graph) GetEdge(id string, loadProp bool) *gdbi.Edge { Label: label, } if loadProp { - entry, err := ggraph.jsonkv.PageCache.Get(context.Background(), e.ID, ggraph.jsonkv.PageLoader) + entry, err := ggraph.jsonkv.LocCache.Get(context.Background(), e.ID) if err != nil { log.Errorf("GetEdge: PageCache.Get( error: %v", err) continue @@ -898,7 +905,7 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g o := make(chan *gdbi.Vertex, 100) go func() { defer close(o) - ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { vPrefix := VertexListPrefix() for it.Seek(vPrefix); it.Valid() && bytes.HasPrefix(it.Key(), vPrefix); it.Next() { select { @@ -915,7 +922,7 @@ func (ggraph *Graph) GetVertexList(ctx context.Context, loadProp bool) <-chan *g Label: string(byteLabel), } if loadProp { - entry, err := ggraph.jsonkv.PageCache.Get(context.Background(), v.ID, ggraph.jsonkv.PageLoader) + entry, err := ggraph.jsonkv.LocCache.Get(ctx, v.ID) if err != nil { log.Errorf("GetVertexList: PageCache.Get on %s error: %s", v.ID, err) continue @@ -1075,7 +1082,7 @@ func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { i++ // Fetch from page cache - loc, err := ggraph.jsonkv.PageCache.Get(ctx, item.id, ggraph.jsonkv.PageLoader) + loc, err := ggraph.jsonkv.LocCache.Get(ctx, item.id) if err != nil { addErr(err) continue @@ -1092,8 +1099,8 @@ func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { } // Position key - localBatch.posKeys = append(localBatch.posKeys, benchtop.NewPosKey(loc.Label, []byte(item.id))) - ggraph.jsonkv.PageCache.Invalidate(item.id) + localBatch.posKeys = append(localBatch.posKeys, benchtop.NewPosKey(loc.TableId, []byte(item.id))) + ggraph.jsonkv.LocCache.Invalidate(item.id) // Send field infos if fields, exists := ggraph.jsonkv.Fields[item.tbl]; exists { @@ -1138,7 +1145,7 @@ func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { defer prodWG.Done() localBatch := keyBatch{singles: make([][]byte, 0, 256), ranges: make([][2][]byte, 0, 256)} - err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for _, vid := range slice { select { case <-ctx.Done(): @@ -1251,7 +1258,7 @@ func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { defer prodWG.Done() localBatch := keyBatch{singles: make([][]byte, 0, 12), ranges: make([][2][]byte, 0, 12)} - err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for _, eid := range slice { select { case <-ctx.Done(): @@ -1320,7 +1327,7 @@ func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { sort.Slice(allFields, func(i, j int) bool { return bytes.Compare(allFields[i].rKey, allFields[j].rKey) < 0 }) - err := ggraph.jsonkv.Pb.View(func(it *pebblebulk.PebbleIterator) error { + err := ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for _, fi := range allFields { if err := it.Seek(fi.rKey); err != nil { return err @@ -1347,7 +1354,7 @@ func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { // Chunked deletes with Pebble batch chunked := func(singles [][]byte, ranges [][2][]byte, posKeys [][]byte, indexDelKeys [][]byte) error { - batch := ggraph.jsonkv.Pb.Db.NewBatch() + batch := ggraph.jsonkv.Pkv.Db.NewBatch() defer batch.Close() for _, k := range singles { if err := batch.Delete(k, nil); err != nil { diff --git a/grids/index.go b/grids/index.go index 11d92811..390b9aae 100644 --- a/grids/index.go +++ b/grids/index.go @@ -4,6 +4,7 @@ import ( "context" "fmt" + "github.com/bmeg/benchtop" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" "github.com/cockroachdb/pebble" @@ -52,7 +53,7 @@ func (ggraph *Graph) DeleteAnyRow(id string, label string, edgeFlag bool) error prefix = "e_" } - loc, err := ggraph.jsonkv.PageCache.Get(context.Background(), id, ggraph.jsonkv.PageLoader) + loc, err := ggraph.jsonkv.LocCache.Get(context.Background(), id) if err != nil { return err } @@ -77,14 +78,19 @@ func (ggraph *Graph) DeleteAnyRow(id string, label string, edgeFlag bool) error return bulkErr.ErrorOrNil() } - err = table.DeleteRow(loc, []byte(id)) + bId := []byte(id) + err = ggraph.jsonkv.Pkv.Delete(benchtop.NewPosKey(table.TableId, bId), nil) + if err != nil { + return err + } + err = table.DeleteRow(loc, bId) if err != nil { if err == pebble.ErrNotFound { - log.Debugf("Pebble not Found: %s", err) + log.Debugf("Pebble not Found: % s", err) return nil } bulkErr = multierror.Append(bulkErr, err) } - ggraph.jsonkv.PageCache.Invalidate(id) + ggraph.jsonkv.LocCache.Invalidate(id) return bulkErr.ErrorOrNil() } diff --git a/gripql/python/setup.py b/gripql/python/setup.py index c949a129..6d673447 100644 --- a/gripql/python/setup.py +++ b/gripql/python/setup.py @@ -51,6 +51,7 @@ def find_version(*file_paths): 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7' + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.12' ], ) diff --git a/test/processors_test.go b/test/processors_test.go index c6611421..b893a023 100644 --- a/test/processors_test.go +++ b/test/processors_test.go @@ -258,15 +258,15 @@ func TestEngine(t *testing.T) { count(15), }, { - Q.V().HasLabel("products").Has(gripql.Inside("price", []interface{}{9.99, 19.99})).Count(), + Q.V().HasLabel("products").Has(gripql.Inside("price", []any{9.99, 19.99})).Count(), count(5), }, { - Q.V().HasLabel("products").Has(gripql.Between("price", []interface{}{9.99, 19.99})).Count(), + Q.V().HasLabel("products").Has(gripql.Between("price", []any{9.99, 19.99})).Count(), count(11), }, { - Q.V().HasLabel("products").Has(gripql.Outside("price", []interface{}{9.99, 19.99})).Count(), + Q.V().HasLabel("products").Has(gripql.Outside("price", []any{9.99, 19.99})).Count(), count(9), }, { @@ -356,8 +356,8 @@ func TestEngine(t *testing.T) { }, { Q.V("users:1").As("a").Out().As("b"). - Render(map[string]interface{}{"user_id": "$a._id", "purchase_id": "$b._id", "purchaser": "$b.name"}), - render(map[string]interface{}{"user_id": "users:1", "purchase_id": "purchases:57", "purchaser": "Letitia Sprau"}), + Render(map[string]any{"user_id": "$a._id", "purchase_id": "$b._id", "purchaser": "$b.name"}), + render(map[string]any{"user_id": "users:1", "purchase_id": "purchases:57", "purchaser": "Letitia Sprau"}), }, } @@ -387,7 +387,7 @@ func vertex(id, label string, d data) *gripql.Vertex { } } -func edge(id interface{}, from, to string, label string, d data) *gripql.Edge { +func edge(id any, from, to string, label string, d data) *gripql.Edge { ds, _ := structpb.NewStruct(d) return &gripql.Edge{ Id: fmt.Sprintf("%v", id), @@ -398,7 +398,7 @@ func edge(id interface{}, from, to string, label string, d data) *gripql.Edge { } } -type data map[string]interface{} +type data map[string]any // This sorts the results to account for non-determinstic ordering from the db. // TODO this will break sort tests @@ -476,7 +476,7 @@ func pickid(id string) *gripql.QueryResult { panic("no vertex or edge found for id") } -func pickRes(ival ...interface{}) checker { +func pickRes(ival ...any) checker { expect := []*gripql.QueryResult{} for _, val := range ival { switch v := val.(type) { @@ -530,7 +530,7 @@ func count(i int) checker { return compare(expect) } -func render(v interface{}) checker { +func render(v any) checker { vs, _ := structpb.NewValue(v) expect := []*gripql.QueryResult{ { From 22c7395fe06007fea6d98abe49028bee27793725 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 15 Sep 2025 12:26:04 -0700 Subject: [PATCH 02/12] commit construction code --- go.mod | 6 ++++-- go.sum | 10 ++++------ grids/graph.go | 18 ++++++++++-------- grids/index.go | 5 ++--- grids/new.go | 1 - grids/processor.go | 22 ++++++++++++++-------- 6 files changed, 34 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index a2d80fde..0b51cdca 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.24 toolchain go1.24.2 +replace github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 => ../benchtop + require ( github.com/IBM/sarama v1.45.1 github.com/Shopify/sarama v1.38.1 @@ -17,7 +19,7 @@ require ( github.com/boltdb/bolt v1.3.1 github.com/bytedance/sonic v1.14.0 github.com/casbin/casbin/v2 v2.97.0 - github.com/cockroachdb/pebble v1.1.2 + github.com/cockroachdb/pebble v1.1.5 github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger/v2 v2.2007.4 github.com/dop251/goja v0.0.0-20240707163329-b1681fb2a2f5 @@ -106,7 +108,7 @@ require ( github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jessevdk/go-flags v1.6.1 // indirect - github.com/klauspost/compress v1.17.11 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect diff --git a/go.sum b/go.sum index 6c09f3d7..b69a852c 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,6 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 h1:sIgPwNZKv3pSuIm/hngszbBrg3pKlZcyJ+HTzeFyjNA= -github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9/go.mod h1:Jy39KqCHrPeU9J3SEAdVnZ5dxE6VZm8tX899z5n6ud8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v6 v6.0.4 h1:AXFAz7G05VZkKretSSU+uacMKF8+C16ONG6pzFzzA7E= @@ -75,8 +73,8 @@ github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a h1:f52TdbU4D5nozM github.com/cockroachdb/fifo v0.0.0-20240616162244-4768e80dfb9a/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= -github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/pebble v1.1.5 h1:5AAWCBWbat0uE0blr8qzufZP5tBjkRyy/jWe1QWLnvw= +github.com/cockroachdb/pebble v1.1.5/go.mod h1:17wO9el1YEigxkP/YtV8NtCivQDgoCyBg5c4VR/eOWo= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= @@ -247,8 +245,8 @@ github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeW github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= diff --git a/grids/graph.go b/grids/graph.go index 8c4b12e9..0344a7fd 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -82,9 +82,9 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) ggraph.jsonkv.LocCache.Set(vertex.ID, rowLoc) } - _, fieldsExist := ggraph.jsonkv.Fields[vertexLabel] - if fieldsExist { - for field := range ggraph.jsonkv.Fields[vertexLabel] { + table, tableExists := ggraph.jsonkv.Tables[vertexLabel] + if tableExists && len(table.Fields) > 0 { + for field := range ggraph.jsonkv.Tables[vertexLabel].Fields { if val := tpath.PathLookup(vertex.Data, field); val != nil { err := tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) if err != nil { @@ -170,9 +170,9 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) } - _, fieldsExist := ggraph.jsonkv.Fields[edgeLabel] - if fieldsExist { - for field := range ggraph.jsonkv.Fields[edgeLabel] { + table, tableExists := ggraph.jsonkv.Tables[edgeLabel] + if tableExists && len(table.Fields) > 0 { + for field := range table.Fields { if val := tpath.PathLookup(edge.Data, field); val != nil { err := tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) if err != nil { @@ -1103,8 +1103,10 @@ func (ggraph *Graph) BulkDel(data *gdbi.DeleteData) error { ggraph.jsonkv.LocCache.Invalidate(item.id) // Send field infos - if fields, exists := ggraph.jsonkv.Fields[item.tbl]; exists { - for field := range fields { + // + table, tableExists := ggraph.jsonkv.Tables[item.tbl] + if tableExists && len(table.Fields) > 0 { + for field := range table.Fields { rKey := benchtop.RFieldKey(item.tbl, field, item.id) select { case fieldChan <- fieldInfo{rKey: rKey, field: field, tbl: item.tbl, id: []byte(item.id)}: diff --git a/grids/index.go b/grids/index.go index 390b9aae..5cce426c 100644 --- a/grids/index.go +++ b/grids/index.go @@ -19,7 +19,6 @@ func (ggraph *Graph) AddVertexIndex(label, field string) error { // DeleteVertexIndex delete index from vertices func (ggraph *Graph) DeleteVertexIndex(label, field string) error { - fmt.Println("HELLO WE HARE HERE") log.WithFields(log.Fields{"label": label, "field": field}).Info("Deleting vertex index") return ggraph.jsonkv.RemoveField(VTABLE_PREFIX+label, field) } @@ -60,8 +59,8 @@ func (ggraph *Graph) DeleteAnyRow(id string, label string, edgeFlag bool) error tableLabel := prefix + label var bulkErr *multierror.Error - if fields, exists := ggraph.jsonkv.Fields[tableLabel]; exists { - for field := range fields { + if table, exists := ggraph.jsonkv.Tables[tableLabel]; exists { + for field := range table.Fields { if err := ggraph.jsonkv.DeleteRowField(tableLabel, field, id); err != nil { log.Errorf("Failed to delete index for field '%s' in table '%s' for row '%s': %v", field, tableLabel, id, err) bulkErr = multierror.Append(bulkErr, err) diff --git a/grids/new.go b/grids/new.go index ccc8da7a..7a5b6001 100644 --- a/grids/new.go +++ b/grids/new.go @@ -105,7 +105,6 @@ func getGraph(baseDir, name string) (*Graph, error) { return nil, fmt.Errorf("error reading VERSION file at %s: %v", versionPath, err) } - //bsonkvPath := fmt.Sprintf("%s", dbPath) jsonkvPath := dbPath tabledr, err := jsontable.LoadJSONDriver(jsonkvPath) if err != nil { diff --git a/grids/processor.go b/grids/processor.go index 5baa8cf7..ba6eac44 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -40,19 +40,19 @@ type lookupVertsHasLabelCondIndexProc struct { } func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - log.Debugln("Entering lookupVertsHasLabelCondIndexProc custom processor", l.loadData) var exists = true // Here if one of l.labels doesn't exist then not going to be querying all the data so leave it like this. cond := l.expr.GetCondition() - exists = len(l.db.jsonkv.Fields) > 0 && cond != nil - if exists { + if cond != nil { for _, iterLabel := range l.labels { - label, ok := l.db.jsonkv.Fields[iterLabel] + /*fmt.Println("LABEL: ", iterLabel) + fmt.Println("TABLES: ", l.db.jsonkv.Tables)*/ + tabel, ok := l.db.jsonkv.Tables[iterLabel] if !ok { exists = false break } - _, exists = label[cond.Key] + _, exists = tabel.Fields[cond.Key] if !exists { break } @@ -61,6 +61,7 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi count := 0 if !exists || (l.expr == nil && cond == nil) { + log.Debugln("Using base case processor lookupVertsHasLabelCondIndexProc") go func() { defer close(out) for t := range in { @@ -90,12 +91,17 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi } }() } else { + log.Debugln("Using optimized custom processor lookupVertsHasLabelCondIndexProc") queryChan := make(chan gdbi.ElementLookup, 100) go func() { defer close(queryChan) for t := range in { cond := l.expr.GetCondition() for _, label := range l.labels { + /*fmt.Println("LABEL: ", label) + fmt.Println("CONDITION: ", cond.Condition.String()) + fmt.Println("KEY: ", cond.Key) + fmt.Println("VALUE: ", cond.Value.AsInterface())*/ for id := range l.db.jsonkv.RowIdsByLabelFieldValue(label, cond.Key, cond.Value.AsInterface(), cond.Condition) { queryChan <- gdbi.ElementLookup{ID: id, Ref: t} } @@ -146,11 +152,11 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager /* Indexing only works if every vertex label is indexed for that specific field and it's only a condition Filter otherwise this lookup will not fetch everything that was asked for */ - allMatch := len(l.db.jsonkv.Fields) > 0 && cond != nil + allMatch := cond != nil if allMatch { for lbl := range l.db.jsonkv.GetLabels(false, false) { - if val, exists := l.db.jsonkv.Fields[lbl]; exists { - if _, ok := val[cond.Key]; !ok { + if table, exists := l.db.jsonkv.Tables[lbl]; exists { + if _, ok := table.Fields[cond.Key]; !ok { allMatch = false break } From b8b1b6b140baa4462309507f7e26647b0d4778e9 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 23 Oct 2025 08:03:54 -0700 Subject: [PATCH 03/12] fix optimizer bug --- grids/optimizer.go | 5 ++++- grids/processor.go | 6 ------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/grids/optimizer.go b/grids/optimizer.go index 16912398..371c3010 100644 --- a/grids/optimizer.go +++ b/grids/optimizer.go @@ -63,7 +63,10 @@ var startOptimizations = []OptimizationRule{ if _, ok := pipe[1].GetStatement().(*gripql.GraphStatement_HasLabel); !ok { return false } - if _, ok := pipe[2].GetStatement().(*gripql.GraphStatement_Has); ok { + if stmt, ok := pipe[2].GetStatement().(*gripql.GraphStatement_Has); ok { + if stmt.Has.GetCondition() == nil { + return false + } return true } return false diff --git a/grids/processor.go b/grids/processor.go index ba6eac44..1b66ebd2 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -45,8 +45,6 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi cond := l.expr.GetCondition() if cond != nil { for _, iterLabel := range l.labels { - /*fmt.Println("LABEL: ", iterLabel) - fmt.Println("TABLES: ", l.db.jsonkv.Tables)*/ tabel, ok := l.db.jsonkv.Tables[iterLabel] if !ok { exists = false @@ -98,10 +96,6 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi for t := range in { cond := l.expr.GetCondition() for _, label := range l.labels { - /*fmt.Println("LABEL: ", label) - fmt.Println("CONDITION: ", cond.Condition.String()) - fmt.Println("KEY: ", cond.Key) - fmt.Println("VALUE: ", cond.Value.AsInterface())*/ for id := range l.db.jsonkv.RowIdsByLabelFieldValue(label, cond.Key, cond.Value.AsInterface(), cond.Condition) { queryChan <- gdbi.ElementLookup{ID: id, Ref: t} } From c6987e173040c0e124b623c36ea6ef3e0d9d4373 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 15 Oct 2025 08:21:31 -0700 Subject: [PATCH 04/12] update jsonschemagraph dep to support single edge gen --- go.mod | 6 ++---- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 0b51cdca..6dc97172 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/bmeg/grip -go 1.24 - -toolchain go1.24.2 +go 1.24.2 replace github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 => ../benchtop @@ -15,7 +13,7 @@ require ( github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v6 v6.0.4 - github.com/bmeg/jsonschemagraph v0.0.4-0.20250828230703-257ca9afd85a + github.com/bmeg/jsonschemagraph v0.0.4-0.20251015150525-9ed100499f63 github.com/boltdb/bolt v1.3.1 github.com/bytedance/sonic v1.14.0 github.com/casbin/casbin/v2 v2.97.0 diff --git a/go.sum b/go.sum index b69a852c..9815baaa 100644 --- a/go.sum +++ b/go.sum @@ -39,8 +39,8 @@ github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v6 v6.0.4 h1:AXFAz7G05VZkKretSSU+uacMKF8+C16ONG6pzFzzA7E= github.com/bmeg/jsonschema/v6 v6.0.4/go.mod h1:gTh32doM+BEZyi/TDPJEp8k3qXTckXY4ohptV2xExQY= -github.com/bmeg/jsonschemagraph v0.0.4-0.20250828230703-257ca9afd85a h1:O0JcMLcazrwVzf8iC/RogUen4CG5UVErrBU76UkxhYQ= -github.com/bmeg/jsonschemagraph v0.0.4-0.20250828230703-257ca9afd85a/go.mod h1:rlek2WcKAhnynqE7NJi8U+RDbUkRFr8Kqpb2SDmcW94= +github.com/bmeg/jsonschemagraph v0.0.4-0.20251015150525-9ed100499f63 h1:vXw9uZsI22wSqJsQDnje1fetSv5iQTkXGJKRFJP8+S4= +github.com/bmeg/jsonschemagraph v0.0.4-0.20251015150525-9ed100499f63/go.mod h1:Ve7jAQhYAMkHUiko99+2CwqXI4Ur0ty/ai8Tfa2ONz4= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= From c4857e52bb11fbc7e34100f5997dbfd90bd929ff Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Fri, 17 Oct 2025 13:55:07 -0700 Subject: [PATCH 05/12] update validator to use custom method type validators --- go.mod | 3 ++- go.sum | 10 ++++++---- server/metagraphs.go | 9 +++++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 6dc97172..7110da73 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v6 v6.0.4 - github.com/bmeg/jsonschemagraph v0.0.4-0.20251015150525-9ed100499f63 + github.com/bmeg/jsonschemagraph v0.0.4-0.20251017205345-236d2de9887c github.com/boltdb/bolt v1.3.1 github.com/bytedance/sonic v1.14.0 github.com/casbin/casbin/v2 v2.97.0 @@ -65,6 +65,7 @@ require ( github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bmeg/golib v0.0.0-20200725232156-e799a31439fc // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/casbin/govaluate v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect diff --git a/go.sum b/go.sum index 9815baaa..372d7c13 100644 --- a/go.sum +++ b/go.sum @@ -14,8 +14,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mo github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ= github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e h1:ZIWapoIRN1VqT8GR8jAwb1Ie9GyehWjVcGh32Y2MznE= -github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= +github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/IBM/sarama v1.45.1 h1:nY30XqYpqyXOXSNoe2XCgjj9jklGM1Ye94ierUb1jQ0= github.com/IBM/sarama v1.45.1/go.mod h1:qifDhA3VWSrQ1TjSMyxDl3nYL3oX2C83u+G6L79sq4w= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -35,12 +35,14 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bmeg/benchtop v0.0.0-20251027212658-046a256eb6fa h1:8gqN6aRKHYkAQGXr8bdOquCl6gzn42jl31aUtznYJlY= +github.com/bmeg/benchtop v0.0.0-20251027212658-046a256eb6fa/go.mod h1:mKIXKgNg/q55XrsWKAeWBI9aeSV9yep6tdqaZYHkDcw= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v6 v6.0.4 h1:AXFAz7G05VZkKretSSU+uacMKF8+C16ONG6pzFzzA7E= github.com/bmeg/jsonschema/v6 v6.0.4/go.mod h1:gTh32doM+BEZyi/TDPJEp8k3qXTckXY4ohptV2xExQY= -github.com/bmeg/jsonschemagraph v0.0.4-0.20251015150525-9ed100499f63 h1:vXw9uZsI22wSqJsQDnje1fetSv5iQTkXGJKRFJP8+S4= -github.com/bmeg/jsonschemagraph v0.0.4-0.20251015150525-9ed100499f63/go.mod h1:Ve7jAQhYAMkHUiko99+2CwqXI4Ur0ty/ai8Tfa2ONz4= +github.com/bmeg/jsonschemagraph v0.0.4-0.20251017205345-236d2de9887c h1:J1EhcEmL1D/YHxoMIw4HeeVv+hRMUFezNlAAlFX/a8M= +github.com/bmeg/jsonschemagraph v0.0.4-0.20251017205345-236d2de9887c/go.mod h1:Ve7jAQhYAMkHUiko99+2CwqXI4Ur0ty/ai8Tfa2ONz4= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= diff --git a/server/metagraphs.go b/server/metagraphs.go index 7f56cef7..afd8df11 100644 --- a/server/metagraphs.go +++ b/server/metagraphs.go @@ -163,6 +163,15 @@ func (server *GripServer) addFullGraph(ctx context.Context, graphName string, sc func (server *GripServer) LoadSchemas(sch *gripql.Graph, out *graph.GraphSchema) (*graph.GraphSchema, error) { compiler := jsonschema.NewCompiler() + + compiler.AssertFormat() + compiler.RegisterFormat(&jsonschema.Format{Name: "date-time", Validate: compile.ValidateFhirDateTime}) + compiler.RegisterFormat(&jsonschema.Format{Name: "date", Validate: compile.ValidateFhirDate}) + compiler.RegisterFormat(&jsonschema.Format{Name: "binary", Validate: compile.ValidateFhirBinary}) + compiler.RegisterFormat(&jsonschema.Format{Name: "binary", Validate: compile.ValidateFhirTime}) + compiler.RegisterFormat(&jsonschema.Format{Name: "uuid", Validate: compile.ValidateFhirUUID}) + compiler.RegisterFormat(&jsonschema.Format{Name: "uri", Validate: compile.ValidateFhirURI}) + compiler.AssertVocabs() vc, err := compile.GetHyperMediaVocab() if err != nil { From 3b9c5e5e005185c30da4e30fc0cc859931e07844 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 20 Oct 2025 08:16:42 -0700 Subject: [PATCH 06/12] fix typo --- server/metagraphs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/metagraphs.go b/server/metagraphs.go index afd8df11..e8f0cdf2 100644 --- a/server/metagraphs.go +++ b/server/metagraphs.go @@ -168,7 +168,7 @@ func (server *GripServer) LoadSchemas(sch *gripql.Graph, out *graph.GraphSchema) compiler.RegisterFormat(&jsonschema.Format{Name: "date-time", Validate: compile.ValidateFhirDateTime}) compiler.RegisterFormat(&jsonschema.Format{Name: "date", Validate: compile.ValidateFhirDate}) compiler.RegisterFormat(&jsonschema.Format{Name: "binary", Validate: compile.ValidateFhirBinary}) - compiler.RegisterFormat(&jsonschema.Format{Name: "binary", Validate: compile.ValidateFhirTime}) + compiler.RegisterFormat(&jsonschema.Format{Name: "time", Validate: compile.ValidateFhirTime}) compiler.RegisterFormat(&jsonschema.Format{Name: "uuid", Validate: compile.ValidateFhirUUID}) compiler.RegisterFormat(&jsonschema.Format{Name: "uri", Validate: compile.ValidateFhirURI}) From d83e4f803eff8109af5cad12cecbca81ec11487f Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Wed, 22 Oct 2025 15:17:10 -0700 Subject: [PATCH 07/12] update kafka image tag --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4fbae641..1d05ed53 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ start-kafka: -e KAFKA_CLIENT_PASSWORDS=adminpassword \ -e KAFKA_CFG_SASL_ENABLED_MECHANISMS=PLAIN \ -e KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN \ - bitnami/kafka:latest + bitnamilegacy/kafka:4.0.0-debian-12-r10 printf '%s\n' \ 'security.protocol=SASL_PLAINTEXT' \ 'sasl.mechanism=PLAIN' \ From d2dd809b00e980588592b25c574b23c23fe44f74 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Tue, 9 Sep 2025 16:25:09 -0700 Subject: [PATCH 08/12] update funcs --- grids/graph.go | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/grids/graph.go b/grids/graph.go index 0344a7fd..e14f9711 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -82,9 +82,9 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) ggraph.jsonkv.LocCache.Set(vertex.ID, rowLoc) } - table, tableExists := ggraph.jsonkv.Tables[vertexLabel] - if tableExists && len(table.Fields) > 0 { - for field := range ggraph.jsonkv.Tables[vertexLabel].Fields { + _, fieldsExist := ggraph.jsonkv.Fields[vertexLabel] + if fieldsExist { + for field := range ggraph.jsonkv.Fields[vertexLabel] { if val := tpath.PathLookup(vertex.Data, field); val != nil { err := tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) if err != nil { @@ -162,6 +162,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error err = ggraph.jsonkv.AddTableEntryInfo(tx, []byte(edge.ID), rowLoc) if err != nil { return fmt.Errorf("indexEdge: jsonkv.AddTableEntryInfo: %s", err) +<<<<<<< HEAD } _, ok = ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) @@ -173,6 +174,19 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error table, tableExists := ggraph.jsonkv.Tables[edgeLabel] if tableExists && len(table.Fields) > 0 { for field := range table.Fields { +======= + } + + _, ok = ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) + if !ok { + ggraph.jsonkv.LocCache.Invalidate(edge.ID) + ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) + } + + _, fieldsExist := ggraph.jsonkv.Fields[edgeLabel] + if fieldsExist { + for field := range ggraph.jsonkv.Fields[edgeLabel] { +>>>>>>> 68f8e438 (update funcs) if val := tpath.PathLookup(edge.Data, field); val != nil { err := tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) if err != nil { From 86c65219d05dc777baa73c94bddefc8674cf0387 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 15 Sep 2025 12:26:04 -0700 Subject: [PATCH 09/12] commit construction code --- go.mod | 3 +-- go.sum | 2 -- grids/graph.go | 12 +++++++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 7110da73..d1d4c551 100644 --- a/go.mod +++ b/go.mod @@ -63,9 +63,8 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 // indirect - github.com/DataDog/zstd v1.5.6-0.20230824185856-869dae002e5e // indirect + github.com/DataDog/zstd v1.5.7 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bmeg/golib v0.0.0-20200725232156-e799a31439fc // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/casbin/govaluate v1.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect diff --git a/go.sum b/go.sum index 372d7c13..1e800efb 100644 --- a/go.sum +++ b/go.sum @@ -35,8 +35,6 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bmeg/benchtop v0.0.0-20251027212658-046a256eb6fa h1:8gqN6aRKHYkAQGXr8bdOquCl6gzn42jl31aUtznYJlY= -github.com/bmeg/benchtop v0.0.0-20251027212658-046a256eb6fa/go.mod h1:mKIXKgNg/q55XrsWKAeWBI9aeSV9yep6tdqaZYHkDcw= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v6 v6.0.4 h1:AXFAz7G05VZkKretSSU+uacMKF8+C16ONG6pzFzzA7E= diff --git a/grids/graph.go b/grids/graph.go index e14f9711..46acf11d 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -82,9 +82,9 @@ func (ggraph *Graph) indexVertex(vertex *gdbi.Vertex, tx *pebblebulk.PebbleBulk) ggraph.jsonkv.LocCache.Set(vertex.ID, rowLoc) } - _, fieldsExist := ggraph.jsonkv.Fields[vertexLabel] - if fieldsExist { - for field := range ggraph.jsonkv.Fields[vertexLabel] { + table, tableExists := ggraph.jsonkv.Tables[vertexLabel] + if tableExists && len(table.Fields) > 0 { + for field := range ggraph.jsonkv.Tables[vertexLabel].Fields { if val := tpath.PathLookup(vertex.Data, field); val != nil { err := tx.Set(benchtop.FieldKey(field, vertexLabel, val, []byte(vertex.ID)), []byte{}, nil) if err != nil { @@ -183,10 +183,16 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) } +<<<<<<< HEAD _, fieldsExist := ggraph.jsonkv.Fields[edgeLabel] if fieldsExist { for field := range ggraph.jsonkv.Fields[edgeLabel] { >>>>>>> 68f8e438 (update funcs) +======= + table, tableExists := ggraph.jsonkv.Tables[edgeLabel] + if tableExists && len(table.Fields) > 0 { + for field := range table.Fields { +>>>>>>> f59d7ce0 (commit construction code) if val := tpath.PathLookup(edge.Data, field); val != nil { err := tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) if err != nil { From add77af8e0caf7b14afbc91434b71f57e9509f01 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 27 Oct 2025 14:35:38 -0700 Subject: [PATCH 10/12] remove E() get edgelist func, upgrade grids --- engine/core/processors.go | 49 ---------- existing-sql/graph.go | 76 +-------------- gdbi/interface.go | 1 - go.mod | 4 +- go.sum | 5 + grids/filters.go | 70 +++++++------- grids/graph.go | 192 +++++++++++++++++++++++--------------- grids/processor.go | 90 +++++++++--------- gripper/graph.go | 41 -------- kvgraph/graph.go | 34 ------- mongo/graph.go | 37 -------- psql/graph.go | 35 ------- sqlite/graph.go | 35 ------- 13 files changed, 208 insertions(+), 461 deletions(-) diff --git a/engine/core/processors.go b/engine/core/processors.go index 9a9cc051..0791421f 100644 --- a/engine/core/processors.go +++ b/engine/core/processors.go @@ -96,55 +96,6 @@ func (l *LookupVertsLabelIndex) Process(ctx context.Context, man gdbi.Manager, i //////////////////////////////////////////////////////////////////////////////// -// LookupEdges starts query by looking up edges -type LookupEdges struct { - db gdbi.GraphInterface - ids []string - loadData bool -} - -// Process runs LookupEdges -func (l *LookupEdges) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { - go func() { - defer close(out) - for t := range in { - if t.IsSignal() { - out <- t - continue - } - if len(l.ids) == 0 { - for v := range l.db.GetEdgeList(ctx, l.loadData) { - out <- t.AddCurrent(&gdbi.DataElement{ - ID: v.ID, - Label: v.Label, - From: v.From, - To: v.To, - Data: v.Data, - Loaded: v.Loaded, - }) - } - } else { - for _, i := range l.ids { - v := l.db.GetEdge(i, l.loadData) - if v != nil { - out <- t.AddCurrent(&gdbi.DataElement{ - ID: v.ID, - Label: v.Label, - From: v.From, - To: v.To, - Data: v.Data, - Loaded: v.Loaded, - }) - } - } - } - } - }() - return ctx -} - -//////////////////////////////////////////////////////////////////////////////// - // Fields selects fields from current element type Fields struct { keys []string diff --git a/existing-sql/graph.go b/existing-sql/graph.go index cc4994cd..d9feb197 100644 --- a/existing-sql/graph.go +++ b/existing-sql/graph.go @@ -214,80 +214,6 @@ func (g *Graph) VertexLabelScan(ctx context.Context, label string) chan string { return o } -// GetEdgeList produces a channel of all edges in the graph -func (g *Graph) GetEdgeList(ctx context.Context, load bool) <-chan *gdbi.Edge { - o := make(chan *gdbi.Edge, 100) - go func() { - defer close(o) - for _, edgeSchema := range g.schema.Edges { - q := "" - switch edgeSchema.Table { - case "": - q = fmt.Sprintf("SELECT %s.%s, %s.%s FROM %s INNER JOIN %s ON %s.%s=%s.%s", - // SELECT - edgeSchema.From.DestTable, g.schema.GetVertexGid(edgeSchema.From.DestTable), - edgeSchema.To.DestTable, g.schema.GetVertexGid(edgeSchema.To.DestTable), - // FROM - edgeSchema.From.DestTable, - // INNER JOIN - edgeSchema.To.DestTable, - // ON - edgeSchema.From.DestTable, edgeSchema.From.DestField, - edgeSchema.To.DestTable, edgeSchema.To.DestField, - ) - rows, err := g.db.QueryxContext(ctx, q) - if err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: QueryxContext") - return - } - defer rows.Close() - for rows.Next() { - var fromGid, toGid string - if err := rows.Scan(&fromGid, &toGid); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: Scan") - return - } - geid := &generatedEdgeID{edgeSchema.Label, edgeSchema.From.DestTable, fromGid, edgeSchema.To.DestTable, toGid} - edge := geid.Edge() - o <- gdbi.NewElementFromEdge(edge) - } - if err := rows.Err(); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: iterating") - return - } - - default: - q = fmt.Sprintf("SELECT * FROM %s", edgeSchema.Table) - rows, err := g.db.QueryxContext(ctx, q) - if err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: QueryxContext") - return - } - types, err := columnTypeMap(rows) - if err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: columnTypeMap") - return - } - - defer rows.Close() - for rows.Next() { - data := make(map[string]interface{}) - if err := rows.MapScan(data); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: MapScan") - return - } - o <- gdbi.NewElementFromEdge(rowDataToEdge(edgeSchema, data, types, load)) - } - if err := rows.Err(); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: iterating") - return - } - } - } - }() - return o -} - // GetVertexChannel is passed a channel of vertex ids and it produces a channel // of vertices func (g *Graph) GetVertexChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool) chan gdbi.ElementLookup { @@ -331,7 +257,7 @@ func (g *Graph) GetVertexChannel(ctx context.Context, reqChan chan gdbi.ElementL } defer rows.Close() for rows.Next() { - data := make(map[string]interface{}) + data := make(map[string]any) if err := rows.MapScan(data); err != nil { log.WithFields(log.Fields{"error": err}).Error("GetVertexChannel: MapScan") return diff --git a/gdbi/interface.go b/gdbi/interface.go index 4de3e1b6..1d37686e 100644 --- a/gdbi/interface.go +++ b/gdbi/interface.go @@ -178,7 +178,6 @@ type GraphInterface interface { GetVertexIndexList() <-chan *gripql.IndexID GetVertexList(ctx context.Context, load bool) <-chan *Vertex - GetEdgeList(ctx context.Context, load bool) <-chan *Edge GetVertexChannel(ctx context.Context, req chan ElementLookup, load bool) chan ElementLookup GetOutChannel(ctx context.Context, req chan ElementLookup, load bool, emitNull bool, edgeLabels []string) chan ElementLookup diff --git a/go.mod b/go.mod index d1d4c551..807fa9dd 100644 --- a/go.mod +++ b/go.mod @@ -2,15 +2,13 @@ module github.com/bmeg/grip go 1.24.2 -replace github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 => ../benchtop - require ( github.com/IBM/sarama v1.45.1 github.com/Shopify/sarama v1.38.1 github.com/Workiva/go-datastructures v1.1.5 github.com/akrylysov/pogreb v0.10.2 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 - github.com/bmeg/benchtop v0.0.0-20250827195345-9810354883b9 + github.com/bmeg/benchtop v0.0.0-20251027212658-046a256eb6fa github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad github.com/bmeg/jsonschema/v6 v6.0.4 github.com/bmeg/jsonschemagraph v0.0.4-0.20251017205345-236d2de9887c diff --git a/go.sum b/go.sum index 1e800efb..03ff7dfe 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,11 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +<<<<<<< HEAD +======= +github.com/bmeg/benchtop v0.0.0-20251027212658-046a256eb6fa h1:8gqN6aRKHYkAQGXr8bdOquCl6gzn42jl31aUtznYJlY= +github.com/bmeg/benchtop v0.0.0-20251027212658-046a256eb6fa/go.mod h1:mKIXKgNg/q55XrsWKAeWBI9aeSV9yep6tdqaZYHkDcw= +>>>>>>> 63cd95d8 (remove E() get edgelist func, upgrade grids) github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v6 v6.0.4 h1:AXFAz7G05VZkKretSSU+uacMKF8+C16ONG6pzFzzA7E= diff --git a/grids/filters.go b/grids/filters.go index c4c4e32e..b9c4bcb7 100644 --- a/grids/filters.go +++ b/grids/filters.go @@ -3,7 +3,6 @@ package grids import ( bFilters "github.com/bmeg/benchtop/filters" "github.com/bmeg/benchtop/jsontable/table" - "github.com/bmeg/benchtop/jsontable/tpath" "github.com/bmeg/grip/gripql" "github.com/bmeg/grip/log" "github.com/bytedance/sonic" @@ -22,8 +21,8 @@ func (f *GripQLFilter) IsNoOp() bool { return f.Expression == nil } -func (f *GripQLFilter) Matches(row any) bool { - return MatchesHasExpression(row, f.Expression) +func (f *GripQLFilter) Matches(row []byte, tableName string) bool { + return MatchesHasExpression(row, f.Expression, tableName) } func (f *GripQLFilter) RequiredFields() []string { @@ -62,26 +61,19 @@ func extractKeys(expr *gripql.HasExpression) []string { } return out } -func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { + +func MatchesHasExpression(row []byte, stmt *gripql.HasExpression, tableName string) bool { switch stmt.Expression.(type) { case *gripql.HasExpression_Condition: cond := stmt.GetCondition() var lookupVal any - - // Handle lookup based on input type - switch v := val.(type) { - case map[string]any: - lookupVal = tpath.PathLookup(v, cond.Key) - case []byte: - pathArr, err := table.ConvertJSONPathToArray(cond.Key) - if err != nil { - log.Errorf("Error converting JSON path: %v", err) - return false - } - node, err := sonic.Get(v, pathArr...) + if cond.Key == "_label" { + lookupVal = tableName[2:] + } else if cond.Key == "_id" { + node, err := sonic.Get(row, []any{"1"}...) if err != nil { if err != ast.ErrNotExist { - log.Errorf("Sonic Fetch err for path: %s on doc %#v: %v", pathArr, string(v), err) + log.Errorf("Sonic Fetch err for path 1 on doc %#v: %v", string(row), err) } return false } @@ -90,9 +82,26 @@ func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { log.Errorf("Error unmarshaling node: %v", err) return false } - default: - log.Errorf("Unsupported input type: %T", val) - return false + } else { + pathArr, err := table.ConvertJSONPathToArray(cond.Key) + if err != nil { + log.Errorf("Error converting JSON path: %v", err) + return false + } + node, err := sonic.Get(row, pathArr...) + if err != nil { + if err != ast.ErrNotExist { + log.Errorf("Sonic Fetch err for path: %s on doc %#v: %v", pathArr, string(row), err) + return false + } + lookupVal = nil + } else { + lookupVal, err = node.Interface() + if err != nil { + log.Errorf("Error unmarshaling node: %v", err) + return false + } + } } return bFilters.ApplyFilterCondition( @@ -105,34 +114,23 @@ func MatchesHasExpression(val any, stmt *gripql.HasExpression) bool { ) case *gripql.HasExpression_And: - and := stmt.GetAnd() - andRes := []bool{} - for _, e := range and.Expressions { - andRes = append(andRes, MatchesHasExpression(val, e)) - } - for _, r := range andRes { - if !r { + for _, e := range stmt.GetAnd().Expressions { + if !MatchesHasExpression(row, e, tableName) { return false } } return true case *gripql.HasExpression_Or: - or := stmt.GetOr() - orRes := []bool{} - for _, e := range or.Expressions { - orRes = append(orRes, MatchesHasExpression(val, e)) - } - for _, r := range orRes { - if r { + for _, e := range stmt.GetOr().Expressions { + if MatchesHasExpression(row, e, tableName) { return true } } return false case *gripql.HasExpression_Not: - e := stmt.GetNot() - return !MatchesHasExpression(val, e) + return !MatchesHasExpression(row, stmt.GetNot(), tableName) default: log.Errorf("unknown where expression type: %T", stmt.Expression) diff --git a/grids/graph.go b/grids/graph.go index 46acf11d..2cb89c65 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -506,44 +506,6 @@ func (ggraph *Graph) DelEdge(eid string) error { return bulkErr.ErrorOrNil() } -// GetEdgeList produces a channel of all edges in the graph -func (ggraph *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdbi.Edge { - o := make(chan *gdbi.Edge, 100) - go func() { - defer close(o) - ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { - ePrefix := EdgeListPrefix() - for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { - select { - case <-ctx.Done(): - return nil - default: - } - eid, sid, did, label := EdgeKeyParse(it.Key()) - e := &gdbi.Edge{ID: eid, Label: label, From: sid, To: did} - if loadProp { - entry, err := ggraph.jsonkv.LocCache.Get(ctx, eid) - if err != nil { - log.Errorf("GetEdgeList: PageCache.Get( error: %v", err) - continue - } - e.Data, err = ggraph.jsonkv.Tables[ETABLE_PREFIX+label].GetRow(entry) - if err != nil { - log.Errorf("GetEdgeList: GetRow error: %v", err) - continue - } - e.Loaded = true - } else { - e.Data = map[string]any{} - } - o <- e - } - return nil - }) - }() - return o -} - // GetVertex loads a vertex given an id. It returns a nil if not found func (ggraph *Graph) GetVertex(id string, loadProp bool) *gdbi.Vertex { ekeyPrefix := VertexKey(id) @@ -587,53 +549,128 @@ type elementData struct { data []byte } +type idEntry struct { + lookup gdbi.ElementLookup + loc *benchtop.RowLoc +} + func (ggraph *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.ElementLookup, load bool) chan gdbi.ElementLookup { out := make(chan gdbi.ElementLookup, 100) go func() { defer close(out) - ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { + if !load { for id := range ids { if id.IsSignal() { out <- id } else { - if load { - prefix := VertexKey(id.ID) - v := gdbi.Vertex{ID: id.ID} - for it.Seek(prefix); it.Valid() && bytes.HasPrefix(it.Key(), prefix); it.Next() { - label, err := it.Value() - if err != nil { - log.Errorln("GetVertexChannel it.Value() err: ", err) - continue - } - v.Label = string(label) - - entry, err := ggraph.jsonkv.LocCache.Get(ctx, id.ID) - if err != nil { - log.Errorf("GetVertexChannel: PageCache.Get( error: %v", err) - continue - } - v.Data, err = ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry) - if err != nil { - log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", id.ID, err) - continue - } - v.Loaded = true - } - id.Vertex = &v - out <- id - - } else { - id.Vertex = &gdbi.Vertex{ID: id.ID} - out <- id - } + id.Vertex = &gdbi.Vertex{ID: id.ID} + out <- id } } - return nil - }) + return + } + var batch []idEntry + for id := range ids { + if id.IsSignal() { + out <- id + continue + } + entry, err := ggraph.jsonkv.LocCache.Get(ctx, id.ID) + if err != nil { + log.Errorf("GetVertexChannel: PageCache.Get error: %v", err) + continue + } + batch = append(batch, idEntry{lookup: id, loc: entry}) + if len(batch) >= 1000 { + processBatchWithLabelCache(ggraph, batch, out) + batch = nil + } + } + if len(batch) > 0 { + processBatchWithLabelCache(ggraph, batch, out) + } }() return out } +type groupKey struct { + TableId uint16 + Section uint16 +} + +func processBatchWithLabelCache(ggraph *Graph, batch []idEntry, out chan gdbi.ElementLookup) { + var wg sync.WaitGroup + sem := make(chan struct{}, 10) + byKey := make(map[groupKey][]idEntry) + for _, entry := range batch { + key := groupKey{TableId: entry.loc.TableId, Section: entry.loc.Section} + byKey[key] = append(byKey[key], entry) + } + for key, entries := range byKey { + wg.Add(1) + go func(key groupKey, entries []idEntry) { + sem <- struct{}{} + defer func() { <-sem; wg.Done() }() + locs := make([]*benchtop.RowLoc, len(entries)) + for i, entry := range entries { + locs[i] = entry.loc + } + Tlabel := ggraph.jsonkv.LabelLookup[key.TableId] + results, errors := ggraph.jsonkv.Tables[VTABLE_PREFIX+Tlabel].GetRows(locs, key.Section) + for i, entry := range entries { + if errors[i] != nil { + log.Errorf("GetVertexChannel: GetRows error for ID %s: %v", entry.lookup.ID, errors[i]) + continue + } + entry.lookup.Vertex = &gdbi.Vertex{ + Data: results[i], + Label: Tlabel, + Loaded: true, + ID: entries[i].lookup.ID, + } + out <- entry.lookup + } + }(key, entries) + } + wg.Wait() +} + +/* +func processBatchWithLabelCache(ggraph *Graph, batch []idEntry, out chan gdbi.ElementLookup) { + var wg sync.WaitGroup + sem := make(chan struct{}, 10) + bySection := make(map[uint16][]idEntry) + for _, entry := range batch { + bySection[entry.loc.Section] = append(bySection[entry.loc.Section], entry) + } + for sectionID, entries := range bySection { + wg.Add(1) + go func(sectionID uint16, entries []idEntry) { + sem <- struct{}{} + defer func() { <-sem; wg.Done() }() + sort.Slice(entries, func(i, j int) bool { + return entries[i].loc.Offset < entries[j].loc.Offset + }) + for _, entry := range entries { + v := gdbi.Vertex{ + ID: entry.lookup.ID, + Label: ggraph.jsonkv.LabelLookup[entry.loc.TableId], + } + data, err := ggraph.jsonkv.Tables[VTABLE_PREFIX+v.Label].GetRow(entry.loc) + if err != nil { + log.Errorf("GetVertexChannel: GetRow error for ID %s: %v", entry.lookup.ID, err) + continue + } + v.Data = data + v.Loaded = true + entry.lookup.Vertex = &v + out <- entry.lookup + } + }(sectionID, entries) + } + wg.Wait() + }*/ + type lookup struct { req gdbi.ElementLookup key string @@ -648,7 +685,12 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen ggraph.jsonkv.Pkv.View(func(it *pebblebulk.PebbleIterator) error { for req := range reqChan { if req.IsSignal() { - lookupChan <- lookup{req: req} + // Use a select statement to send to lookupChan or check for cancellation + select { + case lookupChan <- lookup{req: req}: + case <-ctx.Done(): + return ctx.Err() // Stop if cancelled while trying to send + } } else { found := false skeyPrefix := SrcEdgePrefix(req.ID) @@ -679,7 +721,11 @@ func (ggraph *Graph) GetOutChannel(ctx context.Context, reqChan chan gdbi.Elemen defer close(o) for req := range lookupChan { if req.req.IsSignal() { - o <- req.req + select { + case o <- req.req: + case <-ctx.Done(): + return + } } else { if req.key != "" { entry, err := ggraph.jsonkv.LocCache.Get(ctx, req.key) diff --git a/grids/processor.go b/grids/processor.go index 1b66ebd2..1f8472a5 100644 --- a/grids/processor.go +++ b/grids/processor.go @@ -56,7 +56,6 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi } } } - count := 0 if !exists || (l.expr == nil && cond == nil) { log.Debugln("Using base case processor lookupVertsHasLabelCondIndexProc") @@ -69,21 +68,29 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi log.Debugf("BSONTable for label '%s' is nil. Cannot scan.", label) continue } - for roMaps := range tableFound.Scan(l.loadData, &GripQLFilter{Expression: l.expr}) { - v := gdbi.Vertex{ - Label: label[2:], - Loaded: l.loadData, + if l.loadData { + for roMaps := range tableFound.ScanDoc(&GripQLFilter{Expression: l.expr}) { + v := gdbi.Vertex{ + Label: label[2:], + Loaded: l.loadData, + ID: roMaps["_id"].(string), + } + delete(roMaps, "_id") + v.Data = roMaps + count += 1 + out <- t.AddCurrent(v.Copy()) } - if l.loadData { - v.ID = roMaps.(map[string]any)["_id"].(string) - delete(roMaps.(map[string]any), "_id") - v.Data = roMaps.(map[string]any) - } else { - v.ID = roMaps.(string) - v.Data = map[string]any{} + } else { + for roMaps := range tableFound.ScanId(&GripQLFilter{Expression: l.expr}) { + v := gdbi.Vertex{ + Label: label[2:], + Loaded: l.loadData, + ID: roMaps, + Data: map[string]any{}, + } + count += 1 + out <- t.AddCurrent(v.Copy()) } - count += 1 - out <- t.AddCurrent(v.Copy()) } } } @@ -110,7 +117,6 @@ func (l *lookupVertsHasLabelCondIndexProc) Process(ctx context.Context, man gdbi } }() } - return ctx } @@ -141,11 +147,10 @@ type lookupVertsCondIndexProc struct { func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager, in gdbi.InPipe, out gdbi.OutPipe) context.Context { log.Debugln("Entering lookupVertsCondIndexProc custom processor") - queryChan := make(chan gdbi.ElementLookup, 100) cond := l.expr.GetCondition() - /* Indexing only works if every vertex label is indexed for that specific field and it's only a condition Filter - otherwise this lookup will not fetch everything that was asked for */ + /* Indexing only works if every vertex label is indexed for that specific field and it's only a condition Filter + otherwise this lookup will not fetch everything that was asked for */ allMatch := cond != nil if allMatch { for lbl := range l.db.jsonkv.GetLabels(false, false) { @@ -160,10 +165,12 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager } } } - /* Optimized indexing only works for Simple filters. / - / If compound filter or index doesn't exist use backup method */ + + /* Optimized indexing only works for Simple filters. + If compound filter or index doesn't exist, use backup method */ if cond != nil && allMatch { log.Debugln("Chose index optimized V().Has() statement path") + queryChan := make(chan gdbi.ElementLookup, 100) go func() { defer close(queryChan) for t := range in { @@ -179,36 +186,35 @@ func (l *lookupVertsCondIndexProc) Process(ctx context.Context, man gdbi.Manager } } }() + // Process queryChan with GetVertexChannel for indexed case + go func() { + defer close(out) + for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { + i := v.Ref + out <- i.AddCurrent(v.Vertex.Copy()) + } + }() } else { log.Debugf("Base case GetVertexList is used. No indexing") go func() { - defer close(queryChan) + defer close(out) for t := range in { - for v := range l.db.GetVertexList(ctx, true) { - if MatchesHasExpression( - AddSpecialFields(v), - l.expr, - ) { - queryChan <- gdbi.ElementLookup{ID: v.ID, Ref: t} + for tLabel, table := range l.db.jsonkv.Tables { + if tLabel[:2] == VTABLE_PREFIX { + for v := range table.ScanDoc(&GripQLFilter{Expression: l.expr}) { + vertex := gdbi.Vertex{ + ID: v["_id"].(string), + Label: tLabel[len(VTABLE_PREFIX):], // Extract label from table name + Data: v, // Use full data from ScanDoc + Loaded: l.loadData, // Set Loaded based on l.loadData + } + // Send directly to out channel + out <- t.AddCurrent(vertex.Copy()) + } } - } } }() } - - go func() { - defer close(out) - for v := range l.db.GetVertexChannel(ctx, queryChan, l.loadData) { - i := v.Ref - out <- i.AddCurrent(v.Vertex.Copy()) - } - }() return ctx } - -func AddSpecialFields(v *gdbi.Vertex) any { - v.Data["_label"] = v.Label - v.Data["_id"] = v.ID - return v.Data -} diff --git a/gripper/graph.go b/gripper/graph.go index 801993c9..705e67bf 100644 --- a/gripper/graph.go +++ b/gripper/graph.go @@ -362,47 +362,6 @@ func (t *TabularGraph) GetVertexList(ctx context.Context, load bool) <-chan *gdb return out } -func (t *TabularGraph) GetEdgeList(ctx context.Context, load bool) <-chan *gdbi.Edge { - out := make(chan *gdbi.Edge, 100) - go func() { - log.Infof("Getting edge list") - defer close(out) - for _, source := range t.edgeSourceOrder { - edgeList := t.outEdges[source] - for _, edge := range edgeList { - if ctx.Err() == context.Canceled { - return - } - res := t.client.GetRows(ctx, - edge.config.Data.Source, - edge.config.Data.Collection) - for row := range res { - data := row.Data.AsMap() - if dstStr, err := getFieldString(data, edge.config.Data.ToField); err == nil { - if dstStr != "" { - if srcStr, err := getFieldString(data, edge.config.Data.FromField); err == nil { - if srcStr != "" { - e := gdbi.Edge{ - ID: edge.GenID(srcStr, dstStr), - To: edge.toVertex.prefix + dstStr, - From: edge.fromVertex.prefix + srcStr, - Label: edge.config.Label, - Data: row.Data.AsMap(), - Loaded: true, - } - out <- &e - } - } - } - } - } - } - } - log.Infof("Done with edgelist") - }() - return out -} - func copyPipeline() (chan interface{}, chan interface{}) { in := make(chan interface{}, 10) out := make(chan interface{}, 10) diff --git a/kvgraph/graph.go b/kvgraph/graph.go index 48b11f90..5127b2bc 100644 --- a/kvgraph/graph.go +++ b/kvgraph/graph.go @@ -247,40 +247,6 @@ func (kgdb *KVInterfaceGDB) DelVertex(id string) error { }) } -// GetEdgeList produces a channel of all edges in the graph -func (kgdb *KVInterfaceGDB) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdbi.Edge { - o := make(chan *gdbi.Edge, 100) - go func() { - defer close(o) - kgdb.kvg.kv.View(func(it kvi.KVIterator) error { - ePrefix := EdgeListPrefix(kgdb.graph) - for it.Seek(ePrefix); it.Valid() && bytes.HasPrefix(it.Key(), ePrefix); it.Next() { - select { - case <-ctx.Done(): - return nil - default: - } - keyValue := it.Key() - _, eid, sid, did, label, etype := EdgeKeyParse(keyValue) - if etype == edgeSingle { - if loadProp { - edgeData, _ := it.Value() - ge := &gripql.Edge{} - proto.Unmarshal(edgeData, ge) - e := &gdbi.Edge{ID: ge.Id, Label: ge.Label, From: sid, To: did, Data: ge.Data.AsMap(), Loaded: true} - o <- e - } else { - e := &gdbi.Edge{ID: string(eid), Label: label, From: sid, To: did, Loaded: false} - o <- e - } - } - } - return nil - }) - }() - return o -} - // GetVertex loads a vertex given an id. It returns a nil if not found func (kgdb *KVInterfaceGDB) GetVertex(id string, loadProp bool) *gdbi.Vertex { vkey := VertexKey(kgdb.graph, id) diff --git a/mongo/graph.go b/mongo/graph.go index 870cc7a2..46796295 100644 --- a/mongo/graph.go +++ b/mongo/graph.go @@ -278,43 +278,6 @@ func (mg *Graph) GetVertexList(ctx context.Context, load bool) <-chan *gdbi.Vert return o } -// GetEdgeList produces a channel of all edges in the graph -func (mg *Graph) GetEdgeList(ctx context.Context, loadProp bool) <-chan *gdbi.Edge { - o := make(chan *gdbi.Edge, 100) - - go func() { - defer close(o) - eCol := mg.ar.EdgeCollection(mg.graph) - opts := options.Find() - if !loadProp { - opts.SetProjection(bson.M{FIELD_ID: 1, FIELD_TO: 1, FIELD_FROM: 1, FIELD_LABEL: 1}) - } - query, err := eCol.Find(ctx, bson.M{}, opts) - if err != nil { - return - } - defer query.Close(ctx) - for query.Next(ctx) { - select { - case <-ctx.Done(): - return - default: - } - result := map[string]any{} - if err := query.Decode(&result); err == nil { - if _, ok := result[FIELD_TO]; ok { - e := UnpackEdge(result) - o <- e - } - } else { - log.Errorf("Error decoding edge %#v", result) - } - } - }() - - return o -} - // GetVertexChannel is passed a channel of vertex ids and it produces a channel // of vertices func (mg *Graph) GetVertexChannel(ctx context.Context, ids chan gdbi.ElementLookup, load bool) chan gdbi.ElementLookup { diff --git a/psql/graph.go b/psql/graph.go index a6947c24..6911c8e0 100644 --- a/psql/graph.go +++ b/psql/graph.go @@ -377,41 +377,6 @@ func (g *Graph) VertexLabelScan(ctx context.Context, label string) chan string { return o } -// GetEdgeList produces a channel of all edges in the graph -func (g *Graph) GetEdgeList(ctx context.Context, load bool) <-chan *gdbi.Edge { - o := make(chan *gdbi.Edge, 100) - go func() { - defer close(o) - q := fmt.Sprintf(`SELECT id, label, "from", "to" FROM %s`, g.e) - if load { - q = fmt.Sprintf(`SELECT * FROM %s`, g.e) - } - rows, err := g.db.QueryxContext(ctx, q) - if err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: QueryxContext") - return - } - defer rows.Close() - for rows.Next() { - erow := &Row{} - if err := rows.StructScan(erow); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: StructScan") - continue - } - e, err := ConvertEdgeRow(erow, load) - if err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: convertEdgeRow") - continue - } - o <- e - } - if err := rows.Err(); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: iterating") - } - }() - return o -} - // GetVertexChannel is passed a channel of vertex ids and it produces a channel of vertices func (g *Graph) GetVertexChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool) chan gdbi.ElementLookup { batches := gdbi.LookupBatcher(reqChan, batchSize, time.Microsecond) diff --git a/sqlite/graph.go b/sqlite/graph.go index 0fa117a7..d6ae2024 100644 --- a/sqlite/graph.go +++ b/sqlite/graph.go @@ -368,41 +368,6 @@ func (g *Graph) VertexLabelScan(ctx context.Context, label string) chan string { return o } -// GetEdgeList produces a channel of all edges in the graph -func (g *Graph) GetEdgeList(ctx context.Context, load bool) <-chan *gdbi.Edge { - o := make(chan *gdbi.Edge, 100) - go func() { - defer close(o) - q := fmt.Sprintf(`SELECT id, label, "from", "to" FROM %s`, g.e) - if load { - q = fmt.Sprintf(`SELECT * FROM %s`, g.e) - } - rows, err := g.db.QueryxContext(ctx, q) - if err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: QueryxContext") - return - } - defer rows.Close() - for rows.Next() { - erow := &psql.Row{} - if err := rows.StructScan(erow); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: StructScan") - continue - } - e, err := psql.ConvertEdgeRow(erow, load) - if err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: convertEdgeRow") - continue - } - o <- e - } - if err := rows.Err(); err != nil { - log.WithFields(log.Fields{"error": err}).Error("GetEdgeList: iterating") - } - }() - return o -} - // GetVertexChannel is passed a channel of vertex ids and it produces a channel of vertices func (g *Graph) GetVertexChannel(ctx context.Context, reqChan chan gdbi.ElementLookup, load bool) chan gdbi.ElementLookup { batches := gdbi.LookupBatcher(reqChan, batchSize, time.Microsecond) From a63eab29f577901b0a1f8832866ef7aca31a4044 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Mon, 27 Oct 2025 14:52:06 -0700 Subject: [PATCH 11/12] fix rebase --- go.sum | 3 --- grids/graph.go | 19 ------------------- 2 files changed, 22 deletions(-) diff --git a/go.sum b/go.sum index 03ff7dfe..372d7c13 100644 --- a/go.sum +++ b/go.sum @@ -35,11 +35,8 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -<<<<<<< HEAD -======= github.com/bmeg/benchtop v0.0.0-20251027212658-046a256eb6fa h1:8gqN6aRKHYkAQGXr8bdOquCl6gzn42jl31aUtznYJlY= github.com/bmeg/benchtop v0.0.0-20251027212658-046a256eb6fa/go.mod h1:mKIXKgNg/q55XrsWKAeWBI9aeSV9yep6tdqaZYHkDcw= ->>>>>>> 63cd95d8 (remove E() get edgelist func, upgrade grids) github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad h1:ICgBexeLB7iv/IQz4rsP+MimOXFZUwWSPojEypuOaQ8= github.com/bmeg/jsonpath v0.0.0-20210207014051-cca5355553ad/go.mod h1:ft96Irkp72C7ZrUWRenG7LrF0NKMxXdRvsypo5Njhm4= github.com/bmeg/jsonschema/v6 v6.0.4 h1:AXFAz7G05VZkKretSSU+uacMKF8+C16ONG6pzFzzA7E= diff --git a/grids/graph.go b/grids/graph.go index 2cb89c65..f8fc6bbb 100644 --- a/grids/graph.go +++ b/grids/graph.go @@ -162,19 +162,7 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error err = ggraph.jsonkv.AddTableEntryInfo(tx, []byte(edge.ID), rowLoc) if err != nil { return fmt.Errorf("indexEdge: jsonkv.AddTableEntryInfo: %s", err) -<<<<<<< HEAD - } - - _, ok = ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) - if !ok { - ggraph.jsonkv.LocCache.Invalidate(edge.ID) - ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) - } - table, tableExists := ggraph.jsonkv.Tables[edgeLabel] - if tableExists && len(table.Fields) > 0 { - for field := range table.Fields { -======= } _, ok = ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) @@ -183,16 +171,9 @@ func (ggraph *Graph) indexEdge(edge *gdbi.Edge, tx *pebblebulk.PebbleBulk) error ggraph.jsonkv.LocCache.Set(edge.ID, rowLoc) } -<<<<<<< HEAD - _, fieldsExist := ggraph.jsonkv.Fields[edgeLabel] - if fieldsExist { - for field := range ggraph.jsonkv.Fields[edgeLabel] { ->>>>>>> 68f8e438 (update funcs) -======= table, tableExists := ggraph.jsonkv.Tables[edgeLabel] if tableExists && len(table.Fields) > 0 { for field := range table.Fields { ->>>>>>> f59d7ce0 (commit construction code) if val := tpath.PathLookup(edge.Data, field); val != nil { err := tx.Set(benchtop.FieldKey(field, edgeLabel, val, []byte(edge.ID)), []byte{}, nil) if err != nil { From 7f0784126fbb06ed5b2d27e22ca5ccaccc58ea53 Mon Sep 17 00:00:00 2001 From: matthewpeterkort Date: Thu, 6 Nov 2025 09:49:49 -0800 Subject: [PATCH 12/12] update google apis submodule to not be neolithic --- googleapis | 2 +- gripper/gripper_grpc.pb.go | 259 ++++++++++++++------ gripql/gripql_grpc.pb.go | 468 +++++++++++++++++++++++++------------ 3 files changed, 517 insertions(+), 212 deletions(-) diff --git a/googleapis b/googleapis index b0c32da8..9fcfbea0 160000 --- a/googleapis +++ b/googleapis @@ -1 +1 @@ -Subproject commit b0c32da88f7502f4562e7d3d10c7ded09d9842ab +Subproject commit 9fcfbea0aa5b50fa22e190faceb073d74504172b diff --git a/gripper/gripper_grpc.pb.go b/gripper/gripper_grpc.pb.go index 18e548f0..c32d806e 100644 --- a/gripper/gripper_grpc.pb.go +++ b/gripper/gripper_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.5.1 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.29.3 // source: gripper.proto @@ -15,8 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.64.0 or later. -const _ = grpc.SupportPackageIsVersion9 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 const ( GRIPSource_GetCollections_FullMethodName = "/gripper.GRIPSource/GetCollections" @@ -31,12 +31,12 @@ const ( // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type GRIPSourceClient interface { - GetCollections(ctx context.Context, in *Empty, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Collection], error) + GetCollections(ctx context.Context, in *Empty, opts ...grpc.CallOption) (GRIPSource_GetCollectionsClient, error) GetCollectionInfo(ctx context.Context, in *Collection, opts ...grpc.CallOption) (*CollectionInfo, error) - GetIDs(ctx context.Context, in *Collection, opts ...grpc.CallOption) (grpc.ServerStreamingClient[RowID], error) - GetRows(ctx context.Context, in *Collection, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Row], error) - GetRowsByID(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[RowRequest, Row], error) - GetRowsByField(ctx context.Context, in *FieldRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Row], error) + GetIDs(ctx context.Context, in *Collection, opts ...grpc.CallOption) (GRIPSource_GetIDsClient, error) + GetRows(ctx context.Context, in *Collection, opts ...grpc.CallOption) (GRIPSource_GetRowsClient, error) + GetRowsByID(ctx context.Context, opts ...grpc.CallOption) (GRIPSource_GetRowsByIDClient, error) + GetRowsByField(ctx context.Context, in *FieldRequest, opts ...grpc.CallOption) (GRIPSource_GetRowsByFieldClient, error) } type gRIPSourceClient struct { @@ -47,13 +47,13 @@ func NewGRIPSourceClient(cc grpc.ClientConnInterface) GRIPSourceClient { return &gRIPSourceClient{cc} } -func (c *gRIPSourceClient) GetCollections(ctx context.Context, in *Empty, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Collection], error) { +func (c *gRIPSourceClient) GetCollections(ctx context.Context, in *Empty, opts ...grpc.CallOption) (GRIPSource_GetCollectionsClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &GRIPSource_ServiceDesc.Streams[0], GRIPSource_GetCollections_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[Empty, Collection]{ClientStream: stream} + x := &gRIPSourceGetCollectionsClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -63,8 +63,22 @@ func (c *gRIPSourceClient) GetCollections(ctx context.Context, in *Empty, opts . return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetCollectionsClient = grpc.ServerStreamingClient[Collection] +type GRIPSource_GetCollectionsClient interface { + Recv() (*Collection, error) + grpc.ClientStream +} + +type gRIPSourceGetCollectionsClient struct { + grpc.ClientStream +} + +func (x *gRIPSourceGetCollectionsClient) Recv() (*Collection, error) { + m := new(Collection) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} func (c *gRIPSourceClient) GetCollectionInfo(ctx context.Context, in *Collection, opts ...grpc.CallOption) (*CollectionInfo, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) @@ -76,13 +90,13 @@ func (c *gRIPSourceClient) GetCollectionInfo(ctx context.Context, in *Collection return out, nil } -func (c *gRIPSourceClient) GetIDs(ctx context.Context, in *Collection, opts ...grpc.CallOption) (grpc.ServerStreamingClient[RowID], error) { +func (c *gRIPSourceClient) GetIDs(ctx context.Context, in *Collection, opts ...grpc.CallOption) (GRIPSource_GetIDsClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &GRIPSource_ServiceDesc.Streams[1], GRIPSource_GetIDs_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[Collection, RowID]{ClientStream: stream} + x := &gRIPSourceGetIDsClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -92,16 +106,30 @@ func (c *gRIPSourceClient) GetIDs(ctx context.Context, in *Collection, opts ...g return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetIDsClient = grpc.ServerStreamingClient[RowID] +type GRIPSource_GetIDsClient interface { + Recv() (*RowID, error) + grpc.ClientStream +} + +type gRIPSourceGetIDsClient struct { + grpc.ClientStream +} -func (c *gRIPSourceClient) GetRows(ctx context.Context, in *Collection, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Row], error) { +func (x *gRIPSourceGetIDsClient) Recv() (*RowID, error) { + m := new(RowID) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *gRIPSourceClient) GetRows(ctx context.Context, in *Collection, opts ...grpc.CallOption) (GRIPSource_GetRowsClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &GRIPSource_ServiceDesc.Streams[2], GRIPSource_GetRows_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[Collection, Row]{ClientStream: stream} + x := &gRIPSourceGetRowsClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -111,29 +139,62 @@ func (c *gRIPSourceClient) GetRows(ctx context.Context, in *Collection, opts ... return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetRowsClient = grpc.ServerStreamingClient[Row] +type GRIPSource_GetRowsClient interface { + Recv() (*Row, error) + grpc.ClientStream +} + +type gRIPSourceGetRowsClient struct { + grpc.ClientStream +} -func (c *gRIPSourceClient) GetRowsByID(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[RowRequest, Row], error) { +func (x *gRIPSourceGetRowsClient) Recv() (*Row, error) { + m := new(Row) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *gRIPSourceClient) GetRowsByID(ctx context.Context, opts ...grpc.CallOption) (GRIPSource_GetRowsByIDClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &GRIPSource_ServiceDesc.Streams[3], GRIPSource_GetRowsByID_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[RowRequest, Row]{ClientStream: stream} + x := &gRIPSourceGetRowsByIDClient{ClientStream: stream} return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetRowsByIDClient = grpc.BidiStreamingClient[RowRequest, Row] +type GRIPSource_GetRowsByIDClient interface { + Send(*RowRequest) error + Recv() (*Row, error) + grpc.ClientStream +} + +type gRIPSourceGetRowsByIDClient struct { + grpc.ClientStream +} + +func (x *gRIPSourceGetRowsByIDClient) Send(m *RowRequest) error { + return x.ClientStream.SendMsg(m) +} -func (c *gRIPSourceClient) GetRowsByField(ctx context.Context, in *FieldRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[Row], error) { +func (x *gRIPSourceGetRowsByIDClient) Recv() (*Row, error) { + m := new(Row) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *gRIPSourceClient) GetRowsByField(ctx context.Context, in *FieldRequest, opts ...grpc.CallOption) (GRIPSource_GetRowsByFieldClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &GRIPSource_ServiceDesc.Streams[4], GRIPSource_GetRowsByField_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[FieldRequest, Row]{ClientStream: stream} + x := &gRIPSourceGetRowsByFieldClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -143,49 +204,59 @@ func (c *gRIPSourceClient) GetRowsByField(ctx context.Context, in *FieldRequest, return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetRowsByFieldClient = grpc.ServerStreamingClient[Row] +type GRIPSource_GetRowsByFieldClient interface { + Recv() (*Row, error) + grpc.ClientStream +} + +type gRIPSourceGetRowsByFieldClient struct { + grpc.ClientStream +} + +func (x *gRIPSourceGetRowsByFieldClient) Recv() (*Row, error) { + m := new(Row) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} // GRIPSourceServer is the server API for GRIPSource service. // All implementations must embed UnimplementedGRIPSourceServer -// for forward compatibility. +// for forward compatibility type GRIPSourceServer interface { - GetCollections(*Empty, grpc.ServerStreamingServer[Collection]) error + GetCollections(*Empty, GRIPSource_GetCollectionsServer) error GetCollectionInfo(context.Context, *Collection) (*CollectionInfo, error) - GetIDs(*Collection, grpc.ServerStreamingServer[RowID]) error - GetRows(*Collection, grpc.ServerStreamingServer[Row]) error - GetRowsByID(grpc.BidiStreamingServer[RowRequest, Row]) error - GetRowsByField(*FieldRequest, grpc.ServerStreamingServer[Row]) error + GetIDs(*Collection, GRIPSource_GetIDsServer) error + GetRows(*Collection, GRIPSource_GetRowsServer) error + GetRowsByID(GRIPSource_GetRowsByIDServer) error + GetRowsByField(*FieldRequest, GRIPSource_GetRowsByFieldServer) error mustEmbedUnimplementedGRIPSourceServer() } -// UnimplementedGRIPSourceServer must be embedded to have -// forward compatible implementations. -// -// NOTE: this should be embedded by value instead of pointer to avoid a nil -// pointer dereference when methods are called. -type UnimplementedGRIPSourceServer struct{} +// UnimplementedGRIPSourceServer must be embedded to have forward compatible implementations. +type UnimplementedGRIPSourceServer struct { +} -func (UnimplementedGRIPSourceServer) GetCollections(*Empty, grpc.ServerStreamingServer[Collection]) error { +func (UnimplementedGRIPSourceServer) GetCollections(*Empty, GRIPSource_GetCollectionsServer) error { return status.Errorf(codes.Unimplemented, "method GetCollections not implemented") } func (UnimplementedGRIPSourceServer) GetCollectionInfo(context.Context, *Collection) (*CollectionInfo, error) { return nil, status.Errorf(codes.Unimplemented, "method GetCollectionInfo not implemented") } -func (UnimplementedGRIPSourceServer) GetIDs(*Collection, grpc.ServerStreamingServer[RowID]) error { +func (UnimplementedGRIPSourceServer) GetIDs(*Collection, GRIPSource_GetIDsServer) error { return status.Errorf(codes.Unimplemented, "method GetIDs not implemented") } -func (UnimplementedGRIPSourceServer) GetRows(*Collection, grpc.ServerStreamingServer[Row]) error { +func (UnimplementedGRIPSourceServer) GetRows(*Collection, GRIPSource_GetRowsServer) error { return status.Errorf(codes.Unimplemented, "method GetRows not implemented") } -func (UnimplementedGRIPSourceServer) GetRowsByID(grpc.BidiStreamingServer[RowRequest, Row]) error { +func (UnimplementedGRIPSourceServer) GetRowsByID(GRIPSource_GetRowsByIDServer) error { return status.Errorf(codes.Unimplemented, "method GetRowsByID not implemented") } -func (UnimplementedGRIPSourceServer) GetRowsByField(*FieldRequest, grpc.ServerStreamingServer[Row]) error { +func (UnimplementedGRIPSourceServer) GetRowsByField(*FieldRequest, GRIPSource_GetRowsByFieldServer) error { return status.Errorf(codes.Unimplemented, "method GetRowsByField not implemented") } func (UnimplementedGRIPSourceServer) mustEmbedUnimplementedGRIPSourceServer() {} -func (UnimplementedGRIPSourceServer) testEmbeddedByValue() {} // UnsafeGRIPSourceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to GRIPSourceServer will @@ -195,13 +266,6 @@ type UnsafeGRIPSourceServer interface { } func RegisterGRIPSourceServer(s grpc.ServiceRegistrar, srv GRIPSourceServer) { - // If the following call pancis, it indicates UnimplementedGRIPSourceServer was - // embedded by pointer and is nil. This will cause panics if an - // unimplemented method is ever invoked, so we test this at initialization - // time to prevent it from happening at runtime later due to I/O. - if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { - t.testEmbeddedByValue() - } s.RegisterService(&GRIPSource_ServiceDesc, srv) } @@ -210,11 +274,21 @@ func _GRIPSource_GetCollections_Handler(srv interface{}, stream grpc.ServerStrea if err := stream.RecvMsg(m); err != nil { return err } - return srv.(GRIPSourceServer).GetCollections(m, &grpc.GenericServerStream[Empty, Collection]{ServerStream: stream}) + return srv.(GRIPSourceServer).GetCollections(m, &gRIPSourceGetCollectionsServer{ServerStream: stream}) +} + +type GRIPSource_GetCollectionsServer interface { + Send(*Collection) error + grpc.ServerStream +} + +type gRIPSourceGetCollectionsServer struct { + grpc.ServerStream } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetCollectionsServer = grpc.ServerStreamingServer[Collection] +func (x *gRIPSourceGetCollectionsServer) Send(m *Collection) error { + return x.ServerStream.SendMsg(m) +} func _GRIPSource_GetCollectionInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(Collection) @@ -239,40 +313,89 @@ func _GRIPSource_GetIDs_Handler(srv interface{}, stream grpc.ServerStream) error if err := stream.RecvMsg(m); err != nil { return err } - return srv.(GRIPSourceServer).GetIDs(m, &grpc.GenericServerStream[Collection, RowID]{ServerStream: stream}) + return srv.(GRIPSourceServer).GetIDs(m, &gRIPSourceGetIDsServer{ServerStream: stream}) } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetIDsServer = grpc.ServerStreamingServer[RowID] +type GRIPSource_GetIDsServer interface { + Send(*RowID) error + grpc.ServerStream +} + +type gRIPSourceGetIDsServer struct { + grpc.ServerStream +} + +func (x *gRIPSourceGetIDsServer) Send(m *RowID) error { + return x.ServerStream.SendMsg(m) +} func _GRIPSource_GetRows_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(Collection) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(GRIPSourceServer).GetRows(m, &grpc.GenericServerStream[Collection, Row]{ServerStream: stream}) + return srv.(GRIPSourceServer).GetRows(m, &gRIPSourceGetRowsServer{ServerStream: stream}) } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetRowsServer = grpc.ServerStreamingServer[Row] +type GRIPSource_GetRowsServer interface { + Send(*Row) error + grpc.ServerStream +} + +type gRIPSourceGetRowsServer struct { + grpc.ServerStream +} + +func (x *gRIPSourceGetRowsServer) Send(m *Row) error { + return x.ServerStream.SendMsg(m) +} func _GRIPSource_GetRowsByID_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(GRIPSourceServer).GetRowsByID(&grpc.GenericServerStream[RowRequest, Row]{ServerStream: stream}) + return srv.(GRIPSourceServer).GetRowsByID(&gRIPSourceGetRowsByIDServer{ServerStream: stream}) } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetRowsByIDServer = grpc.BidiStreamingServer[RowRequest, Row] +type GRIPSource_GetRowsByIDServer interface { + Send(*Row) error + Recv() (*RowRequest, error) + grpc.ServerStream +} + +type gRIPSourceGetRowsByIDServer struct { + grpc.ServerStream +} + +func (x *gRIPSourceGetRowsByIDServer) Send(m *Row) error { + return x.ServerStream.SendMsg(m) +} + +func (x *gRIPSourceGetRowsByIDServer) Recv() (*RowRequest, error) { + m := new(RowRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} func _GRIPSource_GetRowsByField_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(FieldRequest) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(GRIPSourceServer).GetRowsByField(m, &grpc.GenericServerStream[FieldRequest, Row]{ServerStream: stream}) + return srv.(GRIPSourceServer).GetRowsByField(m, &gRIPSourceGetRowsByFieldServer{ServerStream: stream}) +} + +type GRIPSource_GetRowsByFieldServer interface { + Send(*Row) error + grpc.ServerStream +} + +type gRIPSourceGetRowsByFieldServer struct { + grpc.ServerStream } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type GRIPSource_GetRowsByFieldServer = grpc.ServerStreamingServer[Row] +func (x *gRIPSourceGetRowsByFieldServer) Send(m *Row) error { + return x.ServerStream.SendMsg(m) +} // GRIPSource_ServiceDesc is the grpc.ServiceDesc for GRIPSource service. // It's only intended for direct use with grpc.RegisterService, diff --git a/gripql/gripql_grpc.pb.go b/gripql/gripql_grpc.pb.go index 10755301..38b70199 100644 --- a/gripql/gripql_grpc.pb.go +++ b/gripql/gripql_grpc.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.5.1 +// - protoc-gen-go-grpc v1.4.0 // - protoc v5.29.3 // source: gripql.proto @@ -15,8 +15,8 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.64.0 or later. -const _ = grpc.SupportPackageIsVersion9 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 const ( Query_Traversal_FullMethodName = "/gripql.Query/Traversal" @@ -35,7 +35,7 @@ const ( // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type QueryClient interface { - Traversal(ctx context.Context, in *GraphQuery, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResult], error) + Traversal(ctx context.Context, in *GraphQuery, opts ...grpc.CallOption) (Query_TraversalClient, error) GetVertex(ctx context.Context, in *ElementID, opts ...grpc.CallOption) (*Vertex, error) GetEdge(ctx context.Context, in *ElementID, opts ...grpc.CallOption) (*Edge, error) GetTimestamp(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (*Timestamp, error) @@ -44,7 +44,7 @@ type QueryClient interface { ListGraphs(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ListGraphsResponse, error) ListIndices(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (*ListIndicesResponse, error) ListLabels(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (*ListLabelsResponse, error) - ListTables(ctx context.Context, in *Empty, opts ...grpc.CallOption) (grpc.ServerStreamingClient[TableInfo], error) + ListTables(ctx context.Context, in *Empty, opts ...grpc.CallOption) (Query_ListTablesClient, error) } type queryClient struct { @@ -55,13 +55,13 @@ func NewQueryClient(cc grpc.ClientConnInterface) QueryClient { return &queryClient{cc} } -func (c *queryClient) Traversal(ctx context.Context, in *GraphQuery, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResult], error) { +func (c *queryClient) Traversal(ctx context.Context, in *GraphQuery, opts ...grpc.CallOption) (Query_TraversalClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Query_ServiceDesc.Streams[0], Query_Traversal_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[GraphQuery, QueryResult]{ClientStream: stream} + x := &queryTraversalClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -71,8 +71,22 @@ func (c *queryClient) Traversal(ctx context.Context, in *GraphQuery, opts ...grp return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Query_TraversalClient = grpc.ServerStreamingClient[QueryResult] +type Query_TraversalClient interface { + Recv() (*QueryResult, error) + grpc.ClientStream +} + +type queryTraversalClient struct { + grpc.ClientStream +} + +func (x *queryTraversalClient) Recv() (*QueryResult, error) { + m := new(QueryResult) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} func (c *queryClient) GetVertex(ctx context.Context, in *ElementID, opts ...grpc.CallOption) (*Vertex, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) @@ -154,13 +168,13 @@ func (c *queryClient) ListLabels(ctx context.Context, in *GraphID, opts ...grpc. return out, nil } -func (c *queryClient) ListTables(ctx context.Context, in *Empty, opts ...grpc.CallOption) (grpc.ServerStreamingClient[TableInfo], error) { +func (c *queryClient) ListTables(ctx context.Context, in *Empty, opts ...grpc.CallOption) (Query_ListTablesClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Query_ServiceDesc.Streams[1], Query_ListTables_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[Empty, TableInfo]{ClientStream: stream} + x := &queryListTablesClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -170,14 +184,28 @@ func (c *queryClient) ListTables(ctx context.Context, in *Empty, opts ...grpc.Ca return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Query_ListTablesClient = grpc.ServerStreamingClient[TableInfo] +type Query_ListTablesClient interface { + Recv() (*TableInfo, error) + grpc.ClientStream +} + +type queryListTablesClient struct { + grpc.ClientStream +} + +func (x *queryListTablesClient) Recv() (*TableInfo, error) { + m := new(TableInfo) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} // QueryServer is the server API for Query service. // All implementations must embed UnimplementedQueryServer -// for forward compatibility. +// for forward compatibility type QueryServer interface { - Traversal(*GraphQuery, grpc.ServerStreamingServer[QueryResult]) error + Traversal(*GraphQuery, Query_TraversalServer) error GetVertex(context.Context, *ElementID) (*Vertex, error) GetEdge(context.Context, *ElementID) (*Edge, error) GetTimestamp(context.Context, *GraphID) (*Timestamp, error) @@ -186,18 +214,15 @@ type QueryServer interface { ListGraphs(context.Context, *Empty) (*ListGraphsResponse, error) ListIndices(context.Context, *GraphID) (*ListIndicesResponse, error) ListLabels(context.Context, *GraphID) (*ListLabelsResponse, error) - ListTables(*Empty, grpc.ServerStreamingServer[TableInfo]) error + ListTables(*Empty, Query_ListTablesServer) error mustEmbedUnimplementedQueryServer() } -// UnimplementedQueryServer must be embedded to have -// forward compatible implementations. -// -// NOTE: this should be embedded by value instead of pointer to avoid a nil -// pointer dereference when methods are called. -type UnimplementedQueryServer struct{} +// UnimplementedQueryServer must be embedded to have forward compatible implementations. +type UnimplementedQueryServer struct { +} -func (UnimplementedQueryServer) Traversal(*GraphQuery, grpc.ServerStreamingServer[QueryResult]) error { +func (UnimplementedQueryServer) Traversal(*GraphQuery, Query_TraversalServer) error { return status.Errorf(codes.Unimplemented, "method Traversal not implemented") } func (UnimplementedQueryServer) GetVertex(context.Context, *ElementID) (*Vertex, error) { @@ -224,11 +249,10 @@ func (UnimplementedQueryServer) ListIndices(context.Context, *GraphID) (*ListInd func (UnimplementedQueryServer) ListLabels(context.Context, *GraphID) (*ListLabelsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method ListLabels not implemented") } -func (UnimplementedQueryServer) ListTables(*Empty, grpc.ServerStreamingServer[TableInfo]) error { +func (UnimplementedQueryServer) ListTables(*Empty, Query_ListTablesServer) error { return status.Errorf(codes.Unimplemented, "method ListTables not implemented") } func (UnimplementedQueryServer) mustEmbedUnimplementedQueryServer() {} -func (UnimplementedQueryServer) testEmbeddedByValue() {} // UnsafeQueryServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to QueryServer will @@ -238,13 +262,6 @@ type UnsafeQueryServer interface { } func RegisterQueryServer(s grpc.ServiceRegistrar, srv QueryServer) { - // If the following call pancis, it indicates UnimplementedQueryServer was - // embedded by pointer and is nil. This will cause panics if an - // unimplemented method is ever invoked, so we test this at initialization - // time to prevent it from happening at runtime later due to I/O. - if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { - t.testEmbeddedByValue() - } s.RegisterService(&Query_ServiceDesc, srv) } @@ -253,11 +270,21 @@ func _Query_Traversal_Handler(srv interface{}, stream grpc.ServerStream) error { if err := stream.RecvMsg(m); err != nil { return err } - return srv.(QueryServer).Traversal(m, &grpc.GenericServerStream[GraphQuery, QueryResult]{ServerStream: stream}) + return srv.(QueryServer).Traversal(m, &queryTraversalServer{ServerStream: stream}) +} + +type Query_TraversalServer interface { + Send(*QueryResult) error + grpc.ServerStream +} + +type queryTraversalServer struct { + grpc.ServerStream } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Query_TraversalServer = grpc.ServerStreamingServer[QueryResult] +func (x *queryTraversalServer) Send(m *QueryResult) error { + return x.ServerStream.SendMsg(m) +} func _Query_GetVertex_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ElementID) @@ -408,11 +435,21 @@ func _Query_ListTables_Handler(srv interface{}, stream grpc.ServerStream) error if err := stream.RecvMsg(m); err != nil { return err } - return srv.(QueryServer).ListTables(m, &grpc.GenericServerStream[Empty, TableInfo]{ServerStream: stream}) + return srv.(QueryServer).ListTables(m, &queryListTablesServer{ServerStream: stream}) +} + +type Query_ListTablesServer interface { + Send(*TableInfo) error + grpc.ServerStream } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Query_ListTablesServer = grpc.ServerStreamingServer[TableInfo] +type queryListTablesServer struct { + grpc.ServerStream +} + +func (x *queryListTablesServer) Send(m *TableInfo) error { + return x.ServerStream.SendMsg(m) +} // Query_ServiceDesc is the grpc.ServiceDesc for Query service. // It's only intended for direct use with grpc.RegisterService, @@ -484,12 +521,12 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type JobClient interface { Submit(ctx context.Context, in *GraphQuery, opts ...grpc.CallOption) (*QueryJob, error) - ListJobs(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryJob], error) - SearchJobs(ctx context.Context, in *GraphQuery, opts ...grpc.CallOption) (grpc.ServerStreamingClient[JobStatus], error) + ListJobs(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (Job_ListJobsClient, error) + SearchJobs(ctx context.Context, in *GraphQuery, opts ...grpc.CallOption) (Job_SearchJobsClient, error) DeleteJob(ctx context.Context, in *QueryJob, opts ...grpc.CallOption) (*JobStatus, error) GetJob(ctx context.Context, in *QueryJob, opts ...grpc.CallOption) (*JobStatus, error) - ViewJob(ctx context.Context, in *QueryJob, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResult], error) - ResumeJob(ctx context.Context, in *ExtendQuery, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResult], error) + ViewJob(ctx context.Context, in *QueryJob, opts ...grpc.CallOption) (Job_ViewJobClient, error) + ResumeJob(ctx context.Context, in *ExtendQuery, opts ...grpc.CallOption) (Job_ResumeJobClient, error) } type jobClient struct { @@ -510,13 +547,13 @@ func (c *jobClient) Submit(ctx context.Context, in *GraphQuery, opts ...grpc.Cal return out, nil } -func (c *jobClient) ListJobs(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryJob], error) { +func (c *jobClient) ListJobs(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (Job_ListJobsClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Job_ServiceDesc.Streams[0], Job_ListJobs_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[GraphID, QueryJob]{ClientStream: stream} + x := &jobListJobsClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -526,16 +563,30 @@ func (c *jobClient) ListJobs(ctx context.Context, in *GraphID, opts ...grpc.Call return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Job_ListJobsClient = grpc.ServerStreamingClient[QueryJob] +type Job_ListJobsClient interface { + Recv() (*QueryJob, error) + grpc.ClientStream +} + +type jobListJobsClient struct { + grpc.ClientStream +} + +func (x *jobListJobsClient) Recv() (*QueryJob, error) { + m := new(QueryJob) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} -func (c *jobClient) SearchJobs(ctx context.Context, in *GraphQuery, opts ...grpc.CallOption) (grpc.ServerStreamingClient[JobStatus], error) { +func (c *jobClient) SearchJobs(ctx context.Context, in *GraphQuery, opts ...grpc.CallOption) (Job_SearchJobsClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Job_ServiceDesc.Streams[1], Job_SearchJobs_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[GraphQuery, JobStatus]{ClientStream: stream} + x := &jobSearchJobsClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -545,8 +596,22 @@ func (c *jobClient) SearchJobs(ctx context.Context, in *GraphQuery, opts ...grpc return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Job_SearchJobsClient = grpc.ServerStreamingClient[JobStatus] +type Job_SearchJobsClient interface { + Recv() (*JobStatus, error) + grpc.ClientStream +} + +type jobSearchJobsClient struct { + grpc.ClientStream +} + +func (x *jobSearchJobsClient) Recv() (*JobStatus, error) { + m := new(JobStatus) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} func (c *jobClient) DeleteJob(ctx context.Context, in *QueryJob, opts ...grpc.CallOption) (*JobStatus, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) @@ -568,13 +633,13 @@ func (c *jobClient) GetJob(ctx context.Context, in *QueryJob, opts ...grpc.CallO return out, nil } -func (c *jobClient) ViewJob(ctx context.Context, in *QueryJob, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResult], error) { +func (c *jobClient) ViewJob(ctx context.Context, in *QueryJob, opts ...grpc.CallOption) (Job_ViewJobClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Job_ServiceDesc.Streams[2], Job_ViewJob_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[QueryJob, QueryResult]{ClientStream: stream} + x := &jobViewJobClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -584,16 +649,30 @@ func (c *jobClient) ViewJob(ctx context.Context, in *QueryJob, opts ...grpc.Call return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Job_ViewJobClient = grpc.ServerStreamingClient[QueryResult] +type Job_ViewJobClient interface { + Recv() (*QueryResult, error) + grpc.ClientStream +} + +type jobViewJobClient struct { + grpc.ClientStream +} + +func (x *jobViewJobClient) Recv() (*QueryResult, error) { + m := new(QueryResult) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} -func (c *jobClient) ResumeJob(ctx context.Context, in *ExtendQuery, opts ...grpc.CallOption) (grpc.ServerStreamingClient[QueryResult], error) { +func (c *jobClient) ResumeJob(ctx context.Context, in *ExtendQuery, opts ...grpc.CallOption) (Job_ResumeJobClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Job_ServiceDesc.Streams[3], Job_ResumeJob_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[ExtendQuery, QueryResult]{ClientStream: stream} + x := &jobResumeJobClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -603,37 +682,48 @@ func (c *jobClient) ResumeJob(ctx context.Context, in *ExtendQuery, opts ...grpc return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Job_ResumeJobClient = grpc.ServerStreamingClient[QueryResult] +type Job_ResumeJobClient interface { + Recv() (*QueryResult, error) + grpc.ClientStream +} + +type jobResumeJobClient struct { + grpc.ClientStream +} + +func (x *jobResumeJobClient) Recv() (*QueryResult, error) { + m := new(QueryResult) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} // JobServer is the server API for Job service. // All implementations must embed UnimplementedJobServer -// for forward compatibility. +// for forward compatibility type JobServer interface { Submit(context.Context, *GraphQuery) (*QueryJob, error) - ListJobs(*GraphID, grpc.ServerStreamingServer[QueryJob]) error - SearchJobs(*GraphQuery, grpc.ServerStreamingServer[JobStatus]) error + ListJobs(*GraphID, Job_ListJobsServer) error + SearchJobs(*GraphQuery, Job_SearchJobsServer) error DeleteJob(context.Context, *QueryJob) (*JobStatus, error) GetJob(context.Context, *QueryJob) (*JobStatus, error) - ViewJob(*QueryJob, grpc.ServerStreamingServer[QueryResult]) error - ResumeJob(*ExtendQuery, grpc.ServerStreamingServer[QueryResult]) error + ViewJob(*QueryJob, Job_ViewJobServer) error + ResumeJob(*ExtendQuery, Job_ResumeJobServer) error mustEmbedUnimplementedJobServer() } -// UnimplementedJobServer must be embedded to have -// forward compatible implementations. -// -// NOTE: this should be embedded by value instead of pointer to avoid a nil -// pointer dereference when methods are called. -type UnimplementedJobServer struct{} +// UnimplementedJobServer must be embedded to have forward compatible implementations. +type UnimplementedJobServer struct { +} func (UnimplementedJobServer) Submit(context.Context, *GraphQuery) (*QueryJob, error) { return nil, status.Errorf(codes.Unimplemented, "method Submit not implemented") } -func (UnimplementedJobServer) ListJobs(*GraphID, grpc.ServerStreamingServer[QueryJob]) error { +func (UnimplementedJobServer) ListJobs(*GraphID, Job_ListJobsServer) error { return status.Errorf(codes.Unimplemented, "method ListJobs not implemented") } -func (UnimplementedJobServer) SearchJobs(*GraphQuery, grpc.ServerStreamingServer[JobStatus]) error { +func (UnimplementedJobServer) SearchJobs(*GraphQuery, Job_SearchJobsServer) error { return status.Errorf(codes.Unimplemented, "method SearchJobs not implemented") } func (UnimplementedJobServer) DeleteJob(context.Context, *QueryJob) (*JobStatus, error) { @@ -642,14 +732,13 @@ func (UnimplementedJobServer) DeleteJob(context.Context, *QueryJob) (*JobStatus, func (UnimplementedJobServer) GetJob(context.Context, *QueryJob) (*JobStatus, error) { return nil, status.Errorf(codes.Unimplemented, "method GetJob not implemented") } -func (UnimplementedJobServer) ViewJob(*QueryJob, grpc.ServerStreamingServer[QueryResult]) error { +func (UnimplementedJobServer) ViewJob(*QueryJob, Job_ViewJobServer) error { return status.Errorf(codes.Unimplemented, "method ViewJob not implemented") } -func (UnimplementedJobServer) ResumeJob(*ExtendQuery, grpc.ServerStreamingServer[QueryResult]) error { +func (UnimplementedJobServer) ResumeJob(*ExtendQuery, Job_ResumeJobServer) error { return status.Errorf(codes.Unimplemented, "method ResumeJob not implemented") } func (UnimplementedJobServer) mustEmbedUnimplementedJobServer() {} -func (UnimplementedJobServer) testEmbeddedByValue() {} // UnsafeJobServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to JobServer will @@ -659,13 +748,6 @@ type UnsafeJobServer interface { } func RegisterJobServer(s grpc.ServiceRegistrar, srv JobServer) { - // If the following call pancis, it indicates UnimplementedJobServer was - // embedded by pointer and is nil. This will cause panics if an - // unimplemented method is ever invoked, so we test this at initialization - // time to prevent it from happening at runtime later due to I/O. - if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { - t.testEmbeddedByValue() - } s.RegisterService(&Job_ServiceDesc, srv) } @@ -692,22 +774,42 @@ func _Job_ListJobs_Handler(srv interface{}, stream grpc.ServerStream) error { if err := stream.RecvMsg(m); err != nil { return err } - return srv.(JobServer).ListJobs(m, &grpc.GenericServerStream[GraphID, QueryJob]{ServerStream: stream}) + return srv.(JobServer).ListJobs(m, &jobListJobsServer{ServerStream: stream}) +} + +type Job_ListJobsServer interface { + Send(*QueryJob) error + grpc.ServerStream +} + +type jobListJobsServer struct { + grpc.ServerStream } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Job_ListJobsServer = grpc.ServerStreamingServer[QueryJob] +func (x *jobListJobsServer) Send(m *QueryJob) error { + return x.ServerStream.SendMsg(m) +} func _Job_SearchJobs_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(GraphQuery) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(JobServer).SearchJobs(m, &grpc.GenericServerStream[GraphQuery, JobStatus]{ServerStream: stream}) + return srv.(JobServer).SearchJobs(m, &jobSearchJobsServer{ServerStream: stream}) } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Job_SearchJobsServer = grpc.ServerStreamingServer[JobStatus] +type Job_SearchJobsServer interface { + Send(*JobStatus) error + grpc.ServerStream +} + +type jobSearchJobsServer struct { + grpc.ServerStream +} + +func (x *jobSearchJobsServer) Send(m *JobStatus) error { + return x.ServerStream.SendMsg(m) +} func _Job_DeleteJob_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(QueryJob) @@ -750,22 +852,42 @@ func _Job_ViewJob_Handler(srv interface{}, stream grpc.ServerStream) error { if err := stream.RecvMsg(m); err != nil { return err } - return srv.(JobServer).ViewJob(m, &grpc.GenericServerStream[QueryJob, QueryResult]{ServerStream: stream}) + return srv.(JobServer).ViewJob(m, &jobViewJobServer{ServerStream: stream}) +} + +type Job_ViewJobServer interface { + Send(*QueryResult) error + grpc.ServerStream +} + +type jobViewJobServer struct { + grpc.ServerStream } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Job_ViewJobServer = grpc.ServerStreamingServer[QueryResult] +func (x *jobViewJobServer) Send(m *QueryResult) error { + return x.ServerStream.SendMsg(m) +} func _Job_ResumeJob_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(ExtendQuery) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(JobServer).ResumeJob(m, &grpc.GenericServerStream[ExtendQuery, QueryResult]{ServerStream: stream}) + return srv.(JobServer).ResumeJob(m, &jobResumeJobServer{ServerStream: stream}) +} + +type Job_ResumeJobServer interface { + Send(*QueryResult) error + grpc.ServerStream } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Job_ResumeJobServer = grpc.ServerStreamingServer[QueryResult] +type jobResumeJobServer struct { + grpc.ServerStream +} + +func (x *jobResumeJobServer) Send(m *QueryResult) error { + return x.ServerStream.SendMsg(m) +} // Job_ServiceDesc is the grpc.ServiceDesc for Job service. // It's only intended for direct use with grpc.RegisterService, @@ -836,8 +958,8 @@ const ( type EditClient interface { AddVertex(ctx context.Context, in *GraphElement, opts ...grpc.CallOption) (*EditResult, error) AddEdge(ctx context.Context, in *GraphElement, opts ...grpc.CallOption) (*EditResult, error) - BulkAdd(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[GraphElement, BulkEditResult], error) - BulkAddRaw(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[RawJson, BulkJsonEditResult], error) + BulkAdd(ctx context.Context, opts ...grpc.CallOption) (Edit_BulkAddClient, error) + BulkAddRaw(ctx context.Context, opts ...grpc.CallOption) (Edit_BulkAddRawClient, error) AddGraph(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (*EditResult, error) DeleteGraph(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (*EditResult, error) BulkDelete(ctx context.Context, in *DeleteData, opts ...grpc.CallOption) (*EditResult, error) @@ -879,31 +1001,75 @@ func (c *editClient) AddEdge(ctx context.Context, in *GraphElement, opts ...grpc return out, nil } -func (c *editClient) BulkAdd(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[GraphElement, BulkEditResult], error) { +func (c *editClient) BulkAdd(ctx context.Context, opts ...grpc.CallOption) (Edit_BulkAddClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Edit_ServiceDesc.Streams[0], Edit_BulkAdd_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[GraphElement, BulkEditResult]{ClientStream: stream} + x := &editBulkAddClient{ClientStream: stream} return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Edit_BulkAddClient = grpc.ClientStreamingClient[GraphElement, BulkEditResult] +type Edit_BulkAddClient interface { + Send(*GraphElement) error + CloseAndRecv() (*BulkEditResult, error) + grpc.ClientStream +} + +type editBulkAddClient struct { + grpc.ClientStream +} + +func (x *editBulkAddClient) Send(m *GraphElement) error { + return x.ClientStream.SendMsg(m) +} + +func (x *editBulkAddClient) CloseAndRecv() (*BulkEditResult, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(BulkEditResult) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} -func (c *editClient) BulkAddRaw(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[RawJson, BulkJsonEditResult], error) { +func (c *editClient) BulkAddRaw(ctx context.Context, opts ...grpc.CallOption) (Edit_BulkAddRawClient, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Edit_ServiceDesc.Streams[1], Edit_BulkAddRaw_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &grpc.GenericClientStream[RawJson, BulkJsonEditResult]{ClientStream: stream} + x := &editBulkAddRawClient{ClientStream: stream} return x, nil } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Edit_BulkAddRawClient = grpc.ClientStreamingClient[RawJson, BulkJsonEditResult] +type Edit_BulkAddRawClient interface { + Send(*RawJson) error + CloseAndRecv() (*BulkJsonEditResult, error) + grpc.ClientStream +} + +type editBulkAddRawClient struct { + grpc.ClientStream +} + +func (x *editBulkAddRawClient) Send(m *RawJson) error { + return x.ClientStream.SendMsg(m) +} + +func (x *editBulkAddRawClient) CloseAndRecv() (*BulkJsonEditResult, error) { + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + m := new(BulkJsonEditResult) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} func (c *editClient) AddGraph(ctx context.Context, in *GraphID, opts ...grpc.CallOption) (*EditResult, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) @@ -1017,12 +1183,12 @@ func (c *editClient) AddMapping(ctx context.Context, in *Graph, opts ...grpc.Cal // EditServer is the server API for Edit service. // All implementations must embed UnimplementedEditServer -// for forward compatibility. +// for forward compatibility type EditServer interface { AddVertex(context.Context, *GraphElement) (*EditResult, error) AddEdge(context.Context, *GraphElement) (*EditResult, error) - BulkAdd(grpc.ClientStreamingServer[GraphElement, BulkEditResult]) error - BulkAddRaw(grpc.ClientStreamingServer[RawJson, BulkJsonEditResult]) error + BulkAdd(Edit_BulkAddServer) error + BulkAddRaw(Edit_BulkAddRawServer) error AddGraph(context.Context, *GraphID) (*EditResult, error) DeleteGraph(context.Context, *GraphID) (*EditResult, error) BulkDelete(context.Context, *DeleteData) (*EditResult, error) @@ -1037,12 +1203,9 @@ type EditServer interface { mustEmbedUnimplementedEditServer() } -// UnimplementedEditServer must be embedded to have -// forward compatible implementations. -// -// NOTE: this should be embedded by value instead of pointer to avoid a nil -// pointer dereference when methods are called. -type UnimplementedEditServer struct{} +// UnimplementedEditServer must be embedded to have forward compatible implementations. +type UnimplementedEditServer struct { +} func (UnimplementedEditServer) AddVertex(context.Context, *GraphElement) (*EditResult, error) { return nil, status.Errorf(codes.Unimplemented, "method AddVertex not implemented") @@ -1050,10 +1213,10 @@ func (UnimplementedEditServer) AddVertex(context.Context, *GraphElement) (*EditR func (UnimplementedEditServer) AddEdge(context.Context, *GraphElement) (*EditResult, error) { return nil, status.Errorf(codes.Unimplemented, "method AddEdge not implemented") } -func (UnimplementedEditServer) BulkAdd(grpc.ClientStreamingServer[GraphElement, BulkEditResult]) error { +func (UnimplementedEditServer) BulkAdd(Edit_BulkAddServer) error { return status.Errorf(codes.Unimplemented, "method BulkAdd not implemented") } -func (UnimplementedEditServer) BulkAddRaw(grpc.ClientStreamingServer[RawJson, BulkJsonEditResult]) error { +func (UnimplementedEditServer) BulkAddRaw(Edit_BulkAddRawServer) error { return status.Errorf(codes.Unimplemented, "method BulkAddRaw not implemented") } func (UnimplementedEditServer) AddGraph(context.Context, *GraphID) (*EditResult, error) { @@ -1090,7 +1253,6 @@ func (UnimplementedEditServer) AddMapping(context.Context, *Graph) (*EditResult, return nil, status.Errorf(codes.Unimplemented, "method AddMapping not implemented") } func (UnimplementedEditServer) mustEmbedUnimplementedEditServer() {} -func (UnimplementedEditServer) testEmbeddedByValue() {} // UnsafeEditServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to EditServer will @@ -1100,13 +1262,6 @@ type UnsafeEditServer interface { } func RegisterEditServer(s grpc.ServiceRegistrar, srv EditServer) { - // If the following call pancis, it indicates UnimplementedEditServer was - // embedded by pointer and is nil. This will cause panics if an - // unimplemented method is ever invoked, so we test this at initialization - // time to prevent it from happening at runtime later due to I/O. - if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { - t.testEmbeddedByValue() - } s.RegisterService(&Edit_ServiceDesc, srv) } @@ -1147,18 +1302,56 @@ func _Edit_AddEdge_Handler(srv interface{}, ctx context.Context, dec func(interf } func _Edit_BulkAdd_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(EditServer).BulkAdd(&grpc.GenericServerStream[GraphElement, BulkEditResult]{ServerStream: stream}) + return srv.(EditServer).BulkAdd(&editBulkAddServer{ServerStream: stream}) +} + +type Edit_BulkAddServer interface { + SendAndClose(*BulkEditResult) error + Recv() (*GraphElement, error) + grpc.ServerStream } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Edit_BulkAddServer = grpc.ClientStreamingServer[GraphElement, BulkEditResult] +type editBulkAddServer struct { + grpc.ServerStream +} + +func (x *editBulkAddServer) SendAndClose(m *BulkEditResult) error { + return x.ServerStream.SendMsg(m) +} + +func (x *editBulkAddServer) Recv() (*GraphElement, error) { + m := new(GraphElement) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} func _Edit_BulkAddRaw_Handler(srv interface{}, stream grpc.ServerStream) error { - return srv.(EditServer).BulkAddRaw(&grpc.GenericServerStream[RawJson, BulkJsonEditResult]{ServerStream: stream}) + return srv.(EditServer).BulkAddRaw(&editBulkAddRawServer{ServerStream: stream}) +} + +type Edit_BulkAddRawServer interface { + SendAndClose(*BulkJsonEditResult) error + Recv() (*RawJson, error) + grpc.ServerStream +} + +type editBulkAddRawServer struct { + grpc.ServerStream } -// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. -type Edit_BulkAddRawServer = grpc.ClientStreamingServer[RawJson, BulkJsonEditResult] +func (x *editBulkAddRawServer) SendAndClose(m *BulkJsonEditResult) error { + return x.ServerStream.SendMsg(m) +} + +func (x *editBulkAddRawServer) Recv() (*RawJson, error) { + m := new(RawJson) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} func _Edit_AddGraph_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GraphID) @@ -1488,7 +1681,7 @@ func (c *configureClient) ListDrivers(ctx context.Context, in *Empty, opts ...gr // ConfigureServer is the server API for Configure service. // All implementations must embed UnimplementedConfigureServer -// for forward compatibility. +// for forward compatibility type ConfigureServer interface { StartPlugin(context.Context, *PluginConfig) (*PluginStatus, error) ListPlugins(context.Context, *Empty) (*ListPluginsResponse, error) @@ -1496,12 +1689,9 @@ type ConfigureServer interface { mustEmbedUnimplementedConfigureServer() } -// UnimplementedConfigureServer must be embedded to have -// forward compatible implementations. -// -// NOTE: this should be embedded by value instead of pointer to avoid a nil -// pointer dereference when methods are called. -type UnimplementedConfigureServer struct{} +// UnimplementedConfigureServer must be embedded to have forward compatible implementations. +type UnimplementedConfigureServer struct { +} func (UnimplementedConfigureServer) StartPlugin(context.Context, *PluginConfig) (*PluginStatus, error) { return nil, status.Errorf(codes.Unimplemented, "method StartPlugin not implemented") @@ -1513,7 +1703,6 @@ func (UnimplementedConfigureServer) ListDrivers(context.Context, *Empty) (*ListD return nil, status.Errorf(codes.Unimplemented, "method ListDrivers not implemented") } func (UnimplementedConfigureServer) mustEmbedUnimplementedConfigureServer() {} -func (UnimplementedConfigureServer) testEmbeddedByValue() {} // UnsafeConfigureServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to ConfigureServer will @@ -1523,13 +1712,6 @@ type UnsafeConfigureServer interface { } func RegisterConfigureServer(s grpc.ServiceRegistrar, srv ConfigureServer) { - // If the following call pancis, it indicates UnimplementedConfigureServer was - // embedded by pointer and is nil. This will cause panics if an - // unimplemented method is ever invoked, so we test this at initialization - // time to prevent it from happening at runtime later due to I/O. - if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { - t.testEmbeddedByValue() - } s.RegisterService(&Configure_ServiceDesc, srv) }