MB-27666: Nested Fields

document/field_nested.go

@@ -0,0 +1,103 @@
+//  Copyright (c) 2025 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"reflect"
+
+	"github.com/blevesearch/bleve/v2/size"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+var reflectStaticSizeNestedField int
+
+func init() {
+	var f NestedField
+	reflectStaticSizeNestedField = int(reflect.TypeOf(f).Size())
+}
+
+const DefaultNestedIndexingOptions = index.IndexField
+
+type NestedField struct {
+	name              string
+	options           index.FieldIndexingOptions
+	numPlainTextBytes uint64
+
+	nestedDocuments []index.Document
+
+	docAnalyzer index.DocumentAnalyzer
+}
+
+func (s *NestedField) Size() int {
+	return reflectStaticSizeNestedField + size.SizeOfPtr +
+		len(s.name)
+}
+
+func (s *NestedField) Name() string {
+	return s.name
+}
+
+func (s *NestedField) ArrayPositions() []uint64 {
+	return nil
+}
+
+func (s *NestedField) Options() index.FieldIndexingOptions {
+	return s.options
+}
+
+func (s *NestedField) NumPlainTextBytes() uint64 {
+	return s.numPlainTextBytes
+}
+
+func (s *NestedField) AnalyzedLength() int {
+	return 0
+}
+
+func (s *NestedField) EncodedFieldType() byte {
+	return 'e'
+}
+
+func (s *NestedField) AnalyzedTokenFrequencies() index.TokenFrequencies {
+	return nil
+}
+
+func (s *NestedField) Analyze() {
+	for _, doc := range s.nestedDocuments {
+		s.docAnalyzer.Analyze(doc)
+	}
+}
+
+func (s *NestedField) Value() []byte {
+	return nil
+}
+
+func (s *NestedField) NumChildren() int {
+	return len(s.nestedDocuments)
+}
+
+func (s *NestedField) VisitChildren(visitor func(arrayPosition int, document index.Document)) {
+	for i, doc := range s.nestedDocuments {
+		visitor(i, doc)
+	}
+}
+
+func NewNestedField(name string, nestedDocuments []index.Document, docAnalyzer index.DocumentAnalyzer) *NestedField {
+	return &NestedField{
+		name:            name,
+		options:         DefaultNestedIndexingOptions,
+		nestedDocuments: nestedDocuments,
+		docAnalyzer:     docAnalyzer,
+	}
+}

index/scorch/snapshot_index.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/RoaringBitmap/roaring/v2"
 	"github.com/blevesearch/bleve/v2/document"
+	"github.com/blevesearch/bleve/v2/search"
 	index "github.com/blevesearch/bleve_index_api"
 	segment "github.com/blevesearch/scorch_segment_api/v2"
 	"github.com/blevesearch/vellum"
@@ -621,6 +622,12 @@ func (is *IndexSnapshot) TermFieldReader(ctx context.Context, term []byte, field
 	rv.includeTermVectors = includeTermVectors
 	rv.currPosting = nil
 	rv.currID = rv.currID[:0]
+	rv.nestedState = nil
+	if ctx != nil {
+		if nestedState, ok := ctx.Value(search.NestedStateKey).(index.NestedState); ok {
+			rv.nestedState = nestedState
+		}
+	}
 
 	if rv.dicts == nil {
 		rv.dicts = make([]segment.TermDictionary, len(is.segment))
@@ -634,7 +641,13 @@ func (is *IndexSnapshot) TermFieldReader(ctx context.Context, term []byte, field
 				segBytesRead := s.segment.BytesRead()
 				rv.incrementBytesRead(segBytesRead)
 			}
-			dict, err := s.segment.Dictionary(field)
+			var dict segment.TermDictionary
+			var err error
+			if ns, ok := s.segment.(segment.NestedSegment); ok && rv.nestedState != nil {
+				dict, err = ns.NestedDictionary(rv.nestedState, field)
+			} else {
+				dict, err = s.segment.Dictionary(field)
+			}
 			if err != nil {
 				return nil, err
 			}

index/scorch/snapshot_index_tfr.go

@@ -51,6 +51,7 @@ type IndexSnapshotTermFieldReader struct {
 	bytesRead          uint64
 	ctx                context.Context
 	unadorned          bool
+	nestedState        index.NestedState
 }
 
 func (i *IndexSnapshotTermFieldReader) incrementBytesRead(val uint64) {

mapping/document.go

@@ -42,6 +42,7 @@ import (
 type DocumentMapping struct {
 	Enabled              bool                        `json:"enabled"`
 	Dynamic              bool                        `json:"dynamic"`
+	Nested               bool                        `json:"nested,omitempty"`
 	Properties           map[string]*DocumentMapping `json:"properties,omitempty"`
 	Fields               []*FieldMapping             `json:"fields,omitempty"`
 	DefaultAnalyzer      string                      `json:"default_analyzer,omitempty"`
@@ -316,6 +317,11 @@ func (dm *DocumentMapping) UnmarshalJSON(data []byte) error {
 			if err != nil {
 				return err
 			}
+		case "nested":
+			err := util.UnmarshalJSON(v, &dm.Nested)
+			if err != nil {
+				return err
+			}
 		case "default_analyzer":
 			err := util.UnmarshalJSON(v, &dm.DefaultAnalyzer)
 			if err != nil {

mapping/index.go

@@ -196,11 +196,17 @@ func (im *IndexMappingImpl) Validate() error {
 	if err != nil {
 		return err
 	}
-	for _, docMapping := range im.TypeMapping {
+	if im.DefaultMapping.Nested {
+		return fmt.Errorf("default mapping cannot be nested")
+	}
+	for typ, docMapping := range im.TypeMapping {
 		err = docMapping.Validate(im.cache, "", fieldAliasCtx)
 		if err != nil {
 			return err
 		}
+		if docMapping.Nested {
+			return fmt.Errorf("document mapping for type '%s' cannot be nested", typ)
+		}
 	}
 
 	if _, ok := index.SupportedScoringModels[im.ScoringModel]; !ok && im.ScoringModel != "" {

search/query/nested.go

@@ -0,0 +1,108 @@
+//  Copyright (c) 2025 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package query
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/blevesearch/bleve/v2/mapping"
+	"github.com/blevesearch/bleve/v2/search"
+	"github.com/blevesearch/bleve/v2/search/searcher"
+	"github.com/blevesearch/bleve/v2/util"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+type NestedQuery struct {
+	Path       string `json:"path"`
+	InnerQuery Query  `json:"query"`
+}
+
+func NewNestedQuery(path string, innerQuery Query) *NestedQuery {
+	return &NestedQuery{
+		Path:       path,
+		InnerQuery: innerQuery,
+	}
+}
+
+func (q *NestedQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) {
+	nr, ok := i.(index.NestedReader)
+	if !ok {
+		return nil, fmt.Errorf("nested searcher requires an index reader that supports nested documents")
+	}
+	if q.Path == "" || q.InnerQuery == nil {
+		return nil, fmt.Errorf("nested searcher requires a valid path and inner query")
+	}
+	var baseState index.NestedState
+	if existing, ok := ctx.Value(search.NestedStateKey).(index.NestedState); ok {
+		baseState = existing
+	} else {
+		baseState = search.NewNestedState()
+	}
+	childCount := nr.ChildCount(baseState, q.Path)
+	if childCount == 0 {
+		return nil, fmt.Errorf("nested searcher: path %q has no child documents", q.Path)
+	}
+	innerSearchers := make([]search.Searcher, 0, childCount)
+	for arrayPos := 0; arrayPos < childCount; arrayPos++ {
+		newState := baseState.Append(q.Path, arrayPos)
+		nctx := context.WithValue(ctx, search.NestedStateKey, newState)
+		innerSearcher, err := q.InnerQuery.Searcher(nctx, i, m, options)
+		if err != nil {
+			return nil, fmt.Errorf("nested searcher: failed to create inner searcher at pos %d: %w", arrayPos, err)
+		}
+		innerSearchers = append(innerSearchers, innerSearcher)
+	}
+	return searcher.NewDisjunctionSearcher(ctx, i, innerSearchers, 0, options)
+}
+
+func (q *NestedQuery) Validate() error {
+	if q.Path == "" {
+		return fmt.Errorf("nested query must have a path")
+	}
+	if q.InnerQuery == nil {
+		return fmt.Errorf("nested query must have a query")
+	}
+	if vq, ok := q.InnerQuery.(ValidatableQuery); ok {
+		if err := vq.Validate(); err != nil {
+			return fmt.Errorf("nested query must have a valid query: %v", err)
+		}
+	}
+	return nil
+}
+
+func (q *NestedQuery) UnmarshalJSON(data []byte) error {
+	tmp := struct {
+		Path  string          `json:"path"`
+		Query json.RawMessage `json:"query"`
+	}{}
+	err := util.UnmarshalJSON(data, &tmp)
+	if err != nil {
+		return err
+	}
+	if tmp.Path == "" {
+		return fmt.Errorf("nested query must have a path")
+	}
+	if tmp.Query == nil {
+		return fmt.Errorf("nested query must have a query")
+	}
+	q.Path = tmp.Path
+	q.InnerQuery, err = ParseQuery(tmp.Query)
+	if err != nil || q.InnerQuery == nil {
+		return fmt.Errorf("nested query must have a valid query: %v", err)
+	}
+	return nil
+}

search/query/query.go

@@ -353,7 +353,6 @@ func ParseQuery(input []byte) (Query, error) {
 		}
 		return &rv, nil
 	}
-
 	_, hasGeo := tmp["geometry"]
 	if hasGeo {
 		var rv GeoShapeQuery
@@ -363,7 +362,6 @@ func ParseQuery(input []byte) (Query, error) {
 		}
 		return &rv, nil
 	}
-
 	_, hasCIDR := tmp["cidr"]
 	if hasCIDR {
 		var rv IPRangeQuery
@@ -373,6 +371,15 @@ func ParseQuery(input []byte) (Query, error) {
 		}
 		return &rv, nil
 	}
+	_, hasNested := tmp["nested"]
+	if hasNested {
+		var rv NestedQuery
+		err := util.UnmarshalJSON(input, &rv)
+		if err != nil {
+			return nil, err
+		}
+		return &rv, nil
+	}
 
 	return nil, fmt.Errorf("unknown query type")
 }