From d18adf576c3dd9b15c8b7640b615cbdc6e2d0f24 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Thu, 9 Jan 2025 21:33:05 -0800 Subject: [PATCH 01/10] update interface to driver and db --- api.go | 52 +++---- api_mutation_gen.go | 4 +- api_mutation_helpers.go | 14 +- api_query_execution.go | 18 +-- api_test.go | 204 +++++++++++++-------------- api_types.go | 6 +- db.go | 280 +++++++++++++++++-------------------- db_test.go | 296 ++++++++++++++++++++++++++++++--------- driver.go | 230 +++++++++++++++++++++++++++++++ driver_test.go | 131 ++++++++++++++++++ live.go | 8 +- live_benchmark_test.go | 14 +- live_test.go | 4 +- namespace.go | 199 --------------------------- namespace_test.go | 297 ---------------------------------------- vector_test.go | 26 ++-- zero.go | 18 +-- 17 files changed, 901 insertions(+), 900 deletions(-) create mode 100644 driver.go create mode 100644 driver_test.go delete mode 100644 namespace.go delete mode 100644 namespace_test.go diff --git a/api.go b/api.go index b880474..ce04050 100644 --- a/api.go +++ b/api.go @@ -18,18 +18,18 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func Create[T any](db *DB, object T, ns ...uint64) (uint64, T, error) { - db.mutex.Lock() - defer db.mutex.Unlock() +func Create[T any](driver *Driver, object T, ns ...uint64) (uint64, T, error) { + driver.mutex.Lock() + defer driver.mutex.Unlock() if len(ns) > 1 { return 0, object, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(db, ns...) + ctx, n, err := getDefaultNamespace(driver, ns...) if err != nil { return 0, object, err } - gid, err := db.z.nextUID() + gid, err := driver.z.nextUID() if err != nil { return 0, object, err } @@ -46,7 +46,7 @@ func Create[T any](db *DB, object T, ns ...uint64) (uint64, T, error) { return 0, object, err } - err = applyDqlMutations(ctx, db, dms) + err = applyDqlMutations(ctx, driver, dms) if err != nil { return 0, object, err } @@ -54,16 +54,16 @@ func Create[T any](db *DB, object T, ns ...uint64) (uint64, T, error) { return getByGid[T](ctx, n, gid) } -func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { +func Upsert[T any](driver *Driver, object T, ns ...uint64) (uint64, T, bool, error) { var wasFound bool - db.mutex.Lock() - defer db.mutex.Unlock() + driver.mutex.Lock() + defer driver.mutex.Unlock() if len(ns) > 1 { return 0, object, false, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(db, ns...) + ctx, n, err := getDefaultNamespace(driver, ns...) if err != nil { return 0, object, false, err } @@ -101,7 +101,7 @@ func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { } if gid == 0 { - gid, err = db.z.nextUID() + gid, err = driver.z.nextUID() if err != nil { return 0, object, false, err } @@ -113,7 +113,7 @@ func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { return 0, object, false, err } - err = applyDqlMutations(ctx, db, dms) + err = applyDqlMutations(ctx, driver, dms) if err != nil { return 0, object, false, err } @@ -126,14 +126,14 @@ func Upsert[T any](db *DB, object T, ns ...uint64) (uint64, T, bool, error) { return gid, object, wasFound, nil } -func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, T, error) { - db.mutex.Lock() - defer db.mutex.Unlock() +func Get[T any, R UniqueField](driver *Driver, uniqueField R, ns ...uint64) (uint64, T, error) { + driver.mutex.Lock() + defer driver.mutex.Unlock() var obj T if len(ns) > 1 { return 0, obj, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(db, ns...) + ctx, n, err := getDefaultNamespace(driver, ns...) if err != nil { return 0, obj, err } @@ -148,13 +148,13 @@ func Get[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, T, return 0, obj, fmt.Errorf("invalid unique field type") } -func Query[T any](db *DB, queryParams QueryParams, ns ...uint64) ([]uint64, []T, error) { - db.mutex.Lock() - defer db.mutex.Unlock() +func Query[T any](driver *Driver, queryParams QueryParams, ns ...uint64) ([]uint64, []T, error) { + driver.mutex.Lock() + defer driver.mutex.Unlock() if len(ns) > 1 { return nil, nil, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(db, ns...) + ctx, n, err := getDefaultNamespace(driver, ns...) if err != nil { return nil, nil, err } @@ -162,14 +162,14 @@ func Query[T any](db *DB, queryParams QueryParams, ns ...uint64) ([]uint64, []T, return executeQuery[T](ctx, n, queryParams, true) } -func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, T, error) { - db.mutex.Lock() - defer db.mutex.Unlock() +func Delete[T any, R UniqueField](driver *Driver, uniqueField R, ns ...uint64) (uint64, T, error) { + driver.mutex.Lock() + defer driver.mutex.Unlock() var zeroObj T if len(ns) > 1 { return 0, zeroObj, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(db, ns...) + ctx, n, err := getDefaultNamespace(driver, ns...) if err != nil { return 0, zeroObj, err } @@ -181,7 +181,7 @@ func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, dms := generateDeleteDqlMutations(n, uid) - err = applyDqlMutations(ctx, db, dms) + err = applyDqlMutations(ctx, driver, dms) if err != nil { return 0, zeroObj, err } @@ -197,7 +197,7 @@ func Delete[T any, R UniqueField](db *DB, uniqueField R, ns ...uint64) (uint64, dms := generateDeleteDqlMutations(n, uid) - err = applyDqlMutations(ctx, db, dms) + err = applyDqlMutations(ctx, driver, dms) if err != nil { return 0, zeroObj, err } diff --git a/api_mutation_gen.go b/api_mutation_gen.go index d8533fc..c2a304b 100644 --- a/api_mutation_gen.go +++ b/api_mutation_gen.go @@ -26,7 +26,7 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, object T, +func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *DB, object T, gid uint64, dms *[]*dql.Mutation, sch *schema.ParsedSchema) error { t := reflect.TypeOf(object) if t.Kind() != reflect.Struct { @@ -134,7 +134,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, return nil } -func generateDeleteDqlMutations(n *Namespace, gid uint64) []*dql.Mutation { +func generateDeleteDqlMutations(n *DB, gid uint64) []*dql.Mutation { return []*dql.Mutation{{ Del: []*api.NQuad{ { diff --git a/api_mutation_helpers.go b/api_mutation_helpers.go index f205d8a..bcb1bf6 100644 --- a/api_mutation_helpers.go +++ b/api_mutation_helpers.go @@ -14,10 +14,10 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func processStructValue(ctx context.Context, value any, n *Namespace) (any, error) { +func processStructValue(ctx context.Context, value any, n *DB) (any, error) { if reflect.TypeOf(value).Kind() == reflect.Struct { value = reflect.ValueOf(value).Interface() - newGid, err := getUidOrMutate(ctx, n.db, n, value) + newGid, err := getUidOrMutate(ctx, n.driver, n, value) if err != nil { return nil, err } @@ -26,7 +26,7 @@ func processStructValue(ctx context.Context, value any, n *Namespace) (any, erro return value, nil } -func processPointerValue(ctx context.Context, value any, n *Namespace) (any, error) { +func processPointerValue(ctx context.Context, value any, n *DB) (any, error) { reflectValueType := reflect.TypeOf(value) if reflectValueType.Kind() == reflect.Pointer { reflectValueType = reflectValueType.Elem() @@ -38,7 +38,7 @@ func processPointerValue(ctx context.Context, value any, n *Namespace) (any, err return value, nil } -func getUidOrMutate[T any](ctx context.Context, db *DB, n *Namespace, object T) (uint64, error) { +func getUidOrMutate[T any](ctx context.Context, db *Driver, n *DB, object T) (uint64, error) { gid, cfKeyValue, err := structreflect.GetUniqueConstraint[T](object) if err != nil { return 0, err @@ -88,14 +88,14 @@ func getUidOrMutate[T any](ctx context.Context, db *DB, n *Namespace, object T) return gid, nil } -func applyDqlMutations(ctx context.Context, db *DB, dms []*dql.Mutation) error { +func applyDqlMutations(ctx context.Context, db *Driver, dms []*dql.Mutation) error { edges, err := query.ToDirectedEdges(dms, nil) if err != nil { return err } - if !db.isOpen { - return ErrClosedDB + if !db.isOpen.Load() { + return ErrClosedDriver } startTs, err := db.z.nextTs() diff --git a/api_query_execution.go b/api_query_execution.go index d6e0323..2cd8ffb 100644 --- a/api_query_execution.go +++ b/api_query_execution.go @@ -20,25 +20,25 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func getByGid[T any](ctx context.Context, n *Namespace, gid uint64) (uint64, T, error) { +func getByGid[T any](ctx context.Context, n *DB, gid uint64) (uint64, T, error) { return executeGet[T](ctx, n, gid) } -func getByGidWithObject[T any](ctx context.Context, n *Namespace, gid uint64, obj T) (uint64, T, error) { +func getByGidWithObject[T any](ctx context.Context, n *DB, gid uint64, obj T) (uint64, T, error) { return executeGetWithObject[T](ctx, n, obj, false, gid) } -func getByConstrainedField[T any](ctx context.Context, n *Namespace, cf ConstrainedField) (uint64, T, error) { +func getByConstrainedField[T any](ctx context.Context, n *DB, cf ConstrainedField) (uint64, T, error) { return executeGet[T](ctx, n, cf) } -func getByConstrainedFieldWithObject[T any](ctx context.Context, n *Namespace, +func getByConstrainedFieldWithObject[T any](ctx context.Context, n *DB, cf ConstrainedField, obj T) (uint64, T, error) { return executeGetWithObject[T](ctx, n, obj, false, cf) } -func executeGet[T any, R UniqueField](ctx context.Context, n *Namespace, args ...R) (uint64, T, error) { +func executeGet[T any, R UniqueField](ctx context.Context, n *DB, args ...R) (uint64, T, error) { var obj T if len(args) != 1 { return 0, obj, fmt.Errorf("expected 1 argument, got %d", len(args)) @@ -47,7 +47,7 @@ func executeGet[T any, R UniqueField](ctx context.Context, n *Namespace, args .. return executeGetWithObject(ctx, n, obj, true, args...) } -func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespace, +func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *DB, obj T, withReverse bool, args ...R) (uint64, T, error) { t := reflect.TypeOf(obj) @@ -107,7 +107,7 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *Namespac return structreflect.ConvertDynamicToTyped[T](result.Obj[0], t) } -func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryParams, +func executeQuery[T any](ctx context.Context, n *DB, queryParams QueryParams, withReverse bool) ([]uint64, []T, error) { var obj T t := reflect.TypeOf(obj) @@ -186,7 +186,7 @@ func executeQuery[T any](ctx context.Context, n *Namespace, queryParams QueryPar return gids, objs, nil } -func getExistingObject[T any](ctx context.Context, n *Namespace, gid uint64, cf *ConstrainedField, +func getExistingObject[T any](ctx context.Context, n *DB, gid uint64, cf *ConstrainedField, object T) (uint64, error) { var err error if gid != 0 { @@ -200,7 +200,7 @@ func getExistingObject[T any](ctx context.Context, n *Namespace, gid uint64, cf return gid, nil } -func getSchema(ctx context.Context, n *Namespace) (*querygen.SchemaResponse, error) { +func getSchema(ctx context.Context, n *DB) (*querygen.SchemaResponse, error) { resp, err := n.queryWithLock(ctx, querygen.SchemaQuery) if err != nil { return nil, err diff --git a/api_test.go b/api_test.go index 0891069..7213b2d 100644 --- a/api_test.go +++ b/api_test.go @@ -24,15 +24,15 @@ type User struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` Age int `json:"age,omitempty"` - ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" driver:"constraint=unique"` } func TestFirstTimeUser(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - gid, user, err := modusdb.Create(db, User{ + gid, user, err := modusdb.Create(driver, User{ Name: "A", Age: 10, ClerkId: "123", @@ -44,7 +44,7 @@ func TestFirstTimeUser(t *testing.T) { require.Equal(t, 10, user.Age) require.Equal(t, "123", user.ClerkId) - gid, queriedUser, err := modusdb.Get[User](db, gid) + gid, queriedUser, err := modusdb.Get[User](driver, gid) require.NoError(t, err) require.Equal(t, queriedUser.Gid, gid) @@ -52,7 +52,7 @@ func TestFirstTimeUser(t *testing.T) { require.Equal(t, "A", queriedUser.Name) require.Equal(t, "123", queriedUser.ClerkId) - gid, queriedUser2, err := modusdb.Get[User](db, modusdb.ConstrainedField{ + gid, queriedUser2, err := modusdb.Get[User](driver, modusdb.ConstrainedField{ Key: "clerk_id", Value: "123", }) @@ -63,10 +63,10 @@ func TestFirstTimeUser(t *testing.T) { require.Equal(t, "A", queriedUser2.Name) require.Equal(t, "123", queriedUser2.ClerkId) - _, _, err = modusdb.Delete[User](db, gid) + _, _, err = modusdb.Delete[User](driver, gid) require.NoError(t, err) - _, queriedUser3, err := modusdb.Get[User](db, gid) + _, queriedUser3, err := modusdb.Get[User](driver, gid) require.Error(t, err) require.Equal(t, "no object found", err.Error()) require.Equal(t, queriedUser3, User{}) @@ -75,11 +75,11 @@ func TestFirstTimeUser(t *testing.T) { func TestCreateApi(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -90,7 +90,7 @@ func TestCreateApi(t *testing.T) { ClerkId: "123", } - gid, user, err := modusdb.Create(db, user, db1.ID()) + gid, user, err := modusdb.Create(driver, user, db1.ID()) require.NoError(t, err) require.Equal(t, "B", user.Name) @@ -145,11 +145,11 @@ func TestCreateApi(t *testing.T) { func TestCreateApiWithNonStruct(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -159,18 +159,18 @@ func TestCreateApiWithNonStruct(t *testing.T) { Age: 20, } - _, _, err = modusdb.Create[*User](db, &user, db1.ID()) + _, _, err = modusdb.Create[*User](driver, &user, db1.ID()) require.Error(t, err) require.Equal(t, "expected struct, got ptr", err.Error()) } func TestGetApi(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -181,10 +181,10 @@ func TestGetApi(t *testing.T) { ClerkId: "123", } - gid, _, err := modusdb.Create(db, user, db1.ID()) + gid, _, err := modusdb.Create(driver, user, db1.ID()) require.NoError(t, err) - gid, queriedUser, err := modusdb.Get[User](db, gid, db1.ID()) + gid, queriedUser, err := modusdb.Get[User](driver, gid, db1.ID()) require.NoError(t, err) require.Equal(t, queriedUser.Gid, gid) @@ -195,11 +195,11 @@ func TestGetApi(t *testing.T) { func TestGetApiWithConstrainedField(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -210,10 +210,10 @@ func TestGetApiWithConstrainedField(t *testing.T) { ClerkId: "123", } - _, _, err = modusdb.Create(db, user, db1.ID()) + _, _, err = modusdb.Create(driver, user, db1.ID()) require.NoError(t, err) - gid, queriedUser, err := modusdb.Get[User](db, modusdb.ConstrainedField{ + gid, queriedUser, err := modusdb.Get[User](driver, modusdb.ConstrainedField{ Key: "clerk_id", Value: "123", }, db1.ID()) @@ -227,11 +227,11 @@ func TestGetApiWithConstrainedField(t *testing.T) { func TestDeleteApi(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -242,18 +242,18 @@ func TestDeleteApi(t *testing.T) { ClerkId: "123", } - gid, _, err := modusdb.Create(db, user, db1.ID()) + gid, _, err := modusdb.Create(driver, user, db1.ID()) require.NoError(t, err) - _, _, err = modusdb.Delete[User](db, gid, db1.ID()) + _, _, err = modusdb.Delete[User](driver, gid, db1.ID()) require.NoError(t, err) - _, queriedUser, err := modusdb.Get[User](db, gid, db1.ID()) + _, queriedUser, err := modusdb.Get[User](driver, gid, db1.ID()) require.Error(t, err) require.Equal(t, "no object found", err.Error()) require.Equal(t, queriedUser, User{}) - _, queriedUser, err = modusdb.Get[User](db, modusdb.ConstrainedField{ + _, queriedUser, err = modusdb.Get[User](driver, modusdb.ConstrainedField{ Key: "clerk_id", Value: "123", }, db1.ID()) @@ -264,11 +264,11 @@ func TestDeleteApi(t *testing.T) { func TestUpsertApi(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -279,16 +279,16 @@ func TestUpsertApi(t *testing.T) { ClerkId: "123", } - gid, user, _, err := modusdb.Upsert(db, user, db1.ID()) + gid, user, _, err := modusdb.Upsert(driver, user, db1.ID()) require.NoError(t, err) require.Equal(t, user.Gid, gid) user.Age = 21 - gid, _, _, err = modusdb.Upsert(db, user, db1.ID()) + gid, _, _, err = modusdb.Upsert(driver, user, db1.ID()) require.NoError(t, err) require.Equal(t, user.Gid, gid) - _, queriedUser, err := modusdb.Get[User](db, gid, db1.ID()) + _, queriedUser, err := modusdb.Get[User](driver, gid, db1.ID()) require.NoError(t, err) require.Equal(t, user.Gid, queriedUser.Gid) require.Equal(t, 21, queriedUser.Age) @@ -298,11 +298,11 @@ func TestUpsertApi(t *testing.T) { func TestQueryApi(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -316,11 +316,11 @@ func TestQueryApi(t *testing.T) { } for _, user := range users { - _, _, err = modusdb.Create(db, user, db1.ID()) + _, _, err = modusdb.Create(driver, user, db1.ID()) require.NoError(t, err) } - gids, queriedUsers, err := modusdb.Query[User](db, modusdb.QueryParams{}, db1.ID()) + gids, queriedUsers, err := modusdb.Query[User](driver, modusdb.QueryParams{}, db1.ID()) require.NoError(t, err) require.Len(t, queriedUsers, 5) require.Len(t, gids, 5) @@ -330,7 +330,7 @@ func TestQueryApi(t *testing.T) { require.Equal(t, "D", queriedUsers[3].Name) require.Equal(t, "E", queriedUsers[4].Name) - gids, queriedUsers, err = modusdb.Query[User](db, modusdb.QueryParams{ + gids, queriedUsers, err = modusdb.Query[User](driver, modusdb.QueryParams{ Filter: &modusdb.Filter{ Field: "age", String: modusdb.StringPredicate{ @@ -353,11 +353,11 @@ func TestQueryApi(t *testing.T) { func TestQueryApiWithPaginiationAndSorting(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -371,11 +371,11 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { } for _, user := range users { - _, _, err = modusdb.Create(db, user, db1.ID()) + _, _, err = modusdb.Create(driver, user, db1.ID()) require.NoError(t, err) } - gids, queriedUsers, err := modusdb.Query[User](db, modusdb.QueryParams{ + gids, queriedUsers, err := modusdb.Query[User](driver, modusdb.QueryParams{ Filter: &modusdb.Filter{ Field: "age", String: modusdb.StringPredicate{ @@ -395,7 +395,7 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { require.Equal(t, "D", queriedUsers[1].Name) require.Equal(t, "E", queriedUsers[2].Name) - gids, queriedUsers, err = modusdb.Query[User](db, modusdb.QueryParams{ + gids, queriedUsers, err = modusdb.Query[User](driver, modusdb.QueryParams{ Pagination: &modusdb.Pagination{ Limit: 3, Offset: 1, @@ -416,29 +416,29 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { type Project struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" driver:"constraint=unique"` Branches []Branch `json:"branches,omitempty" readFrom:"type=Branch,field=proj"` } type Branch struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" driver:"constraint=unique"` Proj Project `json:"proj,omitempty"` } func TestReverseEdgeGet(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) - projGid, project, err := modusdb.Create(db, Project{ + projGid, project, err := modusdb.Create(driver, Project{ Name: "P", ClerkId: "456", Branches: []Branch{ @@ -462,7 +462,7 @@ func TestReverseEdgeGet(t *testing.T) { }, } - branch1Gid, branch1, err := modusdb.Create(db, branch1, db1.ID()) + branch1Gid, branch1, err := modusdb.Create(driver, branch1, db1.ID()) require.NoError(t, err) require.Equal(t, "B", branch1.Name) @@ -478,13 +478,13 @@ func TestReverseEdgeGet(t *testing.T) { }, } - branch2Gid, branch2, err := modusdb.Create(db, branch2, db1.ID()) + branch2Gid, branch2, err := modusdb.Create(driver, branch2, db1.ID()) require.NoError(t, err) require.Equal(t, "B2", branch2.Name) require.Equal(t, branch2.Gid, branch2Gid) require.Equal(t, projGid, branch2.Proj.Gid) - getProjGid, queriedProject, err := modusdb.Get[Project](db, projGid, db1.ID()) + getProjGid, queriedProject, err := modusdb.Get[Project](driver, projGid, db1.ID()) require.NoError(t, err) require.Equal(t, projGid, getProjGid) require.Equal(t, "P", queriedProject.Name) @@ -492,7 +492,7 @@ func TestReverseEdgeGet(t *testing.T) { require.Equal(t, "B", queriedProject.Branches[0].Name) require.Equal(t, "B2", queriedProject.Branches[1].Name) - queryBranchesGids, queriedBranches, err := modusdb.Query[Branch](db, modusdb.QueryParams{}, db1.ID()) + queryBranchesGids, queriedBranches, err := modusdb.Query[Branch](driver, modusdb.QueryParams{}, db1.ID()) require.NoError(t, err) require.Len(t, queriedBranches, 2) require.Len(t, queryBranchesGids, 2) @@ -502,10 +502,10 @@ func TestReverseEdgeGet(t *testing.T) { // max depth is 2, so we should not see the branches within project require.Len(t, queriedBranches[0].Proj.Branches, 0) - _, _, err = modusdb.Delete[Project](db, projGid, db1.ID()) + _, _, err = modusdb.Delete[Project](driver, projGid, db1.ID()) require.NoError(t, err) - queryBranchesGids, queriedBranches, err = modusdb.Query[Branch](db, modusdb.QueryParams{}, db1.ID()) + queryBranchesGids, queriedBranches, err = modusdb.Query[Branch](driver, modusdb.QueryParams{}, db1.ID()) require.NoError(t, err) require.Len(t, queriedBranches, 2) require.Len(t, queryBranchesGids, 2) @@ -515,11 +515,11 @@ func TestReverseEdgeGet(t *testing.T) { func TestReverseEdgeQuery(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -533,7 +533,7 @@ func TestReverseEdgeQuery(t *testing.T) { clerkCounter := 100 for _, project := range projects { - projGid, project, err := modusdb.Create(db, project, db1.ID()) + projGid, project, err := modusdb.Create(driver, project, db1.ID()) require.NoError(t, err) require.Equal(t, project.Name, project.Name) require.Equal(t, project.Gid, projGid) @@ -546,7 +546,7 @@ func TestReverseEdgeQuery(t *testing.T) { clerkCounter += 2 for _, branch := range branches { - branchGid, branch, err := modusdb.Create(db, branch, db1.ID()) + branchGid, branch, err := modusdb.Create(driver, branch, db1.ID()) require.NoError(t, err) require.Equal(t, branch.Name, branch.Name) require.Equal(t, branch.Gid, branchGid) @@ -554,7 +554,7 @@ func TestReverseEdgeQuery(t *testing.T) { } } - queriedProjectsGids, queriedProjects, err := modusdb.Query[Project](db, modusdb.QueryParams{}, db1.ID()) + queriedProjectsGids, queriedProjects, err := modusdb.Query[Project](driver, modusdb.QueryParams{}, db1.ID()) require.NoError(t, err) require.Len(t, queriedProjects, 2) require.Len(t, queriedProjectsGids, 2) @@ -570,11 +570,11 @@ func TestReverseEdgeQuery(t *testing.T) { func TestNestedObjectMutation(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -588,7 +588,7 @@ func TestNestedObjectMutation(t *testing.T) { }, } - gid, branch, err := modusdb.Create(db, branch, db1.ID()) + gid, branch, err := modusdb.Create(driver, branch, db1.ID()) require.NoError(t, err) require.Equal(t, "B", branch.Name) @@ -615,7 +615,7 @@ func TestNestedObjectMutation(t *testing.T) { {"uid":"0x3","Project.name":"P","Project.clerk_id":"456"}}]}`, string(resp.GetJson())) - gid, queriedBranch, err := modusdb.Get[Branch](db, gid, db1.ID()) + gid, queriedBranch, err := modusdb.Get[Branch](driver, gid, db1.ID()) require.NoError(t, err) require.Equal(t, queriedBranch.Gid, gid) require.Equal(t, "B", queriedBranch.Name) @@ -624,16 +624,16 @@ func TestNestedObjectMutation(t *testing.T) { func TestLinkingObjectsByConstrainedFields(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) - projGid, project, err := modusdb.Create(db, Project{ + projGid, project, err := modusdb.Create(driver, Project{ Name: "P", ClerkId: "456", }, db1.ID()) @@ -651,7 +651,7 @@ func TestLinkingObjectsByConstrainedFields(t *testing.T) { }, } - gid, branch, err := modusdb.Create(db, branch, db1.ID()) + gid, branch, err := modusdb.Create(driver, branch, db1.ID()) require.NoError(t, err) require.Equal(t, "B", branch.Name) @@ -678,7 +678,7 @@ func TestLinkingObjectsByConstrainedFields(t *testing.T) { {"uid":"0x2","Project.name":"P","Project.clerk_id":"456"}}]}`, string(resp.GetJson())) - gid, queriedBranch, err := modusdb.Get[Branch](db, gid, db1.ID()) + gid, queriedBranch, err := modusdb.Get[Branch](driver, gid, db1.ID()) require.NoError(t, err) require.Equal(t, queriedBranch.Gid, gid) require.Equal(t, "B", queriedBranch.Name) @@ -687,16 +687,16 @@ func TestLinkingObjectsByConstrainedFields(t *testing.T) { func TestLinkingObjectsByGid(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) - projGid, project, err := modusdb.Create(db, Project{ + projGid, project, err := modusdb.Create(driver, Project{ Name: "P", ClerkId: "456", }, db1.ID()) @@ -713,7 +713,7 @@ func TestLinkingObjectsByGid(t *testing.T) { }, } - gid, branch, err := modusdb.Create(db, branch, db1.ID()) + gid, branch, err := modusdb.Create(driver, branch, db1.ID()) require.NoError(t, err) require.Equal(t, "B", branch.Name) @@ -740,7 +740,7 @@ func TestLinkingObjectsByGid(t *testing.T) { "Branch.proj":{"uid":"0x2","Project.name":"P","Project.clerk_id":"456"}}]}`, string(resp.GetJson())) - gid, queriedBranch, err := modusdb.Get[Branch](db, gid, db1.ID()) + gid, queriedBranch, err := modusdb.Get[Branch](driver, gid, db1.ID()) require.NoError(t, err) require.Equal(t, queriedBranch.Gid, gid) require.Equal(t, "B", queriedBranch.Name) @@ -755,17 +755,17 @@ type BadProject struct { type BadBranch struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" driver:"constraint=unique"` Proj BadProject `json:"proj,omitempty"` } func TestNestedObjectMutationWithBadType(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -779,7 +779,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { }, } - _, _, err = modusdb.Create(db, branch, db1.ID()) + _, _, err = modusdb.Create(driver, branch, db1.ID()) require.Error(t, err) require.Equal(t, fmt.Sprintf(apiutils.NoUniqueConstr, "BadProject"), err.Error()) @@ -788,7 +788,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { ClerkId: "456", } - _, _, err = modusdb.Create(db, proj, db1.ID()) + _, _, err = modusdb.Create(driver, proj, db1.ID()) require.Error(t, err) require.Equal(t, fmt.Sprintf(apiutils.NoUniqueConstr, "BadProject"), err.Error()) @@ -797,16 +797,16 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { type Document struct { Gid uint64 `json:"gid,omitempty"` Text string `json:"text,omitempty"` - TextVec []float32 `json:"textVec,omitempty" db:"constraint=vector"` + TextVec []float32 `json:"textVec,omitempty" driver:"constraint=vector"` } func TestVectorIndexSearchTyped(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -822,7 +822,7 @@ func TestVectorIndexSearchTyped(t *testing.T) { } for _, doc := range documents { - _, _, err = modusdb.Create(db, doc, db1.ID()) + _, _, err = modusdb.Create(driver, doc, db1.ID()) require.NoError(t, err) } @@ -867,11 +867,11 @@ func TestVectorIndexSearchTyped(t *testing.T) { func TestVectorIndexSearchWithQuery(t *testing.T) { ctx := context.Background() - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - db1, err := db.CreateNamespace() + db1, err := driver.CreateDB() require.NoError(t, err) require.NoError(t, db1.DropData(ctx)) @@ -887,11 +887,11 @@ func TestVectorIndexSearchWithQuery(t *testing.T) { } for _, doc := range documents { - _, _, err = modusdb.Create(db, doc, db1.ID()) + _, _, err = modusdb.Create(driver, doc, db1.ID()) require.NoError(t, err) } - gids, docs, err := modusdb.Query[Document](db, modusdb.QueryParams{ + gids, docs, err := modusdb.Query[Document](driver, modusdb.QueryParams{ Filter: &modusdb.Filter{ Field: "textVec", Vector: modusdb.VectorPredicate{ diff --git a/api_types.go b/api_types.go index 8102771..010dd79 100644 --- a/api_types.go +++ b/api_types.go @@ -84,15 +84,15 @@ func WithNamespace(namespace uint64) ModusDbOption { } } -func getDefaultNamespace(db *DB, ns ...uint64) (context.Context, *Namespace, error) { +func getDefaultNamespace(db *Driver, ns ...uint64) (context.Context, *DB, error) { dbOpts := &modusDbOptions{ - namespace: db.defaultNamespace.ID(), + namespace: db.db0.ID(), } for _, ns := range ns { WithNamespace(ns)(dbOpts) } - n, err := db.getNamespaceWithLock(dbOpts.namespace) + n, err := db.getDBWithLock(dbOpts.namespace) if err != nil { return nil, nil, err } diff --git a/db.go b/db.go index d092ecd..a50b58d 100644 --- a/db.go +++ b/db.go @@ -11,219 +11,189 @@ package modusdb import ( "context" - "errors" "fmt" - "path" - "sync" - "sync/atomic" + "strconv" - "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/dql" "github.com/dgraph-io/dgraph/v24/edgraph" - "github.com/dgraph-io/dgraph/v24/posting" "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/query" "github.com/dgraph-io/dgraph/v24/schema" "github.com/dgraph-io/dgraph/v24/worker" "github.com/dgraph-io/dgraph/v24/x" - "github.com/dgraph-io/ristretto/v2/z" ) -var ( - // This ensures that we only have one instance of modusDB in this process. - singleton atomic.Bool - - ErrSingletonOnly = errors.New("only one modusDB instance is supported") - ErrEmptyDataDir = errors.New("data directory is required") - ErrClosedDB = errors.New("modusDB instance is closed") - ErrNonExistentNamespace = errors.New("namespace does not exist") -) - -// DB is an instance of modusDB. -// For now, we only support one instance of modusDB per process. +// DB is one of the namespaces in modusDB. type DB struct { - mutex sync.RWMutex - isOpen bool - - z *zero + id uint64 + driver *Driver +} - // points to default / 0 / galaxy namespace - defaultNamespace *Namespace +func (db *DB) ID() uint64 { + return db.id } -// New returns a new modusDB instance. -func New(conf Config) (*DB, error) { - // Ensure that we do not create another instance of modusDB in the same process - if !singleton.CompareAndSwap(false, true) { - return nil, ErrSingletonOnly - } +// DropData drops all the data in the modusDB instance. +func (db *DB) DropData(ctx context.Context) error { + db.driver.mutex.Lock() + defer db.driver.mutex.Unlock() - if err := conf.validate(); err != nil { - return nil, err + if !db.driver.isOpen.Load() { + return ErrClosedDriver } - // setup data directories - worker.Config.PostingDir = path.Join(conf.dataDir, "p") - worker.Config.WALDir = path.Join(conf.dataDir, "w") - x.WorkerConfig.TmpDir = path.Join(conf.dataDir, "t") - - // TODO: optimize these and more options - x.WorkerConfig.Badger = badger.DefaultOptions("").FromSuperFlag(worker.BadgerDefaults) - x.Config.MaxRetries = 10 - x.Config.Limit = z.NewSuperFlag("max-pending-queries=100000") - x.Config.LimitNormalizeNode = conf.limitNormalizeNode - - // initialize each package - edgraph.Init() - worker.State.InitStorage() - worker.InitForLite(worker.State.Pstore) - schema.Init(worker.State.Pstore) - posting.Init(worker.State.Pstore, 0) // TODO: set cache size + p := &pb.Proposal{Mutations: &pb.Mutations{ + GroupId: 1, + DropOp: pb.Mutations_DATA, + DropValue: strconv.FormatUint(db.ID(), 10), + }} - db := &DB{isOpen: true} - if err := db.reset(); err != nil { - return nil, fmt.Errorf("error resetting db: %w", err) + if err := worker.ApplyMutations(ctx, p); err != nil { + return fmt.Errorf("error applying mutation: %w", err) } - x.UpdateHealthStatus(true) - - db.defaultNamespace = &Namespace{id: 0, db: db} - return db, nil + // TODO: insert drop record + // TODO: should we reset back the timestamp as well? + return nil } -func (db *DB) CreateNamespace() (*Namespace, error) { - db.mutex.RLock() - defer db.mutex.RUnlock() +func (db *DB) AlterSchema(ctx context.Context, sch string) error { + db.driver.mutex.Lock() + defer db.driver.mutex.Unlock() - if !db.isOpen { - return nil, ErrClosedDB + if !db.driver.isOpen.Load() { + return ErrClosedDriver } - startTs, err := db.z.nextTs() + sc, err := schema.ParseWithNamespace(sch, db.ID()) if err != nil { - return nil, err + return fmt.Errorf("error parsing schema: %w", err) } - nsID, err := db.z.nextNS() - if err != nil { - return nil, err - } - - if err := worker.ApplyInitialSchema(nsID, startTs); err != nil { - return nil, fmt.Errorf("error applying initial schema: %w", err) - } - for _, pred := range schema.State().Predicates() { - worker.InitTablet(pred) - } - - return &Namespace{id: nsID, db: db}, nil + return db.alterSchemaWithParsed(ctx, sc) } -func (db *DB) GetNamespace(nsID uint64) (*Namespace, error) { - db.mutex.RLock() - defer db.mutex.RUnlock() - - return db.getNamespaceWithLock(nsID) -} - -func (db *DB) getNamespaceWithLock(nsID uint64) (*Namespace, error) { - if !db.isOpen { - return nil, ErrClosedDB - } - - if nsID > db.z.lastNS { - return nil, ErrNonExistentNamespace +func (db *DB) alterSchemaWithParsed(ctx context.Context, sc *schema.ParsedSchema) error { + for _, pred := range sc.Preds { + worker.InitTablet(pred.Predicate) } - // TODO: when delete namespace is implemented, check if the namespace exists - - return &Namespace{id: nsID, db: db}, nil -} - -// DropAll drops all the data and schema in the modusDB instance. -func (db *DB) DropAll(ctx context.Context) error { - db.mutex.Lock() - defer db.mutex.Unlock() - - if !db.isOpen { - return ErrClosedDB + startTs, err := db.driver.z.nextTs() + if err != nil { + return err } p := &pb.Proposal{Mutations: &pb.Mutations{ GroupId: 1, - DropOp: pb.Mutations_ALL, + StartTs: startTs, + Schema: sc.Preds, + Types: sc.Types, }} if err := worker.ApplyMutations(ctx, p); err != nil { return fmt.Errorf("error applying mutation: %w", err) } - if err := db.reset(); err != nil { - return fmt.Errorf("error resetting db: %w", err) - } - - // TODO: insert drop record return nil } -func (db *DB) DropData(ctx context.Context) error { - return db.defaultNamespace.DropData(ctx) -} - -func (db *DB) AlterSchema(ctx context.Context, sch string) error { - return db.defaultNamespace.AlterSchema(ctx, sch) -} - -func (db *DB) Query(ctx context.Context, q string) (*api.Response, error) { - return db.defaultNamespace.Query(ctx, q) -} - func (db *DB) Mutate(ctx context.Context, ms []*api.Mutation) (map[string]uint64, error) { - return db.defaultNamespace.Mutate(ctx, ms) -} + if len(ms) == 0 { + return nil, nil + } -func (db *DB) Load(ctx context.Context, schemaPath, dataPath string) error { - return db.defaultNamespace.Load(ctx, schemaPath, dataPath) -} + db.driver.mutex.Lock() + defer db.driver.mutex.Unlock() + dms := make([]*dql.Mutation, 0, len(ms)) + for _, mu := range ms { + dm, err := edgraph.ParseMutationObject(mu, false) + if err != nil { + return nil, fmt.Errorf("error parsing mutation: %w", err) + } + dms = append(dms, dm) + } + newUids, err := query.ExtractBlankUIDs(ctx, dms) + if err != nil { + return nil, err + } + if len(newUids) > 0 { + num := &pb.Num{Val: uint64(len(newUids)), Type: pb.Num_UID} + res, err := db.driver.z.nextUIDs(num) + if err != nil { + return nil, err + } -func (db *DB) LoadData(inCtx context.Context, dataDir string) error { - return db.defaultNamespace.LoadData(inCtx, dataDir) -} + curId := res.StartId + for k := range newUids { + x.AssertTruef(curId != 0 && curId <= res.EndId, "not enough uids generated") + newUids[k] = curId + curId++ + } + } -// Close closes the modusDB instance. -func (db *DB) Close() { - db.mutex.Lock() - defer db.mutex.Unlock() + return db.mutateWithDqlMutation(ctx, dms, newUids) +} - if !db.isOpen { - return +func (db *DB) mutateWithDqlMutation(ctx context.Context, dms []*dql.Mutation, + newUids map[string]uint64) (map[string]uint64, error) { + edges, err := query.ToDirectedEdges(dms, newUids) + if err != nil { + return nil, err } + ctx = x.AttachNamespace(ctx, db.ID()) - if !singleton.CompareAndSwap(true, false) { - panic("modusDB instance was not properly opened") + if !db.driver.isOpen.Load() { + return nil, ErrClosedDriver } - db.isOpen = false - x.UpdateHealthStatus(false) - posting.Cleanup() - worker.State.Dispose() -} + startTs, err := db.driver.z.nextTs() + if err != nil { + return nil, err + } + commitTs, err := db.driver.z.nextTs() + if err != nil { + return nil, err + } -func (db *DB) reset() error { - z, restart, err := newZero() + m := &pb.Mutations{ + GroupId: 1, + StartTs: startTs, + Edges: edges, + } + m.Edges, err = query.ExpandEdges(ctx, m) if err != nil { - return fmt.Errorf("error initializing zero: %w", err) + return nil, fmt.Errorf("error expanding edges: %w", err) } - if !restart { - if err := worker.ApplyInitialSchema(0, 1); err != nil { - return fmt.Errorf("error applying initial schema: %w", err) - } + for _, edge := range m.Edges { + worker.InitTablet(edge.Attr) } - if err := schema.LoadFromDb(context.Background()); err != nil { - return fmt.Errorf("error loading schema: %w", err) + p := &pb.Proposal{Mutations: m, StartTs: startTs} + if err := worker.ApplyMutations(ctx, p); err != nil { + return nil, err } - for _, pred := range schema.State().Predicates() { - worker.InitTablet(pred) + + return newUids, worker.ApplyCommited(ctx, &pb.OracleDelta{ + Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, + }) +} + +// Query performs query or mutation or upsert on the given modusDB instance. +func (db *DB) Query(ctx context.Context, query string) (*api.Response, error) { + db.driver.mutex.RLock() + defer db.driver.mutex.RUnlock() + + return db.queryWithLock(ctx, query) +} + +func (db *DB) queryWithLock(ctx context.Context, query string) (*api.Response, error) { + if !db.driver.isOpen.Load() { + return nil, ErrClosedDriver } - db.z = z - return nil + ctx = x.AttachNamespace(ctx, db.ID()) + return (&edgraph.Server{}).QueryNoAuth(ctx, &api.Request{ + ReadOnly: true, + Query: query, + StartTs: db.driver.z.readTs(), + }) } diff --git a/db_test.go b/db_test.go index 5de5d68..6a4045c 100644 --- a/db_test.go +++ b/db_test.go @@ -10,10 +10,7 @@ package modusdb_test import ( - "bytes" "context" - "encoding/binary" - "fmt" "testing" "github.com/dgraph-io/dgo/v240/protos/api" @@ -22,21 +19,21 @@ import ( "github.com/hypermodeinc/modusdb" ) -func TestRestart(t *testing.T) { - dataDir := t.TempDir() +func TestNonGalaxyDB(t *testing.T) { + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer driver.Close() - db, err := modusdb.New(modusdb.NewDefaultConfig(dataDir)) + db1, err := driver.CreateDB() require.NoError(t, err) - defer func() { db.Close() }() - require.NoError(t, db.DropAll(context.Background())) - require.NoError(t, db.AlterSchema(context.Background(), "name: string @index(term) .")) + require.NoError(t, db1.DropData(context.Background())) + require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) - _, err = db.Mutate(context.Background(), []*api.Mutation{ + _, err = db1.Mutate(context.Background(), []*api.Mutation{ { Set: []*api.NQuad{ { - Namespace: 0, Subject: "_:aman", Predicate: "name", ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, @@ -51,81 +48,250 @@ func TestRestart(t *testing.T) { name } }` - qresp, err := db.Query(context.Background(), query) + resp, err := db1.Query(context.Background(), query) + require.NoError(t, err) + require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) + +} + +func TestDropData(t *testing.T) { + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer driver.Close() + + db1, err := driver.CreateDB() + require.NoError(t, err) + + require.NoError(t, db1.DropData(context.Background())) + require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) + + _, err = db1.Mutate(context.Background(), []*api.Mutation{ + { + Set: []*api.NQuad{ + { + Subject: "_:aman", + Predicate: "name", + ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, + }, + }, + }, + }) + require.NoError(t, err) + + query := `{ + me(func: has(name)) { + name + } + }` + resp, err := db1.Query(context.Background(), query) + require.NoError(t, err) + require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) + + require.NoError(t, db1.DropData(context.Background())) + + resp, err = db1.Query(context.Background(), query) + require.NoError(t, err) + require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) +} + +func TestMultipleDBs(t *testing.T) { + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer driver.Close() + + db0, err := driver.GetDB(0) + require.NoError(t, err) + db1, err := driver.CreateDB() + require.NoError(t, err) + + require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, db0.AlterSchema(context.Background(), "name: string @index(exact) .")) + require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) + + _, err = db0.Mutate(context.Background(), []*api.Mutation{ + { + Set: []*api.NQuad{ + { + Subject: "_:aman", + Predicate: "name", + ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, + }, + }, + }, + }) require.NoError(t, err) - require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) - db.Close() - db, err = modusdb.New(modusdb.NewDefaultConfig(dataDir)) + _, err = db1.Mutate(context.Background(), []*api.Mutation{ + { + Set: []*api.NQuad{ + { + Subject: "_:aman", + Predicate: "name", + ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "B"}}, + }, + }, + }, + }) + require.NoError(t, err) + + query := `{ + me(func: has(name)) { + name + } + }` + resp, err := db0.Query(context.Background(), query) require.NoError(t, err) - qresp, err = db.Query(context.Background(), query) + require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) + + resp, err = db1.Query(context.Background(), query) require.NoError(t, err) - require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) + require.JSONEq(t, `{"me":[{"name":"B"}]}`, string(resp.GetJson())) - require.NoError(t, db.DropAll(context.Background())) + require.NoError(t, db1.DropData(context.Background())) + resp, err = db1.Query(context.Background(), query) + require.NoError(t, err) + require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) } -func TestSchemaQuery(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) +func TestQueryWrongDB(t *testing.T) { + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer driver.Close() + + db0, err := driver.GetDB(0) + require.NoError(t, err) + db1, err := driver.CreateDB() require.NoError(t, err) - defer db.Close() - require.NoError(t, db.DropAll(context.Background())) - require.NoError(t, db.AlterSchema(context.Background(), ` - name: string @index(exact) . - age: int . - married: bool . - loc: geo . - dob: datetime . - `)) + require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, db0.AlterSchema(context.Background(), "name: string @index(exact) .")) + require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) - resp, err := db.Query(context.Background(), `schema(pred: [name, age]) {type}`) + _, err = db0.Mutate(context.Background(), []*api.Mutation{ + { + Set: []*api.NQuad{ + { + Namespace: 1, + Subject: "_:aman", + Predicate: "name", + ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, + }, + }, + }, + }) require.NoError(t, err) - require.JSONEq(t, - `{"schema":[{"predicate":"age","type":"int"},{"predicate":"name","type":"string"}]}`, - string(resp.GetJson())) + query := `{ + me(func: has(name)) { + name + } + }` + + resp, err := db1.Query(context.Background(), query) + require.NoError(t, err) + require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) } -func TestBasicVector(t *testing.T) { - vect := []float32{5.1, 5.1, 1.1} - buf := new(bytes.Buffer) - for _, v := range vect { - require.NoError(t, binary.Write(buf, binary.LittleEndian, v)) - } - vectBytes := buf.Bytes() +func TestTwoDBs(t *testing.T) { + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer driver.Close() + + db0, err := driver.GetDB(0) + require.NoError(t, err) + db1, err := driver.CreateDB() + require.NoError(t, err) + + require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, db0.AlterSchema(context.Background(), "foo: string @index(exact) .")) + require.NoError(t, db1.AlterSchema(context.Background(), "bar: string @index(exact) .")) + + _, err = db0.Mutate(context.Background(), []*api.Mutation{ + { + Set: []*api.NQuad{ + { + Subject: "_:aman", + Predicate: "foo", + ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, + }, + }, + }, + }) + require.NoError(t, err) + + _, err = db1.Mutate(context.Background(), []*api.Mutation{ + { + Set: []*api.NQuad{ + { + Subject: "_:aman", + Predicate: "bar", + ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "B"}}, + }, + }, + }, + }) + require.NoError(t, err) + + query := `{ + me(func: has(foo)) { + foo + } + }` + resp, err := db0.Query(context.Background(), query) + require.NoError(t, err) + require.JSONEq(t, `{"me":[{"foo":"A"}]}`, string(resp.GetJson())) - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + query = `{ + me(func: has(bar)) { + bar + } + }` + resp, err = db1.Query(context.Background(), query) require.NoError(t, err) - defer db.Close() + require.JSONEq(t, `{"me":[{"bar":"B"}]}`, string(resp.GetJson())) +} - require.NoError(t, db.DropAll(context.Background())) - require.NoError(t, db.AlterSchema(context.Background(), - `project_description_v: float32vector @index(hnsw(exponent: "5", metric: "euclidean")) .`)) +func TestDBDBRestart(t *testing.T) { + dataDir := t.TempDir() + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(dataDir)) + require.NoError(t, err) + defer func() { driver.Close() }() - uids, err := db.Mutate(context.Background(), []*api.Mutation{{ - Set: []*api.NQuad{{ - Subject: "_:vector", - Predicate: "project_description_v", - ObjectValue: &api.Value{ - Val: &api.Value_Vfloat32Val{Vfloat32Val: vectBytes}, + db1, err := driver.CreateDB() + require.NoError(t, err) + ns1 := db1.ID() + + require.NoError(t, db1.AlterSchema(context.Background(), "bar: string @index(exact) .")) + _, err = db1.Mutate(context.Background(), []*api.Mutation{ + { + Set: []*api.NQuad{ + { + Subject: "_:aman", + Predicate: "bar", + ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "B"}}, + }, }, - }}, - }}) + }, + }) require.NoError(t, err) - uid := uids["_:vector"] - if uid == 0 { - t.Fatalf("Expected non-zero uid") - } + driver.Close() + driver, err = modusdb.NewDriver(modusdb.NewDefaultConfig(dataDir)) + require.NoError(t, err) - resp, err := db.Query(context.Background(), fmt.Sprintf(`query { - q (func: uid(%v)) { - project_description_v - } - }`, uid)) + db2, err := driver.CreateDB() + require.NoError(t, err) + require.Greater(t, db2.ID(), ns1) + + db1, err = driver.GetDB(ns1) + require.NoError(t, err) + + query := `{ + me(func: has(bar)) { + bar + } + }` + resp, err := db1.Query(context.Background(), query) require.NoError(t, err) - require.Equal(t, - `{"q":[{"project_description_v":[5.1E+00,5.1E+00,1.1E+00]}]}`, - string(resp.GetJson())) + require.JSONEq(t, `{"me":[{"bar":"B"}]}`, string(resp.GetJson())) } diff --git a/driver.go b/driver.go new file mode 100644 index 0000000..6a29a01 --- /dev/null +++ b/driver.go @@ -0,0 +1,230 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package modusdb + +import ( + "context" + "errors" + "fmt" + "path" + "sync" + "sync/atomic" + + "github.com/dgraph-io/badger/v4" + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/edgraph" + "github.com/dgraph-io/dgraph/v24/posting" + "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/schema" + "github.com/dgraph-io/dgraph/v24/worker" + "github.com/dgraph-io/dgraph/v24/x" + "github.com/dgraph-io/ristretto/v2/z" +) + +var ( + // This ensures that we only have one instance of modusDB in this process. + singleton atomic.Bool + + ErrSingletonOnly = errors.New("only one modusDB driver is supported") + ErrEmptyDataDir = errors.New("data directory is required") + ErrClosedDriver = errors.New("modusDB driver is closed") + ErrNonExistentDB = errors.New("namespace does not exist") +) + +// Driver is an instance of modusDB. +// For now, we only support one instance of modusDB per process. +type Driver struct { + mutex sync.RWMutex + isOpen atomic.Bool + + z *zero + + // points to default / 0 / galaxy namespace + db0 *DB +} + +// NewDriver returns a new modusDB instance. +func NewDriver(conf Config) (*Driver, error) { + // Ensure that we do not create another instance of modusDB in the same process + if !singleton.CompareAndSwap(false, true) { + return nil, ErrSingletonOnly + } + + if err := conf.validate(); err != nil { + return nil, err + } + + // setup data directories + worker.Config.PostingDir = path.Join(conf.dataDir, "p") + worker.Config.WALDir = path.Join(conf.dataDir, "w") + x.WorkerConfig.TmpDir = path.Join(conf.dataDir, "t") + + // TODO: optimize these and more options + x.WorkerConfig.Badger = badger.DefaultOptions("").FromSuperFlag(worker.BadgerDefaults) + x.Config.MaxRetries = 10 + x.Config.Limit = z.NewSuperFlag("max-pending-queries=100000") + x.Config.LimitNormalizeNode = conf.limitNormalizeNode + + // initialize each package + edgraph.Init() + worker.State.InitStorage() + worker.InitForLite(worker.State.Pstore) + schema.Init(worker.State.Pstore) + posting.Init(worker.State.Pstore, 0) // TODO: set cache size + + driver := &Driver{} + driver.isOpen.Store(true) + if err := driver.reset(); err != nil { + return nil, fmt.Errorf("error resetting db: %w", err) + } + + x.UpdateHealthStatus(true) + + driver.db0 = &DB{id: 0, driver: driver} + return driver, nil +} + +func (db *Driver) CreateDB() (*DB, error) { + db.mutex.RLock() + defer db.mutex.RUnlock() + + if !db.isOpen.Load() { + return nil, ErrClosedDriver + } + + startTs, err := db.z.nextTs() + if err != nil { + return nil, err + } + dbID, err := db.z.nextDB() + if err != nil { + return nil, err + } + + if err := worker.ApplyInitialSchema(dbID, startTs); err != nil { + return nil, fmt.Errorf("error applying initial schema: %w", err) + } + for _, pred := range schema.State().Predicates() { + worker.InitTablet(pred) + } + + return &DB{id: dbID, driver: db}, nil +} + +func (driver *Driver) GetDB(dbID uint64) (*DB, error) { + driver.mutex.RLock() + defer driver.mutex.RUnlock() + + return driver.getDBWithLock(dbID) +} + +func (driver *Driver) getDBWithLock(dbID uint64) (*DB, error) { + if !driver.isOpen.Load() { + return nil, ErrClosedDriver + } + + if dbID > driver.z.lastDB { + return nil, ErrNonExistentDB + } + + // TODO: when delete namespace is implemented, check if the namespace exists + + return &DB{id: dbID, driver: driver}, nil +} + +// DropAll drops all the data and schema in the modusDB instance. +func (driver *Driver) DropAll(ctx context.Context) error { + driver.mutex.Lock() + defer driver.mutex.Unlock() + + if !driver.isOpen.Load() { + return ErrClosedDriver + } + + p := &pb.Proposal{Mutations: &pb.Mutations{ + GroupId: 1, + DropOp: pb.Mutations_ALL, + }} + if err := worker.ApplyMutations(ctx, p); err != nil { + return fmt.Errorf("error applying mutation: %w", err) + } + if err := driver.reset(); err != nil { + return fmt.Errorf("error resetting db: %w", err) + } + + // TODO: insert drop record + return nil +} + +func (driver *Driver) DropData(ctx context.Context) error { + return driver.db0.DropData(ctx) +} + +func (driver *Driver) AlterSchema(ctx context.Context, sch string) error { + return driver.db0.AlterSchema(ctx, sch) +} + +func (driver *Driver) Query(ctx context.Context, q string) (*api.Response, error) { + return driver.db0.Query(ctx, q) +} + +func (driver *Driver) Mutate(ctx context.Context, ms []*api.Mutation) (map[string]uint64, error) { + return driver.db0.Mutate(ctx, ms) +} + +func (driver *Driver) Load(ctx context.Context, schemaPath, dataPath string) error { + return driver.db0.Load(ctx, schemaPath, dataPath) +} + +func (driver *Driver) LoadData(inCtx context.Context, dataDir string) error { + return driver.db0.LoadData(inCtx, dataDir) +} + +// Close closes the modusDB instance. +func (driver *Driver) Close() { + driver.mutex.Lock() + defer driver.mutex.Unlock() + + if !driver.isOpen.Load() { + return + } + + if !singleton.CompareAndSwap(true, false) { + panic("modusDB instance was not properly opened") + } + + driver.isOpen.Store(false) + x.UpdateHealthStatus(false) + posting.Cleanup() + worker.State.Dispose() +} + +func (db *Driver) reset() error { + z, restart, err := newZero() + if err != nil { + return fmt.Errorf("error initializing zero: %w", err) + } + + if !restart { + if err := worker.ApplyInitialSchema(0, 1); err != nil { + return fmt.Errorf("error applying initial schema: %w", err) + } + } + + if err := schema.LoadFromDb(context.Background()); err != nil { + return fmt.Errorf("error loading schema: %w", err) + } + for _, pred := range schema.State().Predicates() { + worker.InitTablet(pred) + } + + db.z = z + return nil +} diff --git a/driver_test.go b/driver_test.go new file mode 100644 index 0000000..24d1a82 --- /dev/null +++ b/driver_test.go @@ -0,0 +1,131 @@ +/* + * Copyright 2025 Hypermode Inc. + * Licensed under the terms of the Apache License, Version 2.0 + * See the LICENSE file that accompanied this code for further details. + * + * SPDX-FileCopyrightText: 2025 Hypermode Inc. + * SPDX-License-Identifier: Apache-2.0 + */ + +package modusdb_test + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "testing" + + "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/stretchr/testify/require" + + "github.com/hypermodeinc/modusdb" +) + +func TestRestart(t *testing.T) { + dataDir := t.TempDir() + + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(dataDir)) + require.NoError(t, err) + defer func() { driver.Close() }() + + require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, driver.AlterSchema(context.Background(), "name: string @index(term) .")) + + _, err = driver.Mutate(context.Background(), []*api.Mutation{ + { + Set: []*api.NQuad{ + { + Namespace: 0, + Subject: "_:aman", + Predicate: "name", + ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, + }, + }, + }, + }) + require.NoError(t, err) + + query := `{ + me(func: has(name)) { + name + } + }` + qresp, err := driver.Query(context.Background(), query) + require.NoError(t, err) + require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) + + driver.Close() + driver, err = modusdb.NewDriver(modusdb.NewDefaultConfig(dataDir)) + require.NoError(t, err) + qresp, err = driver.Query(context.Background(), query) + require.NoError(t, err) + require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) + + require.NoError(t, driver.DropAll(context.Background())) +} + +func TestSchemaQuery(t *testing.T) { + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer driver.Close() + + require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, driver.AlterSchema(context.Background(), ` + name: string @index(exact) . + age: int . + married: bool . + loc: geo . + dob: datetime . + `)) + + resp, err := driver.Query(context.Background(), `schema(pred: [name, age]) {type}`) + require.NoError(t, err) + + require.JSONEq(t, + `{"schema":[{"predicate":"age","type":"int"},{"predicate":"name","type":"string"}]}`, + string(resp.GetJson())) +} + +func TestBasicVector(t *testing.T) { + vect := []float32{5.1, 5.1, 1.1} + buf := new(bytes.Buffer) + for _, v := range vect { + require.NoError(t, binary.Write(buf, binary.LittleEndian, v)) + } + vectBytes := buf.Bytes() + + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + require.NoError(t, err) + defer driver.Close() + + require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, driver.AlterSchema(context.Background(), + `project_description_v: float32vector @index(hnsw(exponent: "5", metric: "euclidean")) .`)) + + uids, err := driver.Mutate(context.Background(), []*api.Mutation{{ + Set: []*api.NQuad{{ + Subject: "_:vector", + Predicate: "project_description_v", + ObjectValue: &api.Value{ + Val: &api.Value_Vfloat32Val{Vfloat32Val: vectBytes}, + }, + }}, + }}) + require.NoError(t, err) + + uid := uids["_:vector"] + if uid == 0 { + t.Fatalf("Expected non-zero uid") + } + + resp, err := driver.Query(context.Background(), fmt.Sprintf(`query { + q (func: uid(%v)) { + project_description_v + } + }`, uid)) + require.NoError(t, err) + require.Equal(t, + `{"q":[{"project_description_v":[5.1E+00,5.1E+00,1.1E+00]}]}`, + string(resp.GetJson())) +} diff --git a/live.go b/live.go index d786fab..1d4f6fb 100644 --- a/live.go +++ b/live.go @@ -34,13 +34,13 @@ const ( ) type liveLoader struct { - n *Namespace + n *DB blankNodes map[string]string mutex sync.RWMutex } -func (n *Namespace) Load(ctx context.Context, schemaPath, dataPath string) error { +func (n *DB) Load(ctx context.Context, schemaPath, dataPath string) error { schemaData, err := os.ReadFile(schemaPath) if err != nil { return fmt.Errorf("error reading schema file [%v]: %w", schemaPath, err) @@ -56,7 +56,7 @@ func (n *Namespace) Load(ctx context.Context, schemaPath, dataPath string) error } // TODO: Add support for CSV file -func (n *Namespace) LoadData(inCtx context.Context, dataDir string) error { +func (n *DB) LoadData(inCtx context.Context, dataDir string) error { fs := filestore.NewFileStore(dataDir) files := fs.FindDataFiles(dataDir, []string{".rdf", ".rdf.gz", ".json", ".json.gz"}) if len(files) == 0 { @@ -246,7 +246,7 @@ func (l *liveLoader) uid(ns uint64, val string) (string, error) { return uid, nil } - asUID, err := l.n.db.LeaseUIDs(1) + asUID, err := l.n.driver.LeaseUIDs(1) if err != nil { return "", fmt.Errorf("error allocating UID: %w", err) } diff --git a/live_benchmark_test.go b/live_benchmark_test.go index 62aee86..f504a58 100644 --- a/live_benchmark_test.go +++ b/live_benchmark_test.go @@ -49,9 +49,9 @@ func BenchmarkDatabaseOperations(b *testing.B) { runtime.ReadMemStats(&ms) initialAlloc := ms.Alloc - db, err := modusdb.New(modusdb.NewDefaultConfig(b.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(b.TempDir())) require.NoError(b, err) - defer db.Close() + defer driver.Close() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -60,7 +60,7 @@ func BenchmarkDatabaseOperations(b *testing.B) { dataFile := filepath.Join(dataFolder, "data.rdf") require.NoError(b, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) require.NoError(b, os.WriteFile(dataFile, []byte(SmallData), 0600)) - require.NoError(b, db.Load(context.Background(), schemaFile, dataFile)) + require.NoError(b, driver.Load(context.Background(), schemaFile, dataFile)) } reportMemStats(b, initialAlloc) }) @@ -75,16 +75,16 @@ func BenchmarkDatabaseOperations(b *testing.B) { initialAlloc := ms.Alloc // Setup database with data once - db, err := modusdb.New(modusdb.NewDefaultConfig(b.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(b.TempDir())) require.NoError(b, err) - defer db.Close() + defer driver.Close() dataFolder := b.TempDir() schemaFile := filepath.Join(dataFolder, "data.schema") dataFile := filepath.Join(dataFolder, "data.rdf") require.NoError(b, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) require.NoError(b, os.WriteFile(dataFile, []byte(SmallData), 0600)) - require.NoError(b, db.Load(context.Background(), schemaFile, dataFile)) + require.NoError(b, driver.Load(context.Background(), schemaFile, dataFile)) const query = `{ caro(func: allofterms(name@en, "Marc Caro")) { @@ -112,7 +112,7 @@ func BenchmarkDatabaseOperations(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - resp, err := db.Query(context.Background(), query) + resp, err := driver.Query(context.Background(), query) require.NoError(b, err) require.JSONEq(b, expected, string(resp.Json)) } diff --git a/live_test.go b/live_test.go index 4b7cf88..9f99373 100644 --- a/live_test.go +++ b/live_test.go @@ -49,7 +49,7 @@ const ( func TestLiveLoaderSmall(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + db, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) defer db.Close() @@ -90,7 +90,7 @@ func TestLiveLoaderSmall(t *testing.T) { } func TestLiveLoader1Million(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + db, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) defer db.Close() diff --git a/namespace.go b/namespace.go deleted file mode 100644 index 710a1e0..0000000 --- a/namespace.go +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Copyright 2025 Hypermode Inc. - * Licensed under the terms of the Apache License, Version 2.0 - * See the LICENSE file that accompanied this code for further details. - * - * SPDX-FileCopyrightText: 2025 Hypermode Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package modusdb - -import ( - "context" - "fmt" - "strconv" - - "github.com/dgraph-io/dgo/v240/protos/api" - "github.com/dgraph-io/dgraph/v24/dql" - "github.com/dgraph-io/dgraph/v24/edgraph" - "github.com/dgraph-io/dgraph/v24/protos/pb" - "github.com/dgraph-io/dgraph/v24/query" - "github.com/dgraph-io/dgraph/v24/schema" - "github.com/dgraph-io/dgraph/v24/worker" - "github.com/dgraph-io/dgraph/v24/x" -) - -// Namespace is one of the namespaces in modusDB. -type Namespace struct { - id uint64 - db *DB -} - -func (n *Namespace) ID() uint64 { - return n.id -} - -// DropData drops all the data in the modusDB instance. -func (n *Namespace) DropData(ctx context.Context) error { - n.db.mutex.Lock() - defer n.db.mutex.Unlock() - - if !n.db.isOpen { - return ErrClosedDB - } - - p := &pb.Proposal{Mutations: &pb.Mutations{ - GroupId: 1, - DropOp: pb.Mutations_DATA, - DropValue: strconv.FormatUint(n.ID(), 10), - }} - - if err := worker.ApplyMutations(ctx, p); err != nil { - return fmt.Errorf("error applying mutation: %w", err) - } - - // TODO: insert drop record - // TODO: should we reset back the timestamp as well? - return nil -} - -func (n *Namespace) AlterSchema(ctx context.Context, sch string) error { - n.db.mutex.Lock() - defer n.db.mutex.Unlock() - - if !n.db.isOpen { - return ErrClosedDB - } - - sc, err := schema.ParseWithNamespace(sch, n.ID()) - if err != nil { - return fmt.Errorf("error parsing schema: %w", err) - } - return n.alterSchemaWithParsed(ctx, sc) -} - -func (n *Namespace) alterSchemaWithParsed(ctx context.Context, sc *schema.ParsedSchema) error { - for _, pred := range sc.Preds { - worker.InitTablet(pred.Predicate) - } - - startTs, err := n.db.z.nextTs() - if err != nil { - return err - } - - p := &pb.Proposal{Mutations: &pb.Mutations{ - GroupId: 1, - StartTs: startTs, - Schema: sc.Preds, - Types: sc.Types, - }} - if err := worker.ApplyMutations(ctx, p); err != nil { - return fmt.Errorf("error applying mutation: %w", err) - } - return nil -} - -func (n *Namespace) Mutate(ctx context.Context, ms []*api.Mutation) (map[string]uint64, error) { - if len(ms) == 0 { - return nil, nil - } - - n.db.mutex.Lock() - defer n.db.mutex.Unlock() - dms := make([]*dql.Mutation, 0, len(ms)) - for _, mu := range ms { - dm, err := edgraph.ParseMutationObject(mu, false) - if err != nil { - return nil, fmt.Errorf("error parsing mutation: %w", err) - } - dms = append(dms, dm) - } - newUids, err := query.ExtractBlankUIDs(ctx, dms) - if err != nil { - return nil, err - } - if len(newUids) > 0 { - num := &pb.Num{Val: uint64(len(newUids)), Type: pb.Num_UID} - res, err := n.db.z.nextUIDs(num) - if err != nil { - return nil, err - } - - curId := res.StartId - for k := range newUids { - x.AssertTruef(curId != 0 && curId <= res.EndId, "not enough uids generated") - newUids[k] = curId - curId++ - } - } - - return n.mutateWithDqlMutation(ctx, dms, newUids) -} - -func (n *Namespace) mutateWithDqlMutation(ctx context.Context, dms []*dql.Mutation, - newUids map[string]uint64) (map[string]uint64, error) { - edges, err := query.ToDirectedEdges(dms, newUids) - if err != nil { - return nil, err - } - ctx = x.AttachNamespace(ctx, n.ID()) - - if !n.db.isOpen { - return nil, ErrClosedDB - } - - startTs, err := n.db.z.nextTs() - if err != nil { - return nil, err - } - commitTs, err := n.db.z.nextTs() - if err != nil { - return nil, err - } - - m := &pb.Mutations{ - GroupId: 1, - StartTs: startTs, - Edges: edges, - } - m.Edges, err = query.ExpandEdges(ctx, m) - if err != nil { - return nil, fmt.Errorf("error expanding edges: %w", err) - } - - for _, edge := range m.Edges { - worker.InitTablet(edge.Attr) - } - - p := &pb.Proposal{Mutations: m, StartTs: startTs} - if err := worker.ApplyMutations(ctx, p); err != nil { - return nil, err - } - - return newUids, worker.ApplyCommited(ctx, &pb.OracleDelta{ - Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, - }) -} - -// Query performs query or mutation or upsert on the given modusDB instance. -func (n *Namespace) Query(ctx context.Context, query string) (*api.Response, error) { - n.db.mutex.RLock() - defer n.db.mutex.RUnlock() - - return n.queryWithLock(ctx, query) -} - -func (n *Namespace) queryWithLock(ctx context.Context, query string) (*api.Response, error) { - if !n.db.isOpen { - return nil, ErrClosedDB - } - - ctx = x.AttachNamespace(ctx, n.ID()) - return (&edgraph.Server{}).QueryNoAuth(ctx, &api.Request{ - ReadOnly: true, - Query: query, - StartTs: n.db.z.readTs(), - }) -} diff --git a/namespace_test.go b/namespace_test.go deleted file mode 100644 index 3f808bc..0000000 --- a/namespace_test.go +++ /dev/null @@ -1,297 +0,0 @@ -/* - * Copyright 2025 Hypermode Inc. - * Licensed under the terms of the Apache License, Version 2.0 - * See the LICENSE file that accompanied this code for further details. - * - * SPDX-FileCopyrightText: 2025 Hypermode Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package modusdb_test - -import ( - "context" - "testing" - - "github.com/dgraph-io/dgo/v240/protos/api" - "github.com/stretchr/testify/require" - - "github.com/hypermodeinc/modusdb" -) - -func TestNonGalaxyNamespace(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) - require.NoError(t, err) - defer db.Close() - - db1, err := db.CreateNamespace() - require.NoError(t, err) - - require.NoError(t, db1.DropData(context.Background())) - require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) - - _, err = db1.Mutate(context.Background(), []*api.Mutation{ - { - Set: []*api.NQuad{ - { - Subject: "_:aman", - Predicate: "name", - ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, - }, - }, - }, - }) - require.NoError(t, err) - - query := `{ - me(func: has(name)) { - name - } - }` - resp, err := db1.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) - -} - -func TestDropData(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) - require.NoError(t, err) - defer db.Close() - - db1, err := db.CreateNamespace() - require.NoError(t, err) - - require.NoError(t, db1.DropData(context.Background())) - require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) - - _, err = db1.Mutate(context.Background(), []*api.Mutation{ - { - Set: []*api.NQuad{ - { - Subject: "_:aman", - Predicate: "name", - ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, - }, - }, - }, - }) - require.NoError(t, err) - - query := `{ - me(func: has(name)) { - name - } - }` - resp, err := db1.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) - - require.NoError(t, db1.DropData(context.Background())) - - resp, err = db1.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) -} - -func TestMultipleNamespaces(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) - require.NoError(t, err) - defer db.Close() - - db0, err := db.GetNamespace(0) - require.NoError(t, err) - db1, err := db.CreateNamespace() - require.NoError(t, err) - - require.NoError(t, db.DropAll(context.Background())) - require.NoError(t, db0.AlterSchema(context.Background(), "name: string @index(exact) .")) - require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) - - _, err = db0.Mutate(context.Background(), []*api.Mutation{ - { - Set: []*api.NQuad{ - { - Subject: "_:aman", - Predicate: "name", - ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, - }, - }, - }, - }) - require.NoError(t, err) - - _, err = db1.Mutate(context.Background(), []*api.Mutation{ - { - Set: []*api.NQuad{ - { - Subject: "_:aman", - Predicate: "name", - ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "B"}}, - }, - }, - }, - }) - require.NoError(t, err) - - query := `{ - me(func: has(name)) { - name - } - }` - resp, err := db0.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) - - resp, err = db1.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[{"name":"B"}]}`, string(resp.GetJson())) - - require.NoError(t, db1.DropData(context.Background())) - resp, err = db1.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) -} - -func TestQueryWrongNamespace(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) - require.NoError(t, err) - defer db.Close() - - db0, err := db.GetNamespace(0) - require.NoError(t, err) - db1, err := db.CreateNamespace() - require.NoError(t, err) - - require.NoError(t, db.DropAll(context.Background())) - require.NoError(t, db0.AlterSchema(context.Background(), "name: string @index(exact) .")) - require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) - - _, err = db0.Mutate(context.Background(), []*api.Mutation{ - { - Set: []*api.NQuad{ - { - Namespace: 1, - Subject: "_:aman", - Predicate: "name", - ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, - }, - }, - }, - }) - require.NoError(t, err) - - query := `{ - me(func: has(name)) { - name - } - }` - - resp, err := db1.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) -} - -func TestTwoNamespaces(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) - require.NoError(t, err) - defer db.Close() - - db0, err := db.GetNamespace(0) - require.NoError(t, err) - db1, err := db.CreateNamespace() - require.NoError(t, err) - - require.NoError(t, db.DropAll(context.Background())) - require.NoError(t, db0.AlterSchema(context.Background(), "foo: string @index(exact) .")) - require.NoError(t, db1.AlterSchema(context.Background(), "bar: string @index(exact) .")) - - _, err = db0.Mutate(context.Background(), []*api.Mutation{ - { - Set: []*api.NQuad{ - { - Subject: "_:aman", - Predicate: "foo", - ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "A"}}, - }, - }, - }, - }) - require.NoError(t, err) - - _, err = db1.Mutate(context.Background(), []*api.Mutation{ - { - Set: []*api.NQuad{ - { - Subject: "_:aman", - Predicate: "bar", - ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "B"}}, - }, - }, - }, - }) - require.NoError(t, err) - - query := `{ - me(func: has(foo)) { - foo - } - }` - resp, err := db0.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[{"foo":"A"}]}`, string(resp.GetJson())) - - query = `{ - me(func: has(bar)) { - bar - } - }` - resp, err = db1.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[{"bar":"B"}]}`, string(resp.GetJson())) -} - -func TestNamespaceDBRestart(t *testing.T) { - dataDir := t.TempDir() - db, err := modusdb.New(modusdb.NewDefaultConfig(dataDir)) - require.NoError(t, err) - defer func() { db.Close() }() - - db1, err := db.CreateNamespace() - require.NoError(t, err) - ns1 := db1.ID() - - require.NoError(t, db1.AlterSchema(context.Background(), "bar: string @index(exact) .")) - _, err = db1.Mutate(context.Background(), []*api.Mutation{ - { - Set: []*api.NQuad{ - { - Subject: "_:aman", - Predicate: "bar", - ObjectValue: &api.Value{Val: &api.Value_StrVal{StrVal: "B"}}, - }, - }, - }, - }) - require.NoError(t, err) - - db.Close() - db, err = modusdb.New(modusdb.NewDefaultConfig(dataDir)) - require.NoError(t, err) - - db2, err := db.CreateNamespace() - require.NoError(t, err) - require.Greater(t, db2.ID(), ns1) - - db1, err = db.GetNamespace(ns1) - require.NoError(t, err) - - query := `{ - me(func: has(bar)) { - bar - } - }` - resp, err := db1.Query(context.Background(), query) - require.NoError(t, err) - require.JSONEq(t, `{"me":[{"bar":"B"}]}`, string(resp.GetJson())) -} diff --git a/vector_test.go b/vector_test.go index e62057d..041bb1a 100644 --- a/vector_test.go +++ b/vector_test.go @@ -29,20 +29,20 @@ const ( ) func TestVectorDelete(t *testing.T) { - db, err := modusdb.New(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() - require.NoError(t, db.DropAll(context.Background())) - require.NoError(t, db.AlterSchema(context.Background(), + require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, driver.AlterSchema(context.Background(), fmt.Sprintf(vectorSchemaWithIndex, "vtest", "4", "euclidean"))) // insert random vectors - assignIDs, err := db.LeaseUIDs(numVectors + 1) + assignIDs, err := driver.LeaseUIDs(numVectors + 1) require.NoError(t, err) //nolint:gosec rdf, vectors := dgraphapi.GenerateRandomVectors(int(assignIDs.StartId)-10, int(assignIDs.EndId)-10, 10, "vtest") - _, err = db.Mutate(context.Background(), []*api.Mutation{{SetNquads: []byte(rdf)}}) + _, err = driver.Mutate(context.Background(), []*api.Mutation{{SetNquads: []byte(rdf)}}) require.NoError(t, err) // check the count of the vectors inserted @@ -51,7 +51,7 @@ func TestVectorDelete(t *testing.T) { count(uid) } }` - resp, err := db.Query(context.Background(), q1) + resp, err := driver.Query(context.Background(), q1) require.NoError(t, err) require.JSONEq(t, fmt.Sprintf(`{"vector":[{"count":%d}]}`, numVectors), string(resp.Json)) @@ -64,11 +64,11 @@ func TestVectorDelete(t *testing.T) { } }` - require.Equal(t, vectors, queryVectors(t, db, vectorQuery)) + require.Equal(t, vectors, queryVectors(t, driver, vectorQuery)) triples := strings.Split(rdf, "\n") deleteTriple := func(idx int) string { - _, err := db.Mutate(context.Background(), []*api.Mutation{{ + _, err := driver.Mutate(context.Background(), []*api.Mutation{{ DelNquads: []byte(triples[idx]), }}) require.NoError(t, err) @@ -80,7 +80,7 @@ func TestVectorDelete(t *testing.T) { } }`, uid[1:len(uid)-1]) - res, err := db.Query(context.Background(), q2) + res, err := driver.Query(context.Background(), q2) require.NoError(t, err) require.JSONEq(t, `{"vector":[]}`, string(res.Json)) return triples[idx] @@ -96,16 +96,16 @@ func TestVectorDelete(t *testing.T) { for i := 0; i < len(triples)-2; i++ { triple := deleteTriple(i) vectorQuery := fmt.Sprintf(q3, strings.Split(triple, `"`)[1]) - respVectors := queryVectors(t, db, vectorQuery) + respVectors := queryVectors(t, driver, vectorQuery) require.Len(t, respVectors, 1) require.Contains(t, vectors, respVectors[0]) } triple := deleteTriple(len(triples) - 2) - _ = queryVectors(t, db, fmt.Sprintf(q3, strings.Split(triple, `"`)[1])) + _ = queryVectors(t, driver, fmt.Sprintf(q3, strings.Split(triple, `"`)[1])) } -func queryVectors(t *testing.T, db *modusdb.DB, query string) [][]float32 { +func queryVectors(t *testing.T, db *modusdb.Driver, query string) [][]float32 { resp, err := db.Query(context.Background(), query) require.NoError(t, err) diff --git a/zero.go b/zero.go index 130e030..33d3e44 100644 --- a/zero.go +++ b/zero.go @@ -1,7 +1,7 @@ /* * Copyright 2025 Hypermode Inc. * Licensed under the terms of the Apache License, Version 2.0 - * See the LICENSE file that accompanied this code for further details. + * See the LICEDBE file that accompanied this code for further details. * * SPDX-FileCopyrightText: 2025 Hypermode Inc. * SPDX-License-Identifier: Apache-2.0 @@ -34,7 +34,7 @@ const ( zeroStateKey = "0-dgraph.modusdb.zero" ) -func (db *DB) LeaseUIDs(numUIDs uint64) (*pb.AssignedIds, error) { +func (db *Driver) LeaseUIDs(numUIDs uint64) (*pb.AssignedIds, error) { num := &pb.Num{Val: numUIDs, Type: pb.Num_UID} return db.z.nextUIDs(num) } @@ -46,7 +46,7 @@ type zero struct { minLeasedTs uint64 maxLeasedTs uint64 - lastNS uint64 + lastDB uint64 } func newZero() (*zero, bool, error) { @@ -62,13 +62,13 @@ func newZero() (*zero, bool, error) { z.maxLeasedUID = initialUID z.minLeasedTs = initialTs z.maxLeasedTs = initialTs - z.lastNS = 0 + z.lastDB = 0 } else { z.minLeasedUID = zs.MaxUID z.maxLeasedUID = zs.MaxUID z.minLeasedTs = zs.MaxTxnTs z.maxLeasedTs = zs.MaxTxnTs - z.lastNS = zs.MaxNsID + z.lastDB = zs.MaxNsID } posting.Oracle().ProcessDelta(&pb.OracleDelta{MaxAssigned: z.minLeasedTs - 1}) worker.SetMaxUID(z.minLeasedUID - 1) @@ -133,12 +133,12 @@ func (z *zero) nextUIDs(num *pb.Num) (*pb.AssignedIds, error) { return resp, nil } -func (z *zero) nextNS() (uint64, error) { - z.lastNS++ +func (z *zero) nextDB() (uint64, error) { + z.lastDB++ if err := z.writeZeroState(); err != nil { return 0, fmt.Errorf("error leasing namespace ID: %w", err) } - return z.lastNS, nil + return z.lastDB, nil } func readZeroState() (*pb.MembershipState, error) { @@ -165,7 +165,7 @@ func readZeroState() (*pb.MembershipState, error) { } func (z *zero) writeZeroState() error { - zeroState := &pb.MembershipState{MaxUID: z.maxLeasedUID, MaxTxnTs: z.maxLeasedTs, MaxNsID: z.lastNS} + zeroState := &pb.MembershipState{MaxUID: z.maxLeasedUID, MaxTxnTs: z.maxLeasedTs, MaxNsID: z.lastDB} data, err := proto.Marshal(zeroState) if err != nil { return fmt.Errorf("error marshalling zero state: %w", err) From 392d2874ff006ca8ba8b2726cee4e02095d89bdd Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Thu, 9 Jan 2025 21:40:12 -0800 Subject: [PATCH 02/10] whoops --- api_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api_test.go b/api_test.go index 7213b2d..89ffce9 100644 --- a/api_test.go +++ b/api_test.go @@ -24,7 +24,7 @@ type User struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` Age int `json:"age,omitempty"` - ClerkId string `json:"clerk_id,omitempty" driver:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` } func TestFirstTimeUser(t *testing.T) { @@ -416,14 +416,14 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { type Project struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" driver:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` Branches []Branch `json:"branches,omitempty" readFrom:"type=Branch,field=proj"` } type Branch struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" driver:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` Proj Project `json:"proj,omitempty"` } @@ -755,7 +755,7 @@ type BadProject struct { type BadBranch struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" driver:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` Proj BadProject `json:"proj,omitempty"` } @@ -797,7 +797,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { type Document struct { Gid uint64 `json:"gid,omitempty"` Text string `json:"text,omitempty"` - TextVec []float32 `json:"textVec,omitempty" driver:"constraint=vector"` + TextVec []float32 `json:"textVec,omitempty" db:"constraint=vector"` } func TestVectorIndexSearchTyped(t *testing.T) { From a9842e586908a6b5d8cd418b02f0d5e2ae935db1 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Thu, 9 Jan 2025 22:14:14 -0800 Subject: [PATCH 03/10] refactor around drivers and dbs, drivers do the core db work, db is a logical separation --- api.go | 60 +++++++------- api_mutation_gen.go | 20 ++--- api_mutation_helpers.go | 30 +++---- api_query_execution.go | 40 +++++----- api_types.go | 20 ++--- config.go | 4 +- db.go | 163 +------------------------------------- driver.go | 170 ++++++++++++++++++++++++++++++++++++++-- driver_test.go | 18 ++--- live.go | 16 ++-- live_benchmark_test.go | 2 +- live_test.go | 18 ++--- vector_test.go | 14 ++-- 13 files changed, 287 insertions(+), 288 deletions(-) diff --git a/api.go b/api.go index ce04050..5d936c2 100644 --- a/api.go +++ b/api.go @@ -18,13 +18,13 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func Create[T any](driver *Driver, object T, ns ...uint64) (uint64, T, error) { +func Create[T any](driver *Driver, object T, dbId ...uint64) (uint64, T, error) { driver.mutex.Lock() defer driver.mutex.Unlock() - if len(ns) > 1 { + if len(dbId) > 1 { return 0, object, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(driver, ns...) + ctx, db, err := getDefaultDB(driver, dbId...) if err != nil { return 0, object, err } @@ -36,12 +36,12 @@ func Create[T any](driver *Driver, object T, ns ...uint64) (uint64, T, error) { dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateSetDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema[T](ctx, db, object, gid, &dms, sch) if err != nil { return 0, object, err } - err = n.alterSchemaWithParsed(ctx, sch) + err = driver.alterSchemaWithParsed(ctx, sch) if err != nil { return 0, object, err } @@ -51,19 +51,19 @@ func Create[T any](driver *Driver, object T, ns ...uint64) (uint64, T, error) { return 0, object, err } - return getByGid[T](ctx, n, gid) + return getByGid[T](ctx, db, gid) } -func Upsert[T any](driver *Driver, object T, ns ...uint64) (uint64, T, bool, error) { +func Upsert[T any](driver *Driver, object T, dbId ...uint64) (uint64, T, bool, error) { var wasFound bool driver.mutex.Lock() defer driver.mutex.Unlock() - if len(ns) > 1 { + if len(dbId) > 1 { return 0, object, false, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(driver, ns...) + ctx, db, err := getDefaultDB(driver, dbId...) if err != nil { return 0, object, false, err } @@ -82,18 +82,18 @@ func Upsert[T any](driver *Driver, object T, ns ...uint64) (uint64, T, bool, err dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateSetDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema[T](ctx, db, object, gid, &dms, sch) if err != nil { return 0, object, false, err } - err = n.alterSchemaWithParsed(ctx, sch) + err = db.driver.alterSchemaWithParsed(ctx, sch) if err != nil { return 0, object, false, err } if gid != 0 || cf != nil { - gid, err = getExistingObject[T](ctx, n, gid, cf, object) + gid, err = getExistingObject[T](ctx, db, gid, cf, object) if err != nil && err != apiutils.ErrNoObjFound { return 0, object, false, err } @@ -108,7 +108,7 @@ func Upsert[T any](driver *Driver, object T, ns ...uint64) (uint64, T, bool, err } dms = make([]*dql.Mutation, 0) - err = generateSetDqlMutationsAndSchema[T](ctx, n, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema[T](ctx, db, object, gid, &dms, sch) if err != nil { return 0, object, false, err } @@ -118,7 +118,7 @@ func Upsert[T any](driver *Driver, object T, ns ...uint64) (uint64, T, bool, err return 0, object, false, err } - gid, object, err = getByGid[T](ctx, n, gid) + gid, object, err = getByGid[T](ctx, db, gid) if err != nil { return 0, object, false, err } @@ -126,60 +126,60 @@ func Upsert[T any](driver *Driver, object T, ns ...uint64) (uint64, T, bool, err return gid, object, wasFound, nil } -func Get[T any, R UniqueField](driver *Driver, uniqueField R, ns ...uint64) (uint64, T, error) { +func Get[T any, R UniqueField](driver *Driver, uniqueField R, dbId ...uint64) (uint64, T, error) { driver.mutex.Lock() defer driver.mutex.Unlock() var obj T - if len(ns) > 1 { + if len(dbId) > 1 { return 0, obj, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(driver, ns...) + ctx, db, err := getDefaultDB(driver, dbId...) if err != nil { return 0, obj, err } if uid, ok := any(uniqueField).(uint64); ok { - return getByGid[T](ctx, n, uid) + return getByGid[T](ctx, db, uid) } if cf, ok := any(uniqueField).(ConstrainedField); ok { - return getByConstrainedField[T](ctx, n, cf) + return getByConstrainedField[T](ctx, db, cf) } return 0, obj, fmt.Errorf("invalid unique field type") } -func Query[T any](driver *Driver, queryParams QueryParams, ns ...uint64) ([]uint64, []T, error) { +func Query[T any](driver *Driver, queryParams QueryParams, dbId ...uint64) ([]uint64, []T, error) { driver.mutex.Lock() defer driver.mutex.Unlock() - if len(ns) > 1 { + if len(dbId) > 1 { return nil, nil, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(driver, ns...) + ctx, db, err := getDefaultDB(driver, dbId...) if err != nil { return nil, nil, err } - return executeQuery[T](ctx, n, queryParams, true) + return executeQuery[T](ctx, db, queryParams, true) } -func Delete[T any, R UniqueField](driver *Driver, uniqueField R, ns ...uint64) (uint64, T, error) { +func Delete[T any, R UniqueField](driver *Driver, uniqueField R, dbId ...uint64) (uint64, T, error) { driver.mutex.Lock() defer driver.mutex.Unlock() var zeroObj T - if len(ns) > 1 { + if len(dbId) > 1 { return 0, zeroObj, fmt.Errorf("only one namespace is allowed") } - ctx, n, err := getDefaultNamespace(driver, ns...) + ctx, db, err := getDefaultDB(driver, dbId...) if err != nil { return 0, zeroObj, err } if uid, ok := any(uniqueField).(uint64); ok { - uid, obj, err := getByGid[T](ctx, n, uid) + uid, obj, err := getByGid[T](ctx, db, uid) if err != nil { return 0, zeroObj, err } - dms := generateDeleteDqlMutations(n, uid) + dms := generateDeleteDqlMutations(db, uid) err = applyDqlMutations(ctx, driver, dms) if err != nil { @@ -190,12 +190,12 @@ func Delete[T any, R UniqueField](driver *Driver, uniqueField R, ns ...uint64) ( } if cf, ok := any(uniqueField).(ConstrainedField); ok { - uid, obj, err := getByConstrainedField[T](ctx, n, cf) + uid, obj, err := getByConstrainedField[T](ctx, db, cf) if err != nil { return 0, zeroObj, err } - dms := generateDeleteDqlMutations(n, uid) + dms := generateDeleteDqlMutations(db, uid) err = applyDqlMutations(ctx, driver, dms) if err != nil { diff --git a/api_mutation_gen.go b/api_mutation_gen.go index c2a304b..1fe4c45 100644 --- a/api_mutation_gen.go +++ b/api_mutation_gen.go @@ -26,7 +26,7 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *DB, object T, +func generateSetDqlMutationsAndSchema[T any](ctx context.Context, d *DB, object T, gid uint64, dms *[]*dql.Mutation, sch *schema.ParsedSchema) error { t := reflect.TypeOf(object) if t.Kind() != reflect.Struct { @@ -49,7 +49,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *DB, object if tagMaps.JsonToReverseEdge[jsonName] != "" { reverseEdgeStr := tagMaps.JsonToReverseEdge[jsonName] typeName := strings.Split(reverseEdgeStr, ".")[0] - currSchema, err := getSchema(ctx, n) + currSchema, err := getSchema(ctx, d) if err != nil { return err } @@ -70,7 +70,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *DB, object } if !(typeFound && predicateFound) { - if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.ID(), sch, + if err := mutations.HandleReverseEdge(jsonName, reflectValueType, d.ID(), sch, reverseEdgeStr); err != nil { return err } @@ -82,17 +82,17 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *DB, object continue } - value, err = processStructValue(ctx, value, n) + value, err = processStructValue(ctx, value, d) if err != nil { return err } - value, err = processPointerValue(ctx, value, n) + value, err = processPointerValue(ctx, value, d) if err != nil { return err } - nquad, u, err := mutations.CreateNQuadAndSchema(value, gid, jsonName, t, n.ID()) + nquad, u, err := mutations.CreateNQuadAndSchema(value, gid, jsonName, t, d.ID()) if err != nil { return err } @@ -111,7 +111,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *DB, object } sch.Types = append(sch.Types, &pb.TypeUpdate{ - TypeName: apiutils.AddNamespace(n.ID(), t.Name()), + TypeName: apiutils.AddNamespace(d.ID(), t.Name()), Fields: sch.Preds, }) @@ -120,7 +120,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *DB, object return err } typeNquad := &api.NQuad{ - Namespace: n.ID(), + Namespace: d.ID(), Subject: fmt.Sprint(gid), Predicate: "dgraph.type", ObjectValue: val, @@ -134,11 +134,11 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *DB, object return nil } -func generateDeleteDqlMutations(n *DB, gid uint64) []*dql.Mutation { +func generateDeleteDqlMutations(d *DB, gid uint64) []*dql.Mutation { return []*dql.Mutation{{ Del: []*api.NQuad{ { - Namespace: n.ID(), + Namespace: d.ID(), Subject: fmt.Sprint(gid), Predicate: x.Star, ObjectValue: &api.Value{ diff --git a/api_mutation_helpers.go b/api_mutation_helpers.go index bcb1bf6..4a1f30b 100644 --- a/api_mutation_helpers.go +++ b/api_mutation_helpers.go @@ -14,10 +14,10 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func processStructValue(ctx context.Context, value any, n *DB) (any, error) { +func processStructValue(ctx context.Context, value any, db *DB) (any, error) { if reflect.TypeOf(value).Kind() == reflect.Struct { value = reflect.ValueOf(value).Interface() - newGid, err := getUidOrMutate(ctx, n.driver, n, value) + newGid, err := getUidOrMutate(ctx, db.driver, db, value) if err != nil { return nil, err } @@ -26,19 +26,19 @@ func processStructValue(ctx context.Context, value any, n *DB) (any, error) { return value, nil } -func processPointerValue(ctx context.Context, value any, n *DB) (any, error) { +func processPointerValue(ctx context.Context, value any, db *DB) (any, error) { reflectValueType := reflect.TypeOf(value) if reflectValueType.Kind() == reflect.Pointer { reflectValueType = reflectValueType.Elem() if reflectValueType.Kind() == reflect.Struct { value = reflect.ValueOf(value).Elem().Interface() - return processStructValue(ctx, value, n) + return processStructValue(ctx, value, db) } } return value, nil } -func getUidOrMutate[T any](ctx context.Context, db *Driver, n *DB, object T) (uint64, error) { +func getUidOrMutate[T any](ctx context.Context, driver *Driver, db *DB, object T) (uint64, error) { gid, cfKeyValue, err := structreflect.GetUniqueConstraint[T](object) if err != nil { return 0, err @@ -50,17 +50,17 @@ func getUidOrMutate[T any](ctx context.Context, db *Driver, n *DB, object T) (ui dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateSetDqlMutationsAndSchema(ctx, n, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema(ctx, db, object, gid, &dms, sch) if err != nil { return 0, err } - err = n.alterSchemaWithParsed(ctx, sch) + err = driver.alterSchemaWithParsed(ctx, sch) if err != nil { return 0, err } if gid != 0 || cf != nil { - gid, err = getExistingObject(ctx, n, gid, cf, object) + gid, err = getExistingObject(ctx, db, gid, cf, object) if err != nil && err != apiutils.ErrNoObjFound { return 0, err } @@ -69,18 +69,18 @@ func getUidOrMutate[T any](ctx context.Context, db *Driver, n *DB, object T) (ui } } - gid, err = db.z.nextUID() + gid, err = driver.z.nextUID() if err != nil { return 0, err } dms = make([]*dql.Mutation, 0) - err = generateSetDqlMutationsAndSchema(ctx, n, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema(ctx, db, object, gid, &dms, sch) if err != nil { return 0, err } - err = applyDqlMutations(ctx, db, dms) + err = applyDqlMutations(ctx, driver, dms) if err != nil { return 0, err } @@ -88,21 +88,21 @@ func getUidOrMutate[T any](ctx context.Context, db *Driver, n *DB, object T) (ui return gid, nil } -func applyDqlMutations(ctx context.Context, db *Driver, dms []*dql.Mutation) error { +func applyDqlMutations(ctx context.Context, driver *Driver, dms []*dql.Mutation) error { edges, err := query.ToDirectedEdges(dms, nil) if err != nil { return err } - if !db.isOpen.Load() { + if !driver.isOpen.Load() { return ErrClosedDriver } - startTs, err := db.z.nextTs() + startTs, err := driver.z.nextTs() if err != nil { return err } - commitTs, err := db.z.nextTs() + commitTs, err := driver.z.nextTs() if err != nil { return err } diff --git a/api_query_execution.go b/api_query_execution.go index 2cd8ffb..421b648 100644 --- a/api_query_execution.go +++ b/api_query_execution.go @@ -20,34 +20,34 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func getByGid[T any](ctx context.Context, n *DB, gid uint64) (uint64, T, error) { - return executeGet[T](ctx, n, gid) +func getByGid[T any](ctx context.Context, db *DB, gid uint64) (uint64, T, error) { + return executeGet[T](ctx, db, gid) } -func getByGidWithObject[T any](ctx context.Context, n *DB, gid uint64, obj T) (uint64, T, error) { - return executeGetWithObject[T](ctx, n, obj, false, gid) +func getByGidWithObject[T any](ctx context.Context, db *DB, gid uint64, obj T) (uint64, T, error) { + return executeGetWithObject[T](ctx, db, obj, false, gid) } -func getByConstrainedField[T any](ctx context.Context, n *DB, cf ConstrainedField) (uint64, T, error) { - return executeGet[T](ctx, n, cf) +func getByConstrainedField[T any](ctx context.Context, db *DB, cf ConstrainedField) (uint64, T, error) { + return executeGet[T](ctx, db, cf) } -func getByConstrainedFieldWithObject[T any](ctx context.Context, n *DB, +func getByConstrainedFieldWithObject[T any](ctx context.Context, db *DB, cf ConstrainedField, obj T) (uint64, T, error) { - return executeGetWithObject[T](ctx, n, obj, false, cf) + return executeGetWithObject[T](ctx, db, obj, false, cf) } -func executeGet[T any, R UniqueField](ctx context.Context, n *DB, args ...R) (uint64, T, error) { +func executeGet[T any, R UniqueField](ctx context.Context, db *DB, args ...R) (uint64, T, error) { var obj T if len(args) != 1 { - return 0, obj, fmt.Errorf("expected 1 argument, got %d", len(args)) + return 0, obj, fmt.Errorf("expected 1 argument, got %db", len(args)) } - return executeGetWithObject(ctx, n, obj, true, args...) + return executeGetWithObject(ctx, db, obj, true, args...) } -func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *DB, +func executeGetWithObject[T any, R UniqueField](ctx context.Context, db *DB, obj T, withReverse bool, args ...R) (uint64, T, error) { t := reflect.TypeOf(obj) @@ -79,7 +79,7 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *DB, return 0, obj, fmt.Errorf("constraint not defined for field %s", cf.Key) } - resp, err := n.queryWithLock(ctx, query) + resp, err := db.driver.queryWithLock(ctx, db, query) if err != nil { return 0, obj, err } @@ -107,7 +107,7 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, n *DB, return structreflect.ConvertDynamicToTyped[T](result.Obj[0], t) } -func executeQuery[T any](ctx context.Context, n *DB, queryParams QueryParams, +func executeQuery[T any](ctx context.Context, db *DB, queryParams QueryParams, withReverse bool) ([]uint64, []T, error) { var obj T t := reflect.TypeOf(obj) @@ -143,7 +143,7 @@ func executeQuery[T any](ctx context.Context, n *DB, queryParams QueryParams, query := querygen.FormatObjsQuery(t.Name(), filterQueryFunc, paginationAndSorting, readFromQuery) - resp, err := n.queryWithLock(ctx, query) + resp, err := db.driver.queryWithLock(ctx, db, query) if err != nil { return nil, nil, err } @@ -186,13 +186,13 @@ func executeQuery[T any](ctx context.Context, n *DB, queryParams QueryParams, return gids, objs, nil } -func getExistingObject[T any](ctx context.Context, n *DB, gid uint64, cf *ConstrainedField, +func getExistingObject[T any](ctx context.Context, db *DB, gid uint64, cf *ConstrainedField, object T) (uint64, error) { var err error if gid != 0 { - gid, _, err = getByGidWithObject[T](ctx, n, gid, object) + gid, _, err = getByGidWithObject[T](ctx, db, gid, object) } else if cf != nil { - gid, _, err = getByConstrainedFieldWithObject[T](ctx, n, *cf, object) + gid, _, err = getByConstrainedFieldWithObject[T](ctx, db, *cf, object) } if err != nil { return 0, err @@ -200,8 +200,8 @@ func getExistingObject[T any](ctx context.Context, n *DB, gid uint64, cf *Constr return gid, nil } -func getSchema(ctx context.Context, n *DB) (*querygen.SchemaResponse, error) { - resp, err := n.queryWithLock(ctx, querygen.SchemaQuery) +func getSchema(ctx context.Context, db *DB) (*querygen.SchemaResponse, error) { + resp, err := db.driver.queryWithLock(ctx, db, querygen.SchemaQuery) if err != nil { return nil, err } diff --git a/api_types.go b/api_types.go index 010dd79..37aae5d 100644 --- a/api_types.go +++ b/api_types.go @@ -75,32 +75,32 @@ type VectorPredicate struct { type ModusDbOption func(*modusDbOptions) type modusDbOptions struct { - namespace uint64 + db uint64 } -func WithNamespace(namespace uint64) ModusDbOption { +func WithDB(db uint64) ModusDbOption { return func(o *modusDbOptions) { - o.namespace = namespace + o.db = db } } -func getDefaultNamespace(db *Driver, ns ...uint64) (context.Context, *DB, error) { +func getDefaultDB(driver *Driver, dbId ...uint64) (context.Context, *DB, error) { dbOpts := &modusDbOptions{ - namespace: db.db0.ID(), + db: driver.db0.ID(), } - for _, ns := range ns { - WithNamespace(ns)(dbOpts) + for _, db := range dbId { + WithDB(db)(dbOpts) } - n, err := db.getDBWithLock(dbOpts.namespace) + d, err := driver.getDBWithLock(dbOpts.db) if err != nil { return nil, nil, err } ctx := context.Background() - ctx = x.AttachNamespace(ctx, n.ID()) + ctx = x.AttachNamespace(ctx, d.ID()) - return ctx, n, nil + return ctx, d, nil } func filterToQueryFunc(typeName string, f Filter) querygen.QueryFunc { diff --git a/config.go b/config.go index 76d3d1b..0eac913 100644 --- a/config.go +++ b/config.go @@ -20,8 +20,8 @@ func NewDefaultConfig(dir string) Config { return Config{dataDir: dir, limitNormalizeNode: 10000} } -func (cc Config) WithLimitNormalizeNode(n int) Config { - cc.limitNormalizeNode = n +func (cc Config) WithLimitNormalizeNode(d int) Config { + cc.limitNormalizeNode = d return cc } diff --git a/db.go b/db.go index a50b58d..6d18cb6 100644 --- a/db.go +++ b/db.go @@ -11,17 +11,8 @@ package modusdb import ( "context" - "fmt" - "strconv" "github.com/dgraph-io/dgo/v240/protos/api" - "github.com/dgraph-io/dgraph/v24/dql" - "github.com/dgraph-io/dgraph/v24/edgraph" - "github.com/dgraph-io/dgraph/v24/protos/pb" - "github.com/dgraph-io/dgraph/v24/query" - "github.com/dgraph-io/dgraph/v24/schema" - "github.com/dgraph-io/dgraph/v24/worker" - "github.com/dgraph-io/dgraph/v24/x" ) // DB is one of the namespaces in modusDB. @@ -36,164 +27,18 @@ func (db *DB) ID() uint64 { // DropData drops all the data in the modusDB instance. func (db *DB) DropData(ctx context.Context) error { - db.driver.mutex.Lock() - defer db.driver.mutex.Unlock() - - if !db.driver.isOpen.Load() { - return ErrClosedDriver - } - - p := &pb.Proposal{Mutations: &pb.Mutations{ - GroupId: 1, - DropOp: pb.Mutations_DATA, - DropValue: strconv.FormatUint(db.ID(), 10), - }} - - if err := worker.ApplyMutations(ctx, p); err != nil { - return fmt.Errorf("error applying mutation: %w", err) - } - - // TODO: insert drop record - // TODO: should we reset back the timestamp as well? - return nil + return db.driver.dropData(ctx, db) } func (db *DB) AlterSchema(ctx context.Context, sch string) error { - db.driver.mutex.Lock() - defer db.driver.mutex.Unlock() - - if !db.driver.isOpen.Load() { - return ErrClosedDriver - } - - sc, err := schema.ParseWithNamespace(sch, db.ID()) - if err != nil { - return fmt.Errorf("error parsing schema: %w", err) - } - return db.alterSchemaWithParsed(ctx, sc) -} - -func (db *DB) alterSchemaWithParsed(ctx context.Context, sc *schema.ParsedSchema) error { - for _, pred := range sc.Preds { - worker.InitTablet(pred.Predicate) - } - - startTs, err := db.driver.z.nextTs() - if err != nil { - return err - } - - p := &pb.Proposal{Mutations: &pb.Mutations{ - GroupId: 1, - StartTs: startTs, - Schema: sc.Preds, - Types: sc.Types, - }} - if err := worker.ApplyMutations(ctx, p); err != nil { - return fmt.Errorf("error applying mutation: %w", err) - } - return nil + return db.driver.alterSchema(ctx, db, sch) } func (db *DB) Mutate(ctx context.Context, ms []*api.Mutation) (map[string]uint64, error) { - if len(ms) == 0 { - return nil, nil - } - - db.driver.mutex.Lock() - defer db.driver.mutex.Unlock() - dms := make([]*dql.Mutation, 0, len(ms)) - for _, mu := range ms { - dm, err := edgraph.ParseMutationObject(mu, false) - if err != nil { - return nil, fmt.Errorf("error parsing mutation: %w", err) - } - dms = append(dms, dm) - } - newUids, err := query.ExtractBlankUIDs(ctx, dms) - if err != nil { - return nil, err - } - if len(newUids) > 0 { - num := &pb.Num{Val: uint64(len(newUids)), Type: pb.Num_UID} - res, err := db.driver.z.nextUIDs(num) - if err != nil { - return nil, err - } - - curId := res.StartId - for k := range newUids { - x.AssertTruef(curId != 0 && curId <= res.EndId, "not enough uids generated") - newUids[k] = curId - curId++ - } - } - - return db.mutateWithDqlMutation(ctx, dms, newUids) -} - -func (db *DB) mutateWithDqlMutation(ctx context.Context, dms []*dql.Mutation, - newUids map[string]uint64) (map[string]uint64, error) { - edges, err := query.ToDirectedEdges(dms, newUids) - if err != nil { - return nil, err - } - ctx = x.AttachNamespace(ctx, db.ID()) - - if !db.driver.isOpen.Load() { - return nil, ErrClosedDriver - } - - startTs, err := db.driver.z.nextTs() - if err != nil { - return nil, err - } - commitTs, err := db.driver.z.nextTs() - if err != nil { - return nil, err - } - - m := &pb.Mutations{ - GroupId: 1, - StartTs: startTs, - Edges: edges, - } - m.Edges, err = query.ExpandEdges(ctx, m) - if err != nil { - return nil, fmt.Errorf("error expanding edges: %w", err) - } - - for _, edge := range m.Edges { - worker.InitTablet(edge.Attr) - } - - p := &pb.Proposal{Mutations: m, StartTs: startTs} - if err := worker.ApplyMutations(ctx, p); err != nil { - return nil, err - } - - return newUids, worker.ApplyCommited(ctx, &pb.OracleDelta{ - Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, - }) + return db.driver.mutate(ctx, db, ms) } // Query performs query or mutation or upsert on the given modusDB instance. func (db *DB) Query(ctx context.Context, query string) (*api.Response, error) { - db.driver.mutex.RLock() - defer db.driver.mutex.RUnlock() - - return db.queryWithLock(ctx, query) -} - -func (db *DB) queryWithLock(ctx context.Context, query string) (*api.Response, error) { - if !db.driver.isOpen.Load() { - return nil, ErrClosedDriver - } - - ctx = x.AttachNamespace(ctx, db.ID()) - return (&edgraph.Server{}).QueryNoAuth(ctx, &api.Request{ - ReadOnly: true, - Query: query, - StartTs: db.driver.z.readTs(), - }) + return db.driver.query(ctx, db, query) } diff --git a/driver.go b/driver.go index 6a29a01..a1e437f 100644 --- a/driver.go +++ b/driver.go @@ -14,14 +14,17 @@ import ( "errors" "fmt" "path" + "strconv" "sync" "sync/atomic" "github.com/dgraph-io/badger/v4" "github.com/dgraph-io/dgo/v240/protos/api" + "github.com/dgraph-io/dgraph/v24/dql" "github.com/dgraph-io/dgraph/v24/edgraph" "github.com/dgraph-io/dgraph/v24/posting" "github.com/dgraph-io/dgraph/v24/protos/pb" + "github.com/dgraph-io/dgraph/v24/query" "github.com/dgraph-io/dgraph/v24/schema" "github.com/dgraph-io/dgraph/v24/worker" "github.com/dgraph-io/dgraph/v24/x" @@ -139,6 +142,10 @@ func (driver *Driver) getDBWithLock(dbID uint64) (*DB, error) { return &DB{id: dbID, driver: driver}, nil } +func (driver *Driver) GetDefaultDB() *DB { + return driver.db0 +} + // DropAll drops all the data and schema in the modusDB instance. func (driver *Driver) DropAll(ctx context.Context) error { driver.mutex.Lock() @@ -163,20 +170,167 @@ func (driver *Driver) DropAll(ctx context.Context) error { return nil } -func (driver *Driver) DropData(ctx context.Context) error { - return driver.db0.DropData(ctx) +func (driver *Driver) dropData(ctx context.Context, db *DB) error { + driver.mutex.Lock() + defer driver.mutex.Unlock() + + if !driver.isOpen.Load() { + return ErrClosedDriver + } + + p := &pb.Proposal{Mutations: &pb.Mutations{ + GroupId: 1, + DropOp: pb.Mutations_DATA, + DropValue: strconv.FormatUint(db.ID(), 10), + }} + + if err := worker.ApplyMutations(ctx, p); err != nil { + return fmt.Errorf("error applying mutation: %w", err) + } + + // TODO: insert drop record + // TODO: should we reset back the timestamp as well? + return nil +} + +func (driver *Driver) alterSchema(ctx context.Context, db *DB, sch string) error { + driver.mutex.Lock() + defer driver.mutex.Unlock() + + if !driver.isOpen.Load() { + return ErrClosedDriver + } + + sc, err := schema.ParseWithNamespace(sch, db.ID()) + if err != nil { + return fmt.Errorf("error parsing schema: %w", err) + } + return driver.alterSchemaWithParsed(ctx, sc) +} + +func (driver *Driver) alterSchemaWithParsed(ctx context.Context, sc *schema.ParsedSchema) error { + for _, pred := range sc.Preds { + worker.InitTablet(pred.Predicate) + } + + startTs, err := driver.z.nextTs() + if err != nil { + return err + } + + p := &pb.Proposal{Mutations: &pb.Mutations{ + GroupId: 1, + StartTs: startTs, + Schema: sc.Preds, + Types: sc.Types, + }} + if err := worker.ApplyMutations(ctx, p); err != nil { + return fmt.Errorf("error applying mutation: %w", err) + } + return nil +} + +func (driver *Driver) query(ctx context.Context, db *DB, q string) (*api.Response, error) { + driver.mutex.RLock() + defer driver.mutex.RUnlock() + + return driver.queryWithLock(ctx, db, q) } -func (driver *Driver) AlterSchema(ctx context.Context, sch string) error { - return driver.db0.AlterSchema(ctx, sch) +func (driver *Driver) queryWithLock(ctx context.Context, db *DB, q string) (*api.Response, error) { + if !driver.isOpen.Load() { + return nil, ErrClosedDriver + } + + ctx = x.AttachNamespace(ctx, db.ID()) + return (&edgraph.Server{}).QueryNoAuth(ctx, &api.Request{ + ReadOnly: true, + Query: q, + StartTs: driver.z.readTs(), + }) } -func (driver *Driver) Query(ctx context.Context, q string) (*api.Response, error) { - return driver.db0.Query(ctx, q) +func (driver *Driver) mutate(ctx context.Context, db *DB, ms []*api.Mutation) (map[string]uint64, error) { + if len(ms) == 0 { + return nil, nil + } + + driver.mutex.Lock() + defer driver.mutex.Unlock() + dms := make([]*dql.Mutation, 0, len(ms)) + for _, mu := range ms { + dm, err := edgraph.ParseMutationObject(mu, false) + if err != nil { + return nil, fmt.Errorf("error parsing mutation: %w", err) + } + dms = append(dms, dm) + } + newUids, err := query.ExtractBlankUIDs(ctx, dms) + if err != nil { + return nil, err + } + if len(newUids) > 0 { + num := &pb.Num{Val: uint64(len(newUids)), Type: pb.Num_UID} + res, err := driver.z.nextUIDs(num) + if err != nil { + return nil, err + } + + curId := res.StartId + for k := range newUids { + x.AssertTruef(curId != 0 && curId <= res.EndId, "not enough uids generated") + newUids[k] = curId + curId++ + } + } + + return driver.mutateWithDqlMutation(ctx, db, dms, newUids) } -func (driver *Driver) Mutate(ctx context.Context, ms []*api.Mutation) (map[string]uint64, error) { - return driver.db0.Mutate(ctx, ms) +func (driver *Driver) mutateWithDqlMutation(ctx context.Context, db *DB, dms []*dql.Mutation, + newUids map[string]uint64) (map[string]uint64, error) { + edges, err := query.ToDirectedEdges(dms, newUids) + if err != nil { + return nil, fmt.Errorf("error converting to directed edges: %w", err) + } + ctx = x.AttachNamespace(ctx, db.ID()) + + if !driver.isOpen.Load() { + return nil, ErrClosedDriver + } + + startTs, err := driver.z.nextTs() + if err != nil { + return nil, err + } + commitTs, err := driver.z.nextTs() + if err != nil { + return nil, err + } + + m := &pb.Mutations{ + GroupId: 1, + StartTs: startTs, + Edges: edges, + } + + m.Edges, err = query.ExpandEdges(ctx, m) + if err != nil { + return nil, fmt.Errorf("error expanding edges: %w", err) + } + + for _, edge := range m.Edges { + worker.InitTablet(edge.Attr) + } + + p := &pb.Proposal{Mutations: m, StartTs: startTs} + if err := worker.ApplyMutations(ctx, p); err != nil { + return nil, err + } + + return newUids, worker.ApplyCommited(ctx, &pb.OracleDelta{ + Txns: []*pb.TxnStatus{{StartTs: startTs, CommitTs: commitTs}}, + }) } func (driver *Driver) Load(ctx context.Context, schemaPath, dataPath string) error { diff --git a/driver_test.go b/driver_test.go index 24d1a82..26dcfaa 100644 --- a/driver_test.go +++ b/driver_test.go @@ -30,9 +30,9 @@ func TestRestart(t *testing.T) { defer func() { driver.Close() }() require.NoError(t, driver.DropAll(context.Background())) - require.NoError(t, driver.AlterSchema(context.Background(), "name: string @index(term) .")) + require.NoError(t, driver.GetDefaultDB().AlterSchema(context.Background(), "name: string @index(term) .")) - _, err = driver.Mutate(context.Background(), []*api.Mutation{ + _, err = driver.GetDefaultDB().Mutate(context.Background(), []*api.Mutation{ { Set: []*api.NQuad{ { @@ -51,14 +51,14 @@ func TestRestart(t *testing.T) { name } }` - qresp, err := driver.Query(context.Background(), query) + qresp, err := driver.GetDefaultDB().Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) driver.Close() driver, err = modusdb.NewDriver(modusdb.NewDefaultConfig(dataDir)) require.NoError(t, err) - qresp, err = driver.Query(context.Background(), query) + qresp, err = driver.GetDefaultDB().Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) @@ -71,7 +71,7 @@ func TestSchemaQuery(t *testing.T) { defer driver.Close() require.NoError(t, driver.DropAll(context.Background())) - require.NoError(t, driver.AlterSchema(context.Background(), ` + require.NoError(t, driver.GetDefaultDB().AlterSchema(context.Background(), ` name: string @index(exact) . age: int . married: bool . @@ -79,7 +79,7 @@ func TestSchemaQuery(t *testing.T) { dob: datetime . `)) - resp, err := driver.Query(context.Background(), `schema(pred: [name, age]) {type}`) + resp, err := driver.GetDefaultDB().Query(context.Background(), `schema(pred: [name, age]) {type}`) require.NoError(t, err) require.JSONEq(t, @@ -100,10 +100,10 @@ func TestBasicVector(t *testing.T) { defer driver.Close() require.NoError(t, driver.DropAll(context.Background())) - require.NoError(t, driver.AlterSchema(context.Background(), + require.NoError(t, driver.GetDefaultDB().AlterSchema(context.Background(), `project_description_v: float32vector @index(hnsw(exponent: "5", metric: "euclidean")) .`)) - uids, err := driver.Mutate(context.Background(), []*api.Mutation{{ + uids, err := driver.GetDefaultDB().Mutate(context.Background(), []*api.Mutation{{ Set: []*api.NQuad{{ Subject: "_:vector", Predicate: "project_description_v", @@ -119,7 +119,7 @@ func TestBasicVector(t *testing.T) { t.Fatalf("Expected non-zero uid") } - resp, err := driver.Query(context.Background(), fmt.Sprintf(`query { + resp, err := driver.GetDefaultDB().Query(context.Background(), fmt.Sprintf(`query { q (func: uid(%v)) { project_description_v } diff --git a/live.go b/live.go index 1d4f6fb..2bfea22 100644 --- a/live.go +++ b/live.go @@ -34,29 +34,29 @@ const ( ) type liveLoader struct { - n *DB + d *DB blankNodes map[string]string mutex sync.RWMutex } -func (n *DB) Load(ctx context.Context, schemaPath, dataPath string) error { +func (d *DB) Load(ctx context.Context, schemaPath, dataPath string) error { schemaData, err := os.ReadFile(schemaPath) if err != nil { return fmt.Errorf("error reading schema file [%v]: %w", schemaPath, err) } - if err := n.AlterSchema(ctx, string(schemaData)); err != nil { + if err := d.AlterSchema(ctx, string(schemaData)); err != nil { return fmt.Errorf("error altering schema: %w", err) } - if err := n.LoadData(ctx, dataPath); err != nil { + if err := d.LoadData(ctx, dataPath); err != nil { return fmt.Errorf("error loading data: %w", err) } return nil } // TODO: Add support for CSV file -func (n *DB) LoadData(inCtx context.Context, dataDir string) error { +func (d *DB) LoadData(inCtx context.Context, dataDir string) error { fs := filestore.NewFileStore(dataDir) files := fs.FindDataFiles(dataDir, []string{".rdf", ".rdf.gz", ".json", ".json.gz"}) if len(files) == 0 { @@ -94,7 +94,7 @@ func (n *DB) LoadData(inCtx context.Context, dataDir string) error { if !ok { return nil } - uids, err := n.Mutate(rootCtx, []*api.Mutation{nqs}) + uids, err := d.Mutate(rootCtx, []*api.Mutation{nqs}) if err != nil { return fmt.Errorf("error applying mutations: %w", err) } @@ -104,7 +104,7 @@ func (n *DB) LoadData(inCtx context.Context, dataDir string) error { } }) - ll := &liveLoader{n: n, blankNodes: make(map[string]string)} + ll := &liveLoader{d: d, blankNodes: make(map[string]string)} for _, datafile := range files { procG.Go(func() error { return ll.processFile(procCtx, fs, datafile, nqch) @@ -246,7 +246,7 @@ func (l *liveLoader) uid(ns uint64, val string) (string, error) { return uid, nil } - asUID, err := l.n.driver.LeaseUIDs(1) + asUID, err := l.d.driver.LeaseUIDs(1) if err != nil { return "", fmt.Errorf("error allocating UID: %w", err) } diff --git a/live_benchmark_test.go b/live_benchmark_test.go index f504a58..036862d 100644 --- a/live_benchmark_test.go +++ b/live_benchmark_test.go @@ -112,7 +112,7 @@ func BenchmarkDatabaseOperations(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - resp, err := driver.Query(context.Background(), query) + resp, err := driver.GetDefaultDB().Query(context.Background(), query) require.NoError(b, err) require.JSONEq(b, expected, string(resp.Json)) } diff --git a/live_test.go b/live_test.go index 9f99373..62a2731 100644 --- a/live_test.go +++ b/live_test.go @@ -49,16 +49,16 @@ const ( func TestLiveLoaderSmall(t *testing.T) { - db, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() dataFolder := t.TempDir() schemaFile := filepath.Join(dataFolder, "data.schema") dataFile := filepath.Join(dataFolder, "data.rdf") require.NoError(t, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) require.NoError(t, os.WriteFile(dataFile, []byte(SmallData), 0600)) - require.NoError(t, db.Load(context.Background(), schemaFile, dataFile)) + require.NoError(t, driver.Load(context.Background(), schemaFile, dataFile)) const query = `{ caro(func: allofterms(name@en, "Marc Caro")) { @@ -84,15 +84,15 @@ func TestLiveLoaderSmall(t *testing.T) { ] }` - resp, err := db.Query(context.Background(), query) + resp, err := driver.GetDefaultDB().Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, expected, string(resp.Json)) } func TestLiveLoader1Million(t *testing.T) { - db, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer db.Close() + defer driver.Close() baseDir := t.TempDir() schResp, err := grab.Get(baseDir, oneMillionSchema) @@ -100,12 +100,12 @@ func TestLiveLoader1Million(t *testing.T) { dataResp, err := grab.Get(baseDir, oneMillionRDF) require.NoError(t, err) - require.NoError(t, db.DropAll(context.Background())) - require.NoError(t, db.Load(context.Background(), schResp.Filename, dataResp.Filename)) + require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, driver.Load(context.Background(), schResp.Filename, dataResp.Filename)) for _, tt := range common.OneMillionTCs { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - resp, err := db.Query(ctx, tt.Query) + resp, err := driver.GetDefaultDB().Query(ctx, tt.Query) cancel() if ctx.Err() == context.DeadlineExceeded { diff --git a/vector_test.go b/vector_test.go index 041bb1a..f944fa1 100644 --- a/vector_test.go +++ b/vector_test.go @@ -34,7 +34,7 @@ func TestVectorDelete(t *testing.T) { defer driver.Close() require.NoError(t, driver.DropAll(context.Background())) - require.NoError(t, driver.AlterSchema(context.Background(), + require.NoError(t, driver.GetDefaultDB().AlterSchema(context.Background(), fmt.Sprintf(vectorSchemaWithIndex, "vtest", "4", "euclidean"))) // insert random vectors @@ -42,7 +42,7 @@ func TestVectorDelete(t *testing.T) { require.NoError(t, err) //nolint:gosec rdf, vectors := dgraphapi.GenerateRandomVectors(int(assignIDs.StartId)-10, int(assignIDs.EndId)-10, 10, "vtest") - _, err = driver.Mutate(context.Background(), []*api.Mutation{{SetNquads: []byte(rdf)}}) + _, err = driver.GetDefaultDB().Mutate(context.Background(), []*api.Mutation{{SetNquads: []byte(rdf)}}) require.NoError(t, err) // check the count of the vectors inserted @@ -51,7 +51,7 @@ func TestVectorDelete(t *testing.T) { count(uid) } }` - resp, err := driver.Query(context.Background(), q1) + resp, err := driver.GetDefaultDB().Query(context.Background(), q1) require.NoError(t, err) require.JSONEq(t, fmt.Sprintf(`{"vector":[{"count":%d}]}`, numVectors), string(resp.Json)) @@ -68,7 +68,7 @@ func TestVectorDelete(t *testing.T) { triples := strings.Split(rdf, "\n") deleteTriple := func(idx int) string { - _, err := driver.Mutate(context.Background(), []*api.Mutation{{ + _, err := driver.GetDefaultDB().Mutate(context.Background(), []*api.Mutation{{ DelNquads: []byte(triples[idx]), }}) require.NoError(t, err) @@ -80,7 +80,7 @@ func TestVectorDelete(t *testing.T) { } }`, uid[1:len(uid)-1]) - res, err := driver.Query(context.Background(), q2) + res, err := driver.GetDefaultDB().Query(context.Background(), q2) require.NoError(t, err) require.JSONEq(t, `{"vector":[]}`, string(res.Json)) return triples[idx] @@ -105,8 +105,8 @@ func TestVectorDelete(t *testing.T) { _ = queryVectors(t, driver, fmt.Sprintf(q3, strings.Split(triple, `"`)[1])) } -func queryVectors(t *testing.T, db *modusdb.Driver, query string) [][]float32 { - resp, err := db.Query(context.Background(), query) +func queryVectors(t *testing.T, driver *modusdb.Driver, query string) [][]float32 { + resp, err := driver.GetDefaultDB().Query(context.Background(), query) require.NoError(t, err) var data struct { From 66739f1cfff99c5d13e04ecb2658c8a6e2b9fc6d Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Thu, 9 Jan 2025 22:20:01 -0800 Subject: [PATCH 04/10] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0255986..c40a7db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ - fix: alter schema on reverse edge after querying schema [#55](https://github.com/hypermodeinc/modusDB/pull/55) +- feat: update interface to driver and db [#56](https://github.com/hypermodeinc/modusDB/pull/56) + ## 2025-01-02 - Version 0.1.0 Baseline for the changelog. From 68932c9b4860a044db921173e64e7bdeae1a784b Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Fri, 10 Jan 2025 08:38:59 -0800 Subject: [PATCH 05/10] update test --- api_test.go | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/api_test.go b/api_test.go index 89ffce9..3c448e8 100644 --- a/api_test.go +++ b/api_test.go @@ -109,37 +109,15 @@ func TestCreateApi(t *testing.T) { require.JSONEq(t, `{"me":[{"uid":"0x2","User.name":"B","User.age":20,"User.clerk_id":"123"}]}`, string(resp.GetJson())) - // TODO schema{} should work schemaQuery := `schema{}` resp, err = db1.Query(ctx, schemaQuery) require.NoError(t, err) require.JSONEq(t, - `{ - "types": [ - { - "name": "User", - "fields": [ - {"name": "User.name"}, - {"name": "User.age"}, - {"name": "User.clerk_id"} - ] - }, - { - "name": "dgraph.graphql", - "fields": [ - {"name": "dgraph.graphql.schema"}, - {"name": "dgraph.graphql.xid"} - ] - }, - { - "name": "dgraph.graphql.persisted_query", - "fields": [ - {"name": "dgraph.graphql.p_query"} - ] - } - ] - }`, + `{"types": [{"name": "User","fields": [{"name": "User.name"},{"name": "User.age"}, + {"name": "User.clerk_id"}]},{"name": "dgraph.graphql","fields": + [{"name": "dgraph.graphql.schema"},{"name": "dgraph.graphql.xid"}]},{"name": + "dgraph.graphql.persisted_query", "fields": [{"name": "dgraph.graphql.p_query"}]}]}`, string(resp.GetJson())) } From 95d75a949bc54653942e103ee1c34171f3204393 Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Fri, 10 Jan 2025 09:05:13 -0800 Subject: [PATCH 06/10] . --- api_test.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/api_test.go b/api_test.go index 3c448e8..6bd5926 100644 --- a/api_test.go +++ b/api_test.go @@ -108,17 +108,6 @@ func TestCreateApi(t *testing.T) { require.NoError(t, err) require.JSONEq(t, `{"me":[{"uid":"0x2","User.name":"B","User.age":20,"User.clerk_id":"123"}]}`, string(resp.GetJson())) - - schemaQuery := `schema{}` - resp, err = db1.Query(ctx, schemaQuery) - require.NoError(t, err) - - require.JSONEq(t, - `{"types": [{"name": "User","fields": [{"name": "User.name"},{"name": "User.age"}, - {"name": "User.clerk_id"}]},{"name": "dgraph.graphql","fields": - [{"name": "dgraph.graphql.schema"},{"name": "dgraph.graphql.xid"}]},{"name": - "dgraph.graphql.persisted_query", "fields": [{"name": "dgraph.graphql.p_query"}]}]}`, - string(resp.GetJson())) } func TestCreateApiWithNonStruct(t *testing.T) { From 4e5214b55b98c102f57864ee40f02c13b570b42d Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:49:04 -0800 Subject: [PATCH 07/10] update to engine & namespace --- .trunk/trunk.yaml | 6 +- CHANGELOG.md | 3 +- README.md | 12 +- api.go | 92 ++++++------ api/structreflect/tagparser.go | 2 +- api_mutation_gen.go | 20 +-- api_mutation_helpers.go | 32 ++-- api_query_execution.go | 40 ++--- api_test.go | 266 ++++++++++++++++----------------- api_types.go | 16 +- db.go | 26 ++-- db_test.go | 106 ++++++------- driver.go | 172 ++++++++++----------- driver_test.go | 42 +++--- live.go | 16 +- live_benchmark_test.go | 14 +- live_test.go | 18 +-- vector_test.go | 28 ++-- zero.go | 18 +-- 19 files changed, 465 insertions(+), 464 deletions(-) diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index ef59a67..ae1945f 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -24,15 +24,15 @@ runtimes: lint: enabled: - actionlint@1.7.6 - - checkov@3.2.350 + - checkov@3.2.352 - git-diff-check - gofmt@1.20.4 - golangci-lint@1.63.4 - markdownlint@0.43.0 - osv-scanner@1.9.2 - prettier@3.4.2 - - renovate@39.92.0 - - trufflehog@3.88.1 + - renovate@39.106.0 + - trufflehog@3.88.2 - yamllint@1.35.1 actions: diff --git a/CHANGELOG.md b/CHANGELOG.md index c40a7db..8b7cbc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ - fix: alter schema on reverse edge after querying schema [#55](https://github.com/hypermodeinc/modusDB/pull/55) -- feat: update interface to driver and db [#56](https://github.com/hypermodeinc/modusDB/pull/56) +- feat: update interface to engine and namespace + [#56](https://github.com/hypermodeinc/modusDB/pull/56) ## 2025-01-02 - Version 0.1.0 diff --git a/README.md b/README.md index 6251714..ae83cd2 100644 --- a/README.md +++ b/README.md @@ -40,19 +40,19 @@ import ( type User struct { Gid uint64 `json:"gid,omitempty"` - Id string `json:"id,omitempty" db:"constraint=unique"` + Id string `json:"id,omitempty" ns:"constraint=unique"` Name string `json:"name,omitempty"` Age int `json:"age,omitempty"` } func main() { - db, err := New(NewDefaultConfig("/tmp/modusdb")) + ns, err := New(NewDefaultConfig("/tmp/modusdb")) if err != nil { panic(err) } - defer db.Close() + defer ns.Close() - gid, user, err := modusdb.Upsert(db, User{ + gid, user, err := modusdb.Upsert(ns, User{ Id: "123", Name: "A", Age: 10, @@ -62,13 +62,13 @@ func main() { } fmt.Println(user) - _, queriedUser, err := modusdb.Get[User](db, gid) + _, queriedUser, err := modusdb.Get[User](ns, gid) if err != nil { panic(err) } fmt.Println(queriedUser) - _, _, err = modusdb.Delete[User](db, gid) + _, _, err = modusdb.Delete[User](ns, gid) if err != nil { panic(err) } diff --git a/api.go b/api.go index 5d936c2..69143ee 100644 --- a/api.go +++ b/api.go @@ -18,52 +18,52 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func Create[T any](driver *Driver, object T, dbId ...uint64) (uint64, T, error) { - driver.mutex.Lock() - defer driver.mutex.Unlock() - if len(dbId) > 1 { +func Create[T any](engine *Engine, object T, nsId ...uint64) (uint64, T, error) { + engine.mutex.Lock() + defer engine.mutex.Unlock() + if len(nsId) > 1 { return 0, object, fmt.Errorf("only one namespace is allowed") } - ctx, db, err := getDefaultDB(driver, dbId...) + ctx, ns, err := getDefaultNamespace(engine, nsId...) if err != nil { return 0, object, err } - gid, err := driver.z.nextUID() + gid, err := engine.z.nextUID() if err != nil { return 0, object, err } dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateSetDqlMutationsAndSchema[T](ctx, db, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema[T](ctx, ns, object, gid, &dms, sch) if err != nil { return 0, object, err } - err = driver.alterSchemaWithParsed(ctx, sch) + err = engine.alterSchemaWithParsed(ctx, sch) if err != nil { return 0, object, err } - err = applyDqlMutations(ctx, driver, dms) + err = applyDqlMutations(ctx, engine, dms) if err != nil { return 0, object, err } - return getByGid[T](ctx, db, gid) + return getByGid[T](ctx, ns, gid) } -func Upsert[T any](driver *Driver, object T, dbId ...uint64) (uint64, T, bool, error) { +func Upsert[T any](engine *Engine, object T, nsId ...uint64) (uint64, T, bool, error) { var wasFound bool - driver.mutex.Lock() - defer driver.mutex.Unlock() - if len(dbId) > 1 { + engine.mutex.Lock() + defer engine.mutex.Unlock() + if len(nsId) > 1 { return 0, object, false, fmt.Errorf("only one namespace is allowed") } - ctx, db, err := getDefaultDB(driver, dbId...) + ctx, ns, err := getDefaultNamespace(engine, nsId...) if err != nil { return 0, object, false, err } @@ -82,18 +82,18 @@ func Upsert[T any](driver *Driver, object T, dbId ...uint64) (uint64, T, bool, e dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateSetDqlMutationsAndSchema[T](ctx, db, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema[T](ctx, ns, object, gid, &dms, sch) if err != nil { return 0, object, false, err } - err = db.driver.alterSchemaWithParsed(ctx, sch) + err = ns.engine.alterSchemaWithParsed(ctx, sch) if err != nil { return 0, object, false, err } if gid != 0 || cf != nil { - gid, err = getExistingObject[T](ctx, db, gid, cf, object) + gid, err = getExistingObject[T](ctx, ns, gid, cf, object) if err != nil && err != apiutils.ErrNoObjFound { return 0, object, false, err } @@ -101,24 +101,24 @@ func Upsert[T any](driver *Driver, object T, dbId ...uint64) (uint64, T, bool, e } if gid == 0 { - gid, err = driver.z.nextUID() + gid, err = engine.z.nextUID() if err != nil { return 0, object, false, err } } dms = make([]*dql.Mutation, 0) - err = generateSetDqlMutationsAndSchema[T](ctx, db, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema[T](ctx, ns, object, gid, &dms, sch) if err != nil { return 0, object, false, err } - err = applyDqlMutations(ctx, driver, dms) + err = applyDqlMutations(ctx, engine, dms) if err != nil { return 0, object, false, err } - gid, object, err = getByGid[T](ctx, db, gid) + gid, object, err = getByGid[T](ctx, ns, gid) if err != nil { return 0, object, false, err } @@ -126,62 +126,62 @@ func Upsert[T any](driver *Driver, object T, dbId ...uint64) (uint64, T, bool, e return gid, object, wasFound, nil } -func Get[T any, R UniqueField](driver *Driver, uniqueField R, dbId ...uint64) (uint64, T, error) { - driver.mutex.Lock() - defer driver.mutex.Unlock() +func Get[T any, R UniqueField](engine *Engine, uniqueField R, nsId ...uint64) (uint64, T, error) { + engine.mutex.Lock() + defer engine.mutex.Unlock() var obj T - if len(dbId) > 1 { + if len(nsId) > 1 { return 0, obj, fmt.Errorf("only one namespace is allowed") } - ctx, db, err := getDefaultDB(driver, dbId...) + ctx, ns, err := getDefaultNamespace(engine, nsId...) if err != nil { return 0, obj, err } if uid, ok := any(uniqueField).(uint64); ok { - return getByGid[T](ctx, db, uid) + return getByGid[T](ctx, ns, uid) } if cf, ok := any(uniqueField).(ConstrainedField); ok { - return getByConstrainedField[T](ctx, db, cf) + return getByConstrainedField[T](ctx, ns, cf) } return 0, obj, fmt.Errorf("invalid unique field type") } -func Query[T any](driver *Driver, queryParams QueryParams, dbId ...uint64) ([]uint64, []T, error) { - driver.mutex.Lock() - defer driver.mutex.Unlock() - if len(dbId) > 1 { +func Query[T any](engine *Engine, queryParams QueryParams, nsId ...uint64) ([]uint64, []T, error) { + engine.mutex.Lock() + defer engine.mutex.Unlock() + if len(nsId) > 1 { return nil, nil, fmt.Errorf("only one namespace is allowed") } - ctx, db, err := getDefaultDB(driver, dbId...) + ctx, ns, err := getDefaultNamespace(engine, nsId...) if err != nil { return nil, nil, err } - return executeQuery[T](ctx, db, queryParams, true) + return executeQuery[T](ctx, ns, queryParams, true) } -func Delete[T any, R UniqueField](driver *Driver, uniqueField R, dbId ...uint64) (uint64, T, error) { - driver.mutex.Lock() - defer driver.mutex.Unlock() +func Delete[T any, R UniqueField](engine *Engine, uniqueField R, nsId ...uint64) (uint64, T, error) { + engine.mutex.Lock() + defer engine.mutex.Unlock() var zeroObj T - if len(dbId) > 1 { + if len(nsId) > 1 { return 0, zeroObj, fmt.Errorf("only one namespace is allowed") } - ctx, db, err := getDefaultDB(driver, dbId...) + ctx, ns, err := getDefaultNamespace(engine, nsId...) if err != nil { return 0, zeroObj, err } if uid, ok := any(uniqueField).(uint64); ok { - uid, obj, err := getByGid[T](ctx, db, uid) + uid, obj, err := getByGid[T](ctx, ns, uid) if err != nil { return 0, zeroObj, err } - dms := generateDeleteDqlMutations(db, uid) + dms := generateDeleteDqlMutations(ns, uid) - err = applyDqlMutations(ctx, driver, dms) + err = applyDqlMutations(ctx, engine, dms) if err != nil { return 0, zeroObj, err } @@ -190,14 +190,14 @@ func Delete[T any, R UniqueField](driver *Driver, uniqueField R, dbId ...uint64) } if cf, ok := any(uniqueField).(ConstrainedField); ok { - uid, obj, err := getByConstrainedField[T](ctx, db, cf) + uid, obj, err := getByConstrainedField[T](ctx, ns, cf) if err != nil { return 0, zeroObj, err } - dms := generateDeleteDqlMutations(db, uid) + dms := generateDeleteDqlMutations(ns, uid) - err = applyDqlMutations(ctx, driver, dms) + err = applyDqlMutations(ctx, engine, dms) if err != nil { return 0, zeroObj, err } diff --git a/api/structreflect/tagparser.go b/api/structreflect/tagparser.go index da9224f..0d94918 100644 --- a/api/structreflect/tagparser.go +++ b/api/structreflect/tagparser.go @@ -26,7 +26,7 @@ func parseJsonTag(field reflect.StructField) (string, error) { } func parseDbTag(field reflect.StructField) *DbTag { - dbConstraintsTag := field.Tag.Get("db") + dbConstraintsTag := field.Tag.Get("ns") if dbConstraintsTag == "" { return nil } diff --git a/api_mutation_gen.go b/api_mutation_gen.go index 1fe4c45..d8533fc 100644 --- a/api_mutation_gen.go +++ b/api_mutation_gen.go @@ -26,7 +26,7 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func generateSetDqlMutationsAndSchema[T any](ctx context.Context, d *DB, object T, +func generateSetDqlMutationsAndSchema[T any](ctx context.Context, n *Namespace, object T, gid uint64, dms *[]*dql.Mutation, sch *schema.ParsedSchema) error { t := reflect.TypeOf(object) if t.Kind() != reflect.Struct { @@ -49,7 +49,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, d *DB, object if tagMaps.JsonToReverseEdge[jsonName] != "" { reverseEdgeStr := tagMaps.JsonToReverseEdge[jsonName] typeName := strings.Split(reverseEdgeStr, ".")[0] - currSchema, err := getSchema(ctx, d) + currSchema, err := getSchema(ctx, n) if err != nil { return err } @@ -70,7 +70,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, d *DB, object } if !(typeFound && predicateFound) { - if err := mutations.HandleReverseEdge(jsonName, reflectValueType, d.ID(), sch, + if err := mutations.HandleReverseEdge(jsonName, reflectValueType, n.ID(), sch, reverseEdgeStr); err != nil { return err } @@ -82,17 +82,17 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, d *DB, object continue } - value, err = processStructValue(ctx, value, d) + value, err = processStructValue(ctx, value, n) if err != nil { return err } - value, err = processPointerValue(ctx, value, d) + value, err = processPointerValue(ctx, value, n) if err != nil { return err } - nquad, u, err := mutations.CreateNQuadAndSchema(value, gid, jsonName, t, d.ID()) + nquad, u, err := mutations.CreateNQuadAndSchema(value, gid, jsonName, t, n.ID()) if err != nil { return err } @@ -111,7 +111,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, d *DB, object } sch.Types = append(sch.Types, &pb.TypeUpdate{ - TypeName: apiutils.AddNamespace(d.ID(), t.Name()), + TypeName: apiutils.AddNamespace(n.ID(), t.Name()), Fields: sch.Preds, }) @@ -120,7 +120,7 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, d *DB, object return err } typeNquad := &api.NQuad{ - Namespace: d.ID(), + Namespace: n.ID(), Subject: fmt.Sprint(gid), Predicate: "dgraph.type", ObjectValue: val, @@ -134,11 +134,11 @@ func generateSetDqlMutationsAndSchema[T any](ctx context.Context, d *DB, object return nil } -func generateDeleteDqlMutations(d *DB, gid uint64) []*dql.Mutation { +func generateDeleteDqlMutations(n *Namespace, gid uint64) []*dql.Mutation { return []*dql.Mutation{{ Del: []*api.NQuad{ { - Namespace: d.ID(), + Namespace: n.ID(), Subject: fmt.Sprint(gid), Predicate: x.Star, ObjectValue: &api.Value{ diff --git a/api_mutation_helpers.go b/api_mutation_helpers.go index 4a1f30b..1a6388b 100644 --- a/api_mutation_helpers.go +++ b/api_mutation_helpers.go @@ -14,10 +14,10 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func processStructValue(ctx context.Context, value any, db *DB) (any, error) { +func processStructValue(ctx context.Context, value any, ns *Namespace) (any, error) { if reflect.TypeOf(value).Kind() == reflect.Struct { value = reflect.ValueOf(value).Interface() - newGid, err := getUidOrMutate(ctx, db.driver, db, value) + newGid, err := getUidOrMutate(ctx, ns.engine, ns, value) if err != nil { return nil, err } @@ -26,19 +26,19 @@ func processStructValue(ctx context.Context, value any, db *DB) (any, error) { return value, nil } -func processPointerValue(ctx context.Context, value any, db *DB) (any, error) { +func processPointerValue(ctx context.Context, value any, ns *Namespace) (any, error) { reflectValueType := reflect.TypeOf(value) if reflectValueType.Kind() == reflect.Pointer { reflectValueType = reflectValueType.Elem() if reflectValueType.Kind() == reflect.Struct { value = reflect.ValueOf(value).Elem().Interface() - return processStructValue(ctx, value, db) + return processStructValue(ctx, value, ns) } } return value, nil } -func getUidOrMutate[T any](ctx context.Context, driver *Driver, db *DB, object T) (uint64, error) { +func getUidOrMutate[T any](ctx context.Context, engine *Engine, ns *Namespace, object T) (uint64, error) { gid, cfKeyValue, err := structreflect.GetUniqueConstraint[T](object) if err != nil { return 0, err @@ -50,17 +50,17 @@ func getUidOrMutate[T any](ctx context.Context, driver *Driver, db *DB, object T dms := make([]*dql.Mutation, 0) sch := &schema.ParsedSchema{} - err = generateSetDqlMutationsAndSchema(ctx, db, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema(ctx, ns, object, gid, &dms, sch) if err != nil { return 0, err } - err = driver.alterSchemaWithParsed(ctx, sch) + err = engine.alterSchemaWithParsed(ctx, sch) if err != nil { return 0, err } if gid != 0 || cf != nil { - gid, err = getExistingObject(ctx, db, gid, cf, object) + gid, err = getExistingObject(ctx, ns, gid, cf, object) if err != nil && err != apiutils.ErrNoObjFound { return 0, err } @@ -69,18 +69,18 @@ func getUidOrMutate[T any](ctx context.Context, driver *Driver, db *DB, object T } } - gid, err = driver.z.nextUID() + gid, err = engine.z.nextUID() if err != nil { return 0, err } dms = make([]*dql.Mutation, 0) - err = generateSetDqlMutationsAndSchema(ctx, db, object, gid, &dms, sch) + err = generateSetDqlMutationsAndSchema(ctx, ns, object, gid, &dms, sch) if err != nil { return 0, err } - err = applyDqlMutations(ctx, driver, dms) + err = applyDqlMutations(ctx, engine, dms) if err != nil { return 0, err } @@ -88,21 +88,21 @@ func getUidOrMutate[T any](ctx context.Context, driver *Driver, db *DB, object T return gid, nil } -func applyDqlMutations(ctx context.Context, driver *Driver, dms []*dql.Mutation) error { +func applyDqlMutations(ctx context.Context, engine *Engine, dms []*dql.Mutation) error { edges, err := query.ToDirectedEdges(dms, nil) if err != nil { return err } - if !driver.isOpen.Load() { - return ErrClosedDriver + if !engine.isOpen.Load() { + return ErrClosedEngine } - startTs, err := driver.z.nextTs() + startTs, err := engine.z.nextTs() if err != nil { return err } - commitTs, err := driver.z.nextTs() + commitTs, err := engine.z.nextTs() if err != nil { return err } diff --git a/api_query_execution.go b/api_query_execution.go index 421b648..4e90527 100644 --- a/api_query_execution.go +++ b/api_query_execution.go @@ -20,34 +20,34 @@ import ( "github.com/hypermodeinc/modusdb/api/structreflect" ) -func getByGid[T any](ctx context.Context, db *DB, gid uint64) (uint64, T, error) { - return executeGet[T](ctx, db, gid) +func getByGid[T any](ctx context.Context, ns *Namespace, gid uint64) (uint64, T, error) { + return executeGet[T](ctx, ns, gid) } -func getByGidWithObject[T any](ctx context.Context, db *DB, gid uint64, obj T) (uint64, T, error) { - return executeGetWithObject[T](ctx, db, obj, false, gid) +func getByGidWithObject[T any](ctx context.Context, ns *Namespace, gid uint64, obj T) (uint64, T, error) { + return executeGetWithObject[T](ctx, ns, obj, false, gid) } -func getByConstrainedField[T any](ctx context.Context, db *DB, cf ConstrainedField) (uint64, T, error) { - return executeGet[T](ctx, db, cf) +func getByConstrainedField[T any](ctx context.Context, ns *Namespace, cf ConstrainedField) (uint64, T, error) { + return executeGet[T](ctx, ns, cf) } -func getByConstrainedFieldWithObject[T any](ctx context.Context, db *DB, +func getByConstrainedFieldWithObject[T any](ctx context.Context, ns *Namespace, cf ConstrainedField, obj T) (uint64, T, error) { - return executeGetWithObject[T](ctx, db, obj, false, cf) + return executeGetWithObject[T](ctx, ns, obj, false, cf) } -func executeGet[T any, R UniqueField](ctx context.Context, db *DB, args ...R) (uint64, T, error) { +func executeGet[T any, R UniqueField](ctx context.Context, ns *Namespace, args ...R) (uint64, T, error) { var obj T if len(args) != 1 { - return 0, obj, fmt.Errorf("expected 1 argument, got %db", len(args)) + return 0, obj, fmt.Errorf("expected 1 argument, got %ds", len(args)) } - return executeGetWithObject(ctx, db, obj, true, args...) + return executeGetWithObject(ctx, ns, obj, true, args...) } -func executeGetWithObject[T any, R UniqueField](ctx context.Context, db *DB, +func executeGetWithObject[T any, R UniqueField](ctx context.Context, ns *Namespace, obj T, withReverse bool, args ...R) (uint64, T, error) { t := reflect.TypeOf(obj) @@ -79,7 +79,7 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, db *DB, return 0, obj, fmt.Errorf("constraint not defined for field %s", cf.Key) } - resp, err := db.driver.queryWithLock(ctx, db, query) + resp, err := ns.engine.queryWithLock(ctx, ns, query) if err != nil { return 0, obj, err } @@ -107,7 +107,7 @@ func executeGetWithObject[T any, R UniqueField](ctx context.Context, db *DB, return structreflect.ConvertDynamicToTyped[T](result.Obj[0], t) } -func executeQuery[T any](ctx context.Context, db *DB, queryParams QueryParams, +func executeQuery[T any](ctx context.Context, ns *Namespace, queryParams QueryParams, withReverse bool) ([]uint64, []T, error) { var obj T t := reflect.TypeOf(obj) @@ -143,7 +143,7 @@ func executeQuery[T any](ctx context.Context, db *DB, queryParams QueryParams, query := querygen.FormatObjsQuery(t.Name(), filterQueryFunc, paginationAndSorting, readFromQuery) - resp, err := db.driver.queryWithLock(ctx, db, query) + resp, err := ns.engine.queryWithLock(ctx, ns, query) if err != nil { return nil, nil, err } @@ -186,13 +186,13 @@ func executeQuery[T any](ctx context.Context, db *DB, queryParams QueryParams, return gids, objs, nil } -func getExistingObject[T any](ctx context.Context, db *DB, gid uint64, cf *ConstrainedField, +func getExistingObject[T any](ctx context.Context, ns *Namespace, gid uint64, cf *ConstrainedField, object T) (uint64, error) { var err error if gid != 0 { - gid, _, err = getByGidWithObject[T](ctx, db, gid, object) + gid, _, err = getByGidWithObject[T](ctx, ns, gid, object) } else if cf != nil { - gid, _, err = getByConstrainedFieldWithObject[T](ctx, db, *cf, object) + gid, _, err = getByConstrainedFieldWithObject[T](ctx, ns, *cf, object) } if err != nil { return 0, err @@ -200,8 +200,8 @@ func getExistingObject[T any](ctx context.Context, db *DB, gid uint64, cf *Const return gid, nil } -func getSchema(ctx context.Context, db *DB) (*querygen.SchemaResponse, error) { - resp, err := db.driver.queryWithLock(ctx, db, querygen.SchemaQuery) +func getSchema(ctx context.Context, ns *Namespace) (*querygen.SchemaResponse, error) { + resp, err := ns.engine.queryWithLock(ctx, ns, querygen.SchemaQuery) if err != nil { return nil, err } diff --git a/api_test.go b/api_test.go index 6bd5926..068089a 100644 --- a/api_test.go +++ b/api_test.go @@ -24,15 +24,15 @@ type User struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` Age int `json:"age,omitempty"` - ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" ns:"constraint=unique"` } func TestFirstTimeUser(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - gid, user, err := modusdb.Create(driver, User{ + gid, user, err := modusdb.Create(engine, User{ Name: "A", Age: 10, ClerkId: "123", @@ -44,7 +44,7 @@ func TestFirstTimeUser(t *testing.T) { require.Equal(t, 10, user.Age) require.Equal(t, "123", user.ClerkId) - gid, queriedUser, err := modusdb.Get[User](driver, gid) + gid, queriedUser, err := modusdb.Get[User](engine, gid) require.NoError(t, err) require.Equal(t, queriedUser.Gid, gid) @@ -52,7 +52,7 @@ func TestFirstTimeUser(t *testing.T) { require.Equal(t, "A", queriedUser.Name) require.Equal(t, "123", queriedUser.ClerkId) - gid, queriedUser2, err := modusdb.Get[User](driver, modusdb.ConstrainedField{ + gid, queriedUser2, err := modusdb.Get[User](engine, modusdb.ConstrainedField{ Key: "clerk_id", Value: "123", }) @@ -63,10 +63,10 @@ func TestFirstTimeUser(t *testing.T) { require.Equal(t, "A", queriedUser2.Name) require.Equal(t, "123", queriedUser2.ClerkId) - _, _, err = modusdb.Delete[User](driver, gid) + _, _, err = modusdb.Delete[User](engine, gid) require.NoError(t, err) - _, queriedUser3, err := modusdb.Get[User](driver, gid) + _, queriedUser3, err := modusdb.Get[User](engine, gid) require.Error(t, err) require.Equal(t, "no object found", err.Error()) require.Equal(t, queriedUser3, User{}) @@ -75,14 +75,14 @@ func TestFirstTimeUser(t *testing.T) { func TestCreateApi(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) user := User{ Name: "B", @@ -90,7 +90,7 @@ func TestCreateApi(t *testing.T) { ClerkId: "123", } - gid, user, err := modusdb.Create(driver, user, db1.ID()) + gid, user, err := modusdb.Create(engine, user, ns1.ID()) require.NoError(t, err) require.Equal(t, "B", user.Name) @@ -104,7 +104,7 @@ func TestCreateApi(t *testing.T) { User.clerk_id } }` - resp, err := db1.Query(ctx, query) + resp, err := ns1.Query(ctx, query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"uid":"0x2","User.name":"B","User.age":20,"User.clerk_id":"123"}]}`, string(resp.GetJson())) @@ -112,35 +112,35 @@ func TestCreateApi(t *testing.T) { func TestCreateApiWithNonStruct(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) user := User{ Name: "B", Age: 20, } - _, _, err = modusdb.Create[*User](driver, &user, db1.ID()) + _, _, err = modusdb.Create[*User](engine, &user, ns1.ID()) require.Error(t, err) require.Equal(t, "expected struct, got ptr", err.Error()) } func TestGetApi(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) user := User{ Name: "B", @@ -148,10 +148,10 @@ func TestGetApi(t *testing.T) { ClerkId: "123", } - gid, _, err := modusdb.Create(driver, user, db1.ID()) + gid, _, err := modusdb.Create(engine, user, ns1.ID()) require.NoError(t, err) - gid, queriedUser, err := modusdb.Get[User](driver, gid, db1.ID()) + gid, queriedUser, err := modusdb.Get[User](engine, gid, ns1.ID()) require.NoError(t, err) require.Equal(t, queriedUser.Gid, gid) @@ -162,14 +162,14 @@ func TestGetApi(t *testing.T) { func TestGetApiWithConstrainedField(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) user := User{ Name: "B", @@ -177,13 +177,13 @@ func TestGetApiWithConstrainedField(t *testing.T) { ClerkId: "123", } - _, _, err = modusdb.Create(driver, user, db1.ID()) + _, _, err = modusdb.Create(engine, user, ns1.ID()) require.NoError(t, err) - gid, queriedUser, err := modusdb.Get[User](driver, modusdb.ConstrainedField{ + gid, queriedUser, err := modusdb.Get[User](engine, modusdb.ConstrainedField{ Key: "clerk_id", Value: "123", - }, db1.ID()) + }, ns1.ID()) require.NoError(t, err) require.Equal(t, queriedUser.Gid, gid) @@ -194,14 +194,14 @@ func TestGetApiWithConstrainedField(t *testing.T) { func TestDeleteApi(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) user := User{ Name: "B", @@ -209,21 +209,21 @@ func TestDeleteApi(t *testing.T) { ClerkId: "123", } - gid, _, err := modusdb.Create(driver, user, db1.ID()) + gid, _, err := modusdb.Create(engine, user, ns1.ID()) require.NoError(t, err) - _, _, err = modusdb.Delete[User](driver, gid, db1.ID()) + _, _, err = modusdb.Delete[User](engine, gid, ns1.ID()) require.NoError(t, err) - _, queriedUser, err := modusdb.Get[User](driver, gid, db1.ID()) + _, queriedUser, err := modusdb.Get[User](engine, gid, ns1.ID()) require.Error(t, err) require.Equal(t, "no object found", err.Error()) require.Equal(t, queriedUser, User{}) - _, queriedUser, err = modusdb.Get[User](driver, modusdb.ConstrainedField{ + _, queriedUser, err = modusdb.Get[User](engine, modusdb.ConstrainedField{ Key: "clerk_id", Value: "123", - }, db1.ID()) + }, ns1.ID()) require.Error(t, err) require.Equal(t, "no object found", err.Error()) require.Equal(t, queriedUser, User{}) @@ -231,14 +231,14 @@ func TestDeleteApi(t *testing.T) { func TestUpsertApi(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) user := User{ Name: "B", @@ -246,16 +246,16 @@ func TestUpsertApi(t *testing.T) { ClerkId: "123", } - gid, user, _, err := modusdb.Upsert(driver, user, db1.ID()) + gid, user, _, err := modusdb.Upsert(engine, user, ns1.ID()) require.NoError(t, err) require.Equal(t, user.Gid, gid) user.Age = 21 - gid, _, _, err = modusdb.Upsert(driver, user, db1.ID()) + gid, _, _, err = modusdb.Upsert(engine, user, ns1.ID()) require.NoError(t, err) require.Equal(t, user.Gid, gid) - _, queriedUser, err := modusdb.Get[User](driver, gid, db1.ID()) + _, queriedUser, err := modusdb.Get[User](engine, gid, ns1.ID()) require.NoError(t, err) require.Equal(t, user.Gid, queriedUser.Gid) require.Equal(t, 21, queriedUser.Age) @@ -265,14 +265,14 @@ func TestUpsertApi(t *testing.T) { func TestQueryApi(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) users := []User{ {Name: "A", Age: 10, ClerkId: "123"}, @@ -283,11 +283,11 @@ func TestQueryApi(t *testing.T) { } for _, user := range users { - _, _, err = modusdb.Create(driver, user, db1.ID()) + _, _, err = modusdb.Create(engine, user, ns1.ID()) require.NoError(t, err) } - gids, queriedUsers, err := modusdb.Query[User](driver, modusdb.QueryParams{}, db1.ID()) + gids, queriedUsers, err := modusdb.Query[User](engine, modusdb.QueryParams{}, ns1.ID()) require.NoError(t, err) require.Len(t, queriedUsers, 5) require.Len(t, gids, 5) @@ -297,7 +297,7 @@ func TestQueryApi(t *testing.T) { require.Equal(t, "D", queriedUsers[3].Name) require.Equal(t, "E", queriedUsers[4].Name) - gids, queriedUsers, err = modusdb.Query[User](driver, modusdb.QueryParams{ + gids, queriedUsers, err = modusdb.Query[User](engine, modusdb.QueryParams{ Filter: &modusdb.Filter{ Field: "age", String: modusdb.StringPredicate{ @@ -307,7 +307,7 @@ func TestQueryApi(t *testing.T) { GreaterOrEqual: fmt.Sprintf("%d", 20), }, }, - }, db1.ID()) + }, ns1.ID()) require.NoError(t, err) require.Len(t, queriedUsers, 4) @@ -320,14 +320,14 @@ func TestQueryApi(t *testing.T) { func TestQueryApiWithPaginiationAndSorting(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) users := []User{ {Name: "A", Age: 10, ClerkId: "123"}, @@ -338,11 +338,11 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { } for _, user := range users { - _, _, err = modusdb.Create(driver, user, db1.ID()) + _, _, err = modusdb.Create(engine, user, ns1.ID()) require.NoError(t, err) } - gids, queriedUsers, err := modusdb.Query[User](driver, modusdb.QueryParams{ + gids, queriedUsers, err := modusdb.Query[User](engine, modusdb.QueryParams{ Filter: &modusdb.Filter{ Field: "age", String: modusdb.StringPredicate{ @@ -353,7 +353,7 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { Limit: 3, Offset: 1, }, - }, db1.ID()) + }, ns1.ID()) require.NoError(t, err) require.Len(t, queriedUsers, 3) @@ -362,7 +362,7 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { require.Equal(t, "D", queriedUsers[1].Name) require.Equal(t, "E", queriedUsers[2].Name) - gids, queriedUsers, err = modusdb.Query[User](driver, modusdb.QueryParams{ + gids, queriedUsers, err = modusdb.Query[User](engine, modusdb.QueryParams{ Pagination: &modusdb.Pagination{ Limit: 3, Offset: 1, @@ -370,7 +370,7 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { Sorting: &modusdb.Sorting{ OrderAscField: "age", }, - }, db1.ID()) + }, ns1.ID()) require.NoError(t, err) require.Len(t, queriedUsers, 3) @@ -383,36 +383,36 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { type Project struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" ns:"constraint=unique"` Branches []Branch `json:"branches,omitempty" readFrom:"type=Branch,field=proj"` } type Branch struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" ns:"constraint=unique"` Proj Project `json:"proj,omitempty"` } func TestReverseEdgeGet(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) - projGid, project, err := modusdb.Create(driver, Project{ + projGid, project, err := modusdb.Create(engine, Project{ Name: "P", ClerkId: "456", Branches: []Branch{ {Name: "B", ClerkId: "123"}, {Name: "B2", ClerkId: "456"}, }, - }, db1.ID()) + }, ns1.ID()) require.NoError(t, err) require.Equal(t, "P", project.Name) @@ -429,7 +429,7 @@ func TestReverseEdgeGet(t *testing.T) { }, } - branch1Gid, branch1, err := modusdb.Create(driver, branch1, db1.ID()) + branch1Gid, branch1, err := modusdb.Create(engine, branch1, ns1.ID()) require.NoError(t, err) require.Equal(t, "B", branch1.Name) @@ -445,13 +445,13 @@ func TestReverseEdgeGet(t *testing.T) { }, } - branch2Gid, branch2, err := modusdb.Create(driver, branch2, db1.ID()) + branch2Gid, branch2, err := modusdb.Create(engine, branch2, ns1.ID()) require.NoError(t, err) require.Equal(t, "B2", branch2.Name) require.Equal(t, branch2.Gid, branch2Gid) require.Equal(t, projGid, branch2.Proj.Gid) - getProjGid, queriedProject, err := modusdb.Get[Project](driver, projGid, db1.ID()) + getProjGid, queriedProject, err := modusdb.Get[Project](engine, projGid, ns1.ID()) require.NoError(t, err) require.Equal(t, projGid, getProjGid) require.Equal(t, "P", queriedProject.Name) @@ -459,7 +459,7 @@ func TestReverseEdgeGet(t *testing.T) { require.Equal(t, "B", queriedProject.Branches[0].Name) require.Equal(t, "B2", queriedProject.Branches[1].Name) - queryBranchesGids, queriedBranches, err := modusdb.Query[Branch](driver, modusdb.QueryParams{}, db1.ID()) + queryBranchesGids, queriedBranches, err := modusdb.Query[Branch](engine, modusdb.QueryParams{}, ns1.ID()) require.NoError(t, err) require.Len(t, queriedBranches, 2) require.Len(t, queryBranchesGids, 2) @@ -469,10 +469,10 @@ func TestReverseEdgeGet(t *testing.T) { // max depth is 2, so we should not see the branches within project require.Len(t, queriedBranches[0].Proj.Branches, 0) - _, _, err = modusdb.Delete[Project](driver, projGid, db1.ID()) + _, _, err = modusdb.Delete[Project](engine, projGid, ns1.ID()) require.NoError(t, err) - queryBranchesGids, queriedBranches, err = modusdb.Query[Branch](driver, modusdb.QueryParams{}, db1.ID()) + queryBranchesGids, queriedBranches, err = modusdb.Query[Branch](engine, modusdb.QueryParams{}, ns1.ID()) require.NoError(t, err) require.Len(t, queriedBranches, 2) require.Len(t, queryBranchesGids, 2) @@ -482,14 +482,14 @@ func TestReverseEdgeGet(t *testing.T) { func TestReverseEdgeQuery(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) projects := []Project{ {Name: "P1", ClerkId: "456"}, @@ -500,7 +500,7 @@ func TestReverseEdgeQuery(t *testing.T) { clerkCounter := 100 for _, project := range projects { - projGid, project, err := modusdb.Create(driver, project, db1.ID()) + projGid, project, err := modusdb.Create(engine, project, ns1.ID()) require.NoError(t, err) require.Equal(t, project.Name, project.Name) require.Equal(t, project.Gid, projGid) @@ -513,7 +513,7 @@ func TestReverseEdgeQuery(t *testing.T) { clerkCounter += 2 for _, branch := range branches { - branchGid, branch, err := modusdb.Create(driver, branch, db1.ID()) + branchGid, branch, err := modusdb.Create(engine, branch, ns1.ID()) require.NoError(t, err) require.Equal(t, branch.Name, branch.Name) require.Equal(t, branch.Gid, branchGid) @@ -521,7 +521,7 @@ func TestReverseEdgeQuery(t *testing.T) { } } - queriedProjectsGids, queriedProjects, err := modusdb.Query[Project](driver, modusdb.QueryParams{}, db1.ID()) + queriedProjectsGids, queriedProjects, err := modusdb.Query[Project](engine, modusdb.QueryParams{}, ns1.ID()) require.NoError(t, err) require.Len(t, queriedProjects, 2) require.Len(t, queriedProjectsGids, 2) @@ -537,14 +537,14 @@ func TestReverseEdgeQuery(t *testing.T) { func TestNestedObjectMutation(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) branch := Branch{ Name: "B", @@ -555,7 +555,7 @@ func TestNestedObjectMutation(t *testing.T) { }, } - gid, branch, err := modusdb.Create(driver, branch, db1.ID()) + gid, branch, err := modusdb.Create(engine, branch, ns1.ID()) require.NoError(t, err) require.Equal(t, "B", branch.Name) @@ -575,14 +575,14 @@ func TestNestedObjectMutation(t *testing.T) { } } }` - resp, err := db1.Query(ctx, query) + resp, err := ns1.Query(ctx, query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"uid":"0x2","Branch.name":"B","Branch.clerk_id":"123","Branch.proj": {"uid":"0x3","Project.name":"P","Project.clerk_id":"456"}}]}`, string(resp.GetJson())) - gid, queriedBranch, err := modusdb.Get[Branch](driver, gid, db1.ID()) + gid, queriedBranch, err := modusdb.Get[Branch](engine, gid, ns1.ID()) require.NoError(t, err) require.Equal(t, queriedBranch.Gid, gid) require.Equal(t, "B", queriedBranch.Name) @@ -591,19 +591,19 @@ func TestNestedObjectMutation(t *testing.T) { func TestLinkingObjectsByConstrainedFields(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) - projGid, project, err := modusdb.Create(driver, Project{ + projGid, project, err := modusdb.Create(engine, Project{ Name: "P", ClerkId: "456", - }, db1.ID()) + }, ns1.ID()) require.NoError(t, err) require.Equal(t, "P", project.Name) @@ -618,7 +618,7 @@ func TestLinkingObjectsByConstrainedFields(t *testing.T) { }, } - gid, branch, err := modusdb.Create(driver, branch, db1.ID()) + gid, branch, err := modusdb.Create(engine, branch, ns1.ID()) require.NoError(t, err) require.Equal(t, "B", branch.Name) @@ -638,14 +638,14 @@ func TestLinkingObjectsByConstrainedFields(t *testing.T) { } } }` - resp, err := db1.Query(ctx, query) + resp, err := ns1.Query(ctx, query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"uid":"0x3","Branch.name":"B","Branch.clerk_id":"123","Branch.proj": {"uid":"0x2","Project.name":"P","Project.clerk_id":"456"}}]}`, string(resp.GetJson())) - gid, queriedBranch, err := modusdb.Get[Branch](driver, gid, db1.ID()) + gid, queriedBranch, err := modusdb.Get[Branch](engine, gid, ns1.ID()) require.NoError(t, err) require.Equal(t, queriedBranch.Gid, gid) require.Equal(t, "B", queriedBranch.Name) @@ -654,19 +654,19 @@ func TestLinkingObjectsByConstrainedFields(t *testing.T) { func TestLinkingObjectsByGid(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) - projGid, project, err := modusdb.Create(driver, Project{ + projGid, project, err := modusdb.Create(engine, Project{ Name: "P", ClerkId: "456", - }, db1.ID()) + }, ns1.ID()) require.NoError(t, err) require.Equal(t, "P", project.Name) @@ -680,7 +680,7 @@ func TestLinkingObjectsByGid(t *testing.T) { }, } - gid, branch, err := modusdb.Create(driver, branch, db1.ID()) + gid, branch, err := modusdb.Create(engine, branch, ns1.ID()) require.NoError(t, err) require.Equal(t, "B", branch.Name) @@ -700,14 +700,14 @@ func TestLinkingObjectsByGid(t *testing.T) { } } }` - resp, err := db1.Query(ctx, query) + resp, err := ns1.Query(ctx, query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"uid":"0x3","Branch.name":"B","Branch.clerk_id":"123", "Branch.proj":{"uid":"0x2","Project.name":"P","Project.clerk_id":"456"}}]}`, string(resp.GetJson())) - gid, queriedBranch, err := modusdb.Get[Branch](driver, gid, db1.ID()) + gid, queriedBranch, err := modusdb.Get[Branch](engine, gid, ns1.ID()) require.NoError(t, err) require.Equal(t, queriedBranch.Gid, gid) require.Equal(t, "B", queriedBranch.Name) @@ -722,20 +722,20 @@ type BadProject struct { type BadBranch struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" ns:"constraint=unique"` Proj BadProject `json:"proj,omitempty"` } func TestNestedObjectMutationWithBadType(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) branch := BadBranch{ Name: "B", @@ -746,7 +746,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { }, } - _, _, err = modusdb.Create(driver, branch, db1.ID()) + _, _, err = modusdb.Create(engine, branch, ns1.ID()) require.Error(t, err) require.Equal(t, fmt.Sprintf(apiutils.NoUniqueConstr, "BadProject"), err.Error()) @@ -755,7 +755,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { ClerkId: "456", } - _, _, err = modusdb.Create(driver, proj, db1.ID()) + _, _, err = modusdb.Create(engine, proj, ns1.ID()) require.Error(t, err) require.Equal(t, fmt.Sprintf(apiutils.NoUniqueConstr, "BadProject"), err.Error()) @@ -764,19 +764,19 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { type Document struct { Gid uint64 `json:"gid,omitempty"` Text string `json:"text,omitempty"` - TextVec []float32 `json:"textVec,omitempty" db:"constraint=vector"` + TextVec []float32 `json:"textVec,omitempty" ns:"constraint=vector"` } func TestVectorIndexSearchTyped(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) documents := []Document{ {Text: "apple", TextVec: []float32{0.1, 0.1, 0.0}}, @@ -789,7 +789,7 @@ func TestVectorIndexSearchTyped(t *testing.T) { } for _, doc := range documents { - _, _, err = modusdb.Create(driver, doc, db1.ID()) + _, _, err = modusdb.Create(engine, doc, ns1.ID()) require.NoError(t, err) } @@ -800,7 +800,7 @@ func TestVectorIndexSearchTyped(t *testing.T) { } }` - resp, err := db1.Query(ctx, query) + resp, err := ns1.Query(ctx, query) require.NoError(t, err) require.JSONEq(t, `{ "documents":[ @@ -819,7 +819,7 @@ func TestVectorIndexSearchTyped(t *testing.T) { } }` - resp, err = db1.Query(ctx, query2) + resp, err = ns1.Query(ctx, query2) require.NoError(t, err) require.JSONEq(t, `{ "documents":[ @@ -834,14 +834,14 @@ func TestVectorIndexSearchTyped(t *testing.T) { func TestVectorIndexSearchWithQuery(t *testing.T) { ctx := context.Background() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(ctx)) + require.NoError(t, ns1.DropData(ctx)) documents := []Document{ {Text: "apple", TextVec: []float32{0.1, 0.1, 0.0}}, @@ -854,11 +854,11 @@ func TestVectorIndexSearchWithQuery(t *testing.T) { } for _, doc := range documents { - _, _, err = modusdb.Create(driver, doc, db1.ID()) + _, _, err = modusdb.Create(engine, doc, ns1.ID()) require.NoError(t, err) } - gids, docs, err := modusdb.Query[Document](driver, modusdb.QueryParams{ + gids, docs, err := modusdb.Query[Document](engine, modusdb.QueryParams{ Filter: &modusdb.Filter{ Field: "textVec", Vector: modusdb.VectorPredicate{ @@ -866,7 +866,7 @@ func TestVectorIndexSearchWithQuery(t *testing.T) { TopK: 5, }, }, - }, db1.ID()) + }, ns1.ID()) require.NoError(t, err) require.Len(t, docs, 5) diff --git a/api_types.go b/api_types.go index 37aae5d..d2b9408 100644 --- a/api_types.go +++ b/api_types.go @@ -75,24 +75,24 @@ type VectorPredicate struct { type ModusDbOption func(*modusDbOptions) type modusDbOptions struct { - db uint64 + ns uint64 } -func WithDB(db uint64) ModusDbOption { +func WithNamespace(ns uint64) ModusDbOption { return func(o *modusDbOptions) { - o.db = db + o.ns = ns } } -func getDefaultDB(driver *Driver, dbId ...uint64) (context.Context, *DB, error) { +func getDefaultNamespace(engine *Engine, nsId ...uint64) (context.Context, *Namespace, error) { dbOpts := &modusDbOptions{ - db: driver.db0.ID(), + ns: engine.db0.ID(), } - for _, db := range dbId { - WithDB(db)(dbOpts) + for _, ns := range nsId { + WithNamespace(ns)(dbOpts) } - d, err := driver.getDBWithLock(dbOpts.db) + d, err := engine.getNamespaceWithLock(dbOpts.ns) if err != nil { return nil, nil, err } diff --git a/db.go b/db.go index 6d18cb6..b40bbf8 100644 --- a/db.go +++ b/db.go @@ -15,30 +15,30 @@ import ( "github.com/dgraph-io/dgo/v240/protos/api" ) -// DB is one of the namespaces in modusDB. -type DB struct { +// Namespace is one of the namespaces in modusDB. +type Namespace struct { id uint64 - driver *Driver + engine *Engine } -func (db *DB) ID() uint64 { - return db.id +func (ns *Namespace) ID() uint64 { + return ns.id } // DropData drops all the data in the modusDB instance. -func (db *DB) DropData(ctx context.Context) error { - return db.driver.dropData(ctx, db) +func (ns *Namespace) DropData(ctx context.Context) error { + return ns.engine.dropData(ctx, ns) } -func (db *DB) AlterSchema(ctx context.Context, sch string) error { - return db.driver.alterSchema(ctx, db, sch) +func (ns *Namespace) AlterSchema(ctx context.Context, sch string) error { + return ns.engine.alterSchema(ctx, ns, sch) } -func (db *DB) Mutate(ctx context.Context, ms []*api.Mutation) (map[string]uint64, error) { - return db.driver.mutate(ctx, db, ms) +func (ns *Namespace) Mutate(ctx context.Context, ms []*api.Mutation) (map[string]uint64, error) { + return ns.engine.mutate(ctx, ns, ms) } // Query performs query or mutation or upsert on the given modusDB instance. -func (db *DB) Query(ctx context.Context, query string) (*api.Response, error) { - return db.driver.query(ctx, db, query) +func (ns *Namespace) Query(ctx context.Context, query string) (*api.Response, error) { + return ns.engine.query(ctx, ns, query) } diff --git a/db_test.go b/db_test.go index 6a4045c..0e554b0 100644 --- a/db_test.go +++ b/db_test.go @@ -20,17 +20,17 @@ import ( ) func TestNonGalaxyDB(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(context.Background())) - require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) + require.NoError(t, ns1.DropData(context.Background())) + require.NoError(t, ns1.AlterSchema(context.Background(), "name: string @index(exact) .")) - _, err = db1.Mutate(context.Background(), []*api.Mutation{ + _, err = ns1.Mutate(context.Background(), []*api.Mutation{ { Set: []*api.NQuad{ { @@ -48,24 +48,24 @@ func TestNonGalaxyDB(t *testing.T) { name } }` - resp, err := db1.Query(context.Background(), query) + resp, err := ns1.Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) } func TestDropData(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, db1.DropData(context.Background())) - require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) + require.NoError(t, ns1.DropData(context.Background())) + require.NoError(t, ns1.AlterSchema(context.Background(), "name: string @index(exact) .")) - _, err = db1.Mutate(context.Background(), []*api.Mutation{ + _, err = ns1.Mutate(context.Background(), []*api.Mutation{ { Set: []*api.NQuad{ { @@ -83,30 +83,30 @@ func TestDropData(t *testing.T) { name } }` - resp, err := db1.Query(context.Background(), query) + resp, err := ns1.Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) - require.NoError(t, db1.DropData(context.Background())) + require.NoError(t, ns1.DropData(context.Background())) - resp, err = db1.Query(context.Background(), query) + resp, err = ns1.Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) } func TestMultipleDBs(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db0, err := driver.GetDB(0) + db0, err := engine.GetNamespace(0) require.NoError(t, err) - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, engine.DropAll(context.Background())) require.NoError(t, db0.AlterSchema(context.Background(), "name: string @index(exact) .")) - require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) + require.NoError(t, ns1.AlterSchema(context.Background(), "name: string @index(exact) .")) _, err = db0.Mutate(context.Background(), []*api.Mutation{ { @@ -121,7 +121,7 @@ func TestMultipleDBs(t *testing.T) { }) require.NoError(t, err) - _, err = db1.Mutate(context.Background(), []*api.Mutation{ + _, err = ns1.Mutate(context.Background(), []*api.Mutation{ { Set: []*api.NQuad{ { @@ -143,29 +143,29 @@ func TestMultipleDBs(t *testing.T) { require.NoError(t, err) require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(resp.GetJson())) - resp, err = db1.Query(context.Background(), query) + resp, err = ns1.Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"name":"B"}]}`, string(resp.GetJson())) - require.NoError(t, db1.DropData(context.Background())) - resp, err = db1.Query(context.Background(), query) + require.NoError(t, ns1.DropData(context.Background())) + resp, err = ns1.Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) } func TestQueryWrongDB(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db0, err := driver.GetDB(0) + db0, err := engine.GetNamespace(0) require.NoError(t, err) - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, engine.DropAll(context.Background())) require.NoError(t, db0.AlterSchema(context.Background(), "name: string @index(exact) .")) - require.NoError(t, db1.AlterSchema(context.Background(), "name: string @index(exact) .")) + require.NoError(t, ns1.AlterSchema(context.Background(), "name: string @index(exact) .")) _, err = db0.Mutate(context.Background(), []*api.Mutation{ { @@ -187,24 +187,24 @@ func TestQueryWrongDB(t *testing.T) { } }` - resp, err := db1.Query(context.Background(), query) + resp, err := ns1.Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[]}`, string(resp.GetJson())) } func TestTwoDBs(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - db0, err := driver.GetDB(0) + db0, err := engine.GetNamespace(0) require.NoError(t, err) - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, engine.DropAll(context.Background())) require.NoError(t, db0.AlterSchema(context.Background(), "foo: string @index(exact) .")) - require.NoError(t, db1.AlterSchema(context.Background(), "bar: string @index(exact) .")) + require.NoError(t, ns1.AlterSchema(context.Background(), "bar: string @index(exact) .")) _, err = db0.Mutate(context.Background(), []*api.Mutation{ { @@ -219,7 +219,7 @@ func TestTwoDBs(t *testing.T) { }) require.NoError(t, err) - _, err = db1.Mutate(context.Background(), []*api.Mutation{ + _, err = ns1.Mutate(context.Background(), []*api.Mutation{ { Set: []*api.NQuad{ { @@ -246,23 +246,23 @@ func TestTwoDBs(t *testing.T) { bar } }` - resp, err = db1.Query(context.Background(), query) + resp, err = ns1.Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"bar":"B"}]}`, string(resp.GetJson())) } func TestDBDBRestart(t *testing.T) { dataDir := t.TempDir() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(dataDir)) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(dataDir)) require.NoError(t, err) - defer func() { driver.Close() }() + defer func() { engine.Close() }() - db1, err := driver.CreateDB() + ns1, err := engine.CreateNamespace() require.NoError(t, err) - ns1 := db1.ID() + ns1Id := ns1.ID() - require.NoError(t, db1.AlterSchema(context.Background(), "bar: string @index(exact) .")) - _, err = db1.Mutate(context.Background(), []*api.Mutation{ + require.NoError(t, ns1.AlterSchema(context.Background(), "bar: string @index(exact) .")) + _, err = ns1.Mutate(context.Background(), []*api.Mutation{ { Set: []*api.NQuad{ { @@ -275,15 +275,15 @@ func TestDBDBRestart(t *testing.T) { }) require.NoError(t, err) - driver.Close() - driver, err = modusdb.NewDriver(modusdb.NewDefaultConfig(dataDir)) + engine.Close() + engine, err = modusdb.NewEngine(modusdb.NewDefaultConfig(dataDir)) require.NoError(t, err) - db2, err := driver.CreateDB() + db2, err := engine.CreateNamespace() require.NoError(t, err) - require.Greater(t, db2.ID(), ns1) + require.Greater(t, db2.ID(), ns1Id) - db1, err = driver.GetDB(ns1) + ns1, err = engine.GetNamespace(ns1Id) require.NoError(t, err) query := `{ @@ -291,7 +291,7 @@ func TestDBDBRestart(t *testing.T) { bar } }` - resp, err := db1.Query(context.Background(), query) + resp, err := ns1.Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"bar":"B"}]}`, string(resp.GetJson())) } diff --git a/driver.go b/driver.go index a1e437f..fb8fe3c 100644 --- a/driver.go +++ b/driver.go @@ -35,26 +35,26 @@ var ( // This ensures that we only have one instance of modusDB in this process. singleton atomic.Bool - ErrSingletonOnly = errors.New("only one modusDB driver is supported") + ErrSingletonOnly = errors.New("only one modusDB engine is supported") ErrEmptyDataDir = errors.New("data directory is required") - ErrClosedDriver = errors.New("modusDB driver is closed") + ErrClosedEngine = errors.New("modusDB engine is closed") ErrNonExistentDB = errors.New("namespace does not exist") ) -// Driver is an instance of modusDB. +// Engine is an instance of modusDB. // For now, we only support one instance of modusDB per process. -type Driver struct { +type Engine struct { mutex sync.RWMutex isOpen atomic.Bool z *zero // points to default / 0 / galaxy namespace - db0 *DB + db0 *Namespace } -// NewDriver returns a new modusDB instance. -func NewDriver(conf Config) (*Driver, error) { +// NewEngine returns a new modusDB instance. +func NewEngine(conf Config) (*Engine, error) { // Ensure that we do not create another instance of modusDB in the same process if !singleton.CompareAndSwap(false, true) { return nil, ErrSingletonOnly @@ -82,77 +82,77 @@ func NewDriver(conf Config) (*Driver, error) { schema.Init(worker.State.Pstore) posting.Init(worker.State.Pstore, 0) // TODO: set cache size - driver := &Driver{} - driver.isOpen.Store(true) - if err := driver.reset(); err != nil { - return nil, fmt.Errorf("error resetting db: %w", err) + engine := &Engine{} + engine.isOpen.Store(true) + if err := engine.reset(); err != nil { + return nil, fmt.Errorf("error resetting ns: %w", err) } x.UpdateHealthStatus(true) - driver.db0 = &DB{id: 0, driver: driver} - return driver, nil + engine.db0 = &Namespace{id: 0, engine: engine} + return engine, nil } -func (db *Driver) CreateDB() (*DB, error) { - db.mutex.RLock() - defer db.mutex.RUnlock() +func (engine *Engine) CreateNamespace() (*Namespace, error) { + engine.mutex.RLock() + defer engine.mutex.RUnlock() - if !db.isOpen.Load() { - return nil, ErrClosedDriver + if !engine.isOpen.Load() { + return nil, ErrClosedEngine } - startTs, err := db.z.nextTs() + startTs, err := engine.z.nextTs() if err != nil { return nil, err } - dbID, err := db.z.nextDB() + nsID, err := engine.z.nextNamespace() if err != nil { return nil, err } - if err := worker.ApplyInitialSchema(dbID, startTs); err != nil { + if err := worker.ApplyInitialSchema(nsID, startTs); err != nil { return nil, fmt.Errorf("error applying initial schema: %w", err) } for _, pred := range schema.State().Predicates() { worker.InitTablet(pred) } - return &DB{id: dbID, driver: db}, nil + return &Namespace{id: nsID, engine: engine}, nil } -func (driver *Driver) GetDB(dbID uint64) (*DB, error) { - driver.mutex.RLock() - defer driver.mutex.RUnlock() +func (engine *Engine) GetNamespace(nsID uint64) (*Namespace, error) { + engine.mutex.RLock() + defer engine.mutex.RUnlock() - return driver.getDBWithLock(dbID) + return engine.getNamespaceWithLock(nsID) } -func (driver *Driver) getDBWithLock(dbID uint64) (*DB, error) { - if !driver.isOpen.Load() { - return nil, ErrClosedDriver +func (engine *Engine) getNamespaceWithLock(nsID uint64) (*Namespace, error) { + if !engine.isOpen.Load() { + return nil, ErrClosedEngine } - if dbID > driver.z.lastDB { + if nsID > engine.z.lastNamespace { return nil, ErrNonExistentDB } // TODO: when delete namespace is implemented, check if the namespace exists - return &DB{id: dbID, driver: driver}, nil + return &Namespace{id: nsID, engine: engine}, nil } -func (driver *Driver) GetDefaultDB() *DB { - return driver.db0 +func (engine *Engine) GetDefaultNamespace() *Namespace { + return engine.db0 } // DropAll drops all the data and schema in the modusDB instance. -func (driver *Driver) DropAll(ctx context.Context) error { - driver.mutex.Lock() - defer driver.mutex.Unlock() +func (engine *Engine) DropAll(ctx context.Context) error { + engine.mutex.Lock() + defer engine.mutex.Unlock() - if !driver.isOpen.Load() { - return ErrClosedDriver + if !engine.isOpen.Load() { + return ErrClosedEngine } p := &pb.Proposal{Mutations: &pb.Mutations{ @@ -162,26 +162,26 @@ func (driver *Driver) DropAll(ctx context.Context) error { if err := worker.ApplyMutations(ctx, p); err != nil { return fmt.Errorf("error applying mutation: %w", err) } - if err := driver.reset(); err != nil { - return fmt.Errorf("error resetting db: %w", err) + if err := engine.reset(); err != nil { + return fmt.Errorf("error resetting ns: %w", err) } // TODO: insert drop record return nil } -func (driver *Driver) dropData(ctx context.Context, db *DB) error { - driver.mutex.Lock() - defer driver.mutex.Unlock() +func (engine *Engine) dropData(ctx context.Context, ns *Namespace) error { + engine.mutex.Lock() + defer engine.mutex.Unlock() - if !driver.isOpen.Load() { - return ErrClosedDriver + if !engine.isOpen.Load() { + return ErrClosedEngine } p := &pb.Proposal{Mutations: &pb.Mutations{ GroupId: 1, DropOp: pb.Mutations_DATA, - DropValue: strconv.FormatUint(db.ID(), 10), + DropValue: strconv.FormatUint(ns.ID(), 10), }} if err := worker.ApplyMutations(ctx, p); err != nil { @@ -193,27 +193,27 @@ func (driver *Driver) dropData(ctx context.Context, db *DB) error { return nil } -func (driver *Driver) alterSchema(ctx context.Context, db *DB, sch string) error { - driver.mutex.Lock() - defer driver.mutex.Unlock() +func (engine *Engine) alterSchema(ctx context.Context, ns *Namespace, sch string) error { + engine.mutex.Lock() + defer engine.mutex.Unlock() - if !driver.isOpen.Load() { - return ErrClosedDriver + if !engine.isOpen.Load() { + return ErrClosedEngine } - sc, err := schema.ParseWithNamespace(sch, db.ID()) + sc, err := schema.ParseWithNamespace(sch, ns.ID()) if err != nil { return fmt.Errorf("error parsing schema: %w", err) } - return driver.alterSchemaWithParsed(ctx, sc) + return engine.alterSchemaWithParsed(ctx, sc) } -func (driver *Driver) alterSchemaWithParsed(ctx context.Context, sc *schema.ParsedSchema) error { +func (engine *Engine) alterSchemaWithParsed(ctx context.Context, sc *schema.ParsedSchema) error { for _, pred := range sc.Preds { worker.InitTablet(pred.Predicate) } - startTs, err := driver.z.nextTs() + startTs, err := engine.z.nextTs() if err != nil { return err } @@ -230,33 +230,33 @@ func (driver *Driver) alterSchemaWithParsed(ctx context.Context, sc *schema.Pars return nil } -func (driver *Driver) query(ctx context.Context, db *DB, q string) (*api.Response, error) { - driver.mutex.RLock() - defer driver.mutex.RUnlock() +func (engine *Engine) query(ctx context.Context, ns *Namespace, q string) (*api.Response, error) { + engine.mutex.RLock() + defer engine.mutex.RUnlock() - return driver.queryWithLock(ctx, db, q) + return engine.queryWithLock(ctx, ns, q) } -func (driver *Driver) queryWithLock(ctx context.Context, db *DB, q string) (*api.Response, error) { - if !driver.isOpen.Load() { - return nil, ErrClosedDriver +func (engine *Engine) queryWithLock(ctx context.Context, ns *Namespace, q string) (*api.Response, error) { + if !engine.isOpen.Load() { + return nil, ErrClosedEngine } - ctx = x.AttachNamespace(ctx, db.ID()) + ctx = x.AttachNamespace(ctx, ns.ID()) return (&edgraph.Server{}).QueryNoAuth(ctx, &api.Request{ ReadOnly: true, Query: q, - StartTs: driver.z.readTs(), + StartTs: engine.z.readTs(), }) } -func (driver *Driver) mutate(ctx context.Context, db *DB, ms []*api.Mutation) (map[string]uint64, error) { +func (engine *Engine) mutate(ctx context.Context, ns *Namespace, ms []*api.Mutation) (map[string]uint64, error) { if len(ms) == 0 { return nil, nil } - driver.mutex.Lock() - defer driver.mutex.Unlock() + engine.mutex.Lock() + defer engine.mutex.Unlock() dms := make([]*dql.Mutation, 0, len(ms)) for _, mu := range ms { dm, err := edgraph.ParseMutationObject(mu, false) @@ -271,7 +271,7 @@ func (driver *Driver) mutate(ctx context.Context, db *DB, ms []*api.Mutation) (m } if len(newUids) > 0 { num := &pb.Num{Val: uint64(len(newUids)), Type: pb.Num_UID} - res, err := driver.z.nextUIDs(num) + res, err := engine.z.nextUIDs(num) if err != nil { return nil, err } @@ -284,26 +284,26 @@ func (driver *Driver) mutate(ctx context.Context, db *DB, ms []*api.Mutation) (m } } - return driver.mutateWithDqlMutation(ctx, db, dms, newUids) + return engine.mutateWithDqlMutation(ctx, ns, dms, newUids) } -func (driver *Driver) mutateWithDqlMutation(ctx context.Context, db *DB, dms []*dql.Mutation, +func (engine *Engine) mutateWithDqlMutation(ctx context.Context, ns *Namespace, dms []*dql.Mutation, newUids map[string]uint64) (map[string]uint64, error) { edges, err := query.ToDirectedEdges(dms, newUids) if err != nil { return nil, fmt.Errorf("error converting to directed edges: %w", err) } - ctx = x.AttachNamespace(ctx, db.ID()) + ctx = x.AttachNamespace(ctx, ns.ID()) - if !driver.isOpen.Load() { - return nil, ErrClosedDriver + if !engine.isOpen.Load() { + return nil, ErrClosedEngine } - startTs, err := driver.z.nextTs() + startTs, err := engine.z.nextTs() if err != nil { return nil, err } - commitTs, err := driver.z.nextTs() + commitTs, err := engine.z.nextTs() if err != nil { return nil, err } @@ -333,20 +333,20 @@ func (driver *Driver) mutateWithDqlMutation(ctx context.Context, db *DB, dms []* }) } -func (driver *Driver) Load(ctx context.Context, schemaPath, dataPath string) error { - return driver.db0.Load(ctx, schemaPath, dataPath) +func (engine *Engine) Load(ctx context.Context, schemaPath, dataPath string) error { + return engine.db0.Load(ctx, schemaPath, dataPath) } -func (driver *Driver) LoadData(inCtx context.Context, dataDir string) error { - return driver.db0.LoadData(inCtx, dataDir) +func (engine *Engine) LoadData(inCtx context.Context, dataDir string) error { + return engine.db0.LoadData(inCtx, dataDir) } // Close closes the modusDB instance. -func (driver *Driver) Close() { - driver.mutex.Lock() - defer driver.mutex.Unlock() +func (engine *Engine) Close() { + engine.mutex.Lock() + defer engine.mutex.Unlock() - if !driver.isOpen.Load() { + if !engine.isOpen.Load() { return } @@ -354,13 +354,13 @@ func (driver *Driver) Close() { panic("modusDB instance was not properly opened") } - driver.isOpen.Store(false) + engine.isOpen.Store(false) x.UpdateHealthStatus(false) posting.Cleanup() worker.State.Dispose() } -func (db *Driver) reset() error { +func (ns *Engine) reset() error { z, restart, err := newZero() if err != nil { return fmt.Errorf("error initializing zero: %w", err) @@ -379,6 +379,6 @@ func (db *Driver) reset() error { worker.InitTablet(pred) } - db.z = z + ns.z = z return nil } diff --git a/driver_test.go b/driver_test.go index 26dcfaa..9eee107 100644 --- a/driver_test.go +++ b/driver_test.go @@ -25,14 +25,14 @@ import ( func TestRestart(t *testing.T) { dataDir := t.TempDir() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(dataDir)) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(dataDir)) require.NoError(t, err) - defer func() { driver.Close() }() + defer func() { engine.Close() }() - require.NoError(t, driver.DropAll(context.Background())) - require.NoError(t, driver.GetDefaultDB().AlterSchema(context.Background(), "name: string @index(term) .")) + require.NoError(t, engine.DropAll(context.Background())) + require.NoError(t, engine.GetDefaultNamespace().AlterSchema(context.Background(), "name: string @index(term) .")) - _, err = driver.GetDefaultDB().Mutate(context.Background(), []*api.Mutation{ + _, err = engine.GetDefaultNamespace().Mutate(context.Background(), []*api.Mutation{ { Set: []*api.NQuad{ { @@ -51,27 +51,27 @@ func TestRestart(t *testing.T) { name } }` - qresp, err := driver.GetDefaultDB().Query(context.Background(), query) + qresp, err := engine.GetDefaultNamespace().Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) - driver.Close() - driver, err = modusdb.NewDriver(modusdb.NewDefaultConfig(dataDir)) + engine.Close() + engine, err = modusdb.NewEngine(modusdb.NewDefaultConfig(dataDir)) require.NoError(t, err) - qresp, err = driver.GetDefaultDB().Query(context.Background(), query) + qresp, err = engine.GetDefaultNamespace().Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, `{"me":[{"name":"A"}]}`, string(qresp.GetJson())) - require.NoError(t, driver.DropAll(context.Background())) + require.NoError(t, engine.DropAll(context.Background())) } func TestSchemaQuery(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - require.NoError(t, driver.DropAll(context.Background())) - require.NoError(t, driver.GetDefaultDB().AlterSchema(context.Background(), ` + require.NoError(t, engine.DropAll(context.Background())) + require.NoError(t, engine.GetDefaultNamespace().AlterSchema(context.Background(), ` name: string @index(exact) . age: int . married: bool . @@ -79,7 +79,7 @@ func TestSchemaQuery(t *testing.T) { dob: datetime . `)) - resp, err := driver.GetDefaultDB().Query(context.Background(), `schema(pred: [name, age]) {type}`) + resp, err := engine.GetDefaultNamespace().Query(context.Background(), `schema(pred: [name, age]) {type}`) require.NoError(t, err) require.JSONEq(t, @@ -95,15 +95,15 @@ func TestBasicVector(t *testing.T) { } vectBytes := buf.Bytes() - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - require.NoError(t, driver.DropAll(context.Background())) - require.NoError(t, driver.GetDefaultDB().AlterSchema(context.Background(), + require.NoError(t, engine.DropAll(context.Background())) + require.NoError(t, engine.GetDefaultNamespace().AlterSchema(context.Background(), `project_description_v: float32vector @index(hnsw(exponent: "5", metric: "euclidean")) .`)) - uids, err := driver.GetDefaultDB().Mutate(context.Background(), []*api.Mutation{{ + uids, err := engine.GetDefaultNamespace().Mutate(context.Background(), []*api.Mutation{{ Set: []*api.NQuad{{ Subject: "_:vector", Predicate: "project_description_v", @@ -119,7 +119,7 @@ func TestBasicVector(t *testing.T) { t.Fatalf("Expected non-zero uid") } - resp, err := driver.GetDefaultDB().Query(context.Background(), fmt.Sprintf(`query { + resp, err := engine.GetDefaultNamespace().Query(context.Background(), fmt.Sprintf(`query { q (func: uid(%v)) { project_description_v } diff --git a/live.go b/live.go index 2bfea22..911a9a4 100644 --- a/live.go +++ b/live.go @@ -34,29 +34,29 @@ const ( ) type liveLoader struct { - d *DB + n *Namespace blankNodes map[string]string mutex sync.RWMutex } -func (d *DB) Load(ctx context.Context, schemaPath, dataPath string) error { +func (n *Namespace) Load(ctx context.Context, schemaPath, dataPath string) error { schemaData, err := os.ReadFile(schemaPath) if err != nil { return fmt.Errorf("error reading schema file [%v]: %w", schemaPath, err) } - if err := d.AlterSchema(ctx, string(schemaData)); err != nil { + if err := n.AlterSchema(ctx, string(schemaData)); err != nil { return fmt.Errorf("error altering schema: %w", err) } - if err := d.LoadData(ctx, dataPath); err != nil { + if err := n.LoadData(ctx, dataPath); err != nil { return fmt.Errorf("error loading data: %w", err) } return nil } // TODO: Add support for CSV file -func (d *DB) LoadData(inCtx context.Context, dataDir string) error { +func (n *Namespace) LoadData(inCtx context.Context, dataDir string) error { fs := filestore.NewFileStore(dataDir) files := fs.FindDataFiles(dataDir, []string{".rdf", ".rdf.gz", ".json", ".json.gz"}) if len(files) == 0 { @@ -94,7 +94,7 @@ func (d *DB) LoadData(inCtx context.Context, dataDir string) error { if !ok { return nil } - uids, err := d.Mutate(rootCtx, []*api.Mutation{nqs}) + uids, err := n.Mutate(rootCtx, []*api.Mutation{nqs}) if err != nil { return fmt.Errorf("error applying mutations: %w", err) } @@ -104,7 +104,7 @@ func (d *DB) LoadData(inCtx context.Context, dataDir string) error { } }) - ll := &liveLoader{d: d, blankNodes: make(map[string]string)} + ll := &liveLoader{n: n, blankNodes: make(map[string]string)} for _, datafile := range files { procG.Go(func() error { return ll.processFile(procCtx, fs, datafile, nqch) @@ -246,7 +246,7 @@ func (l *liveLoader) uid(ns uint64, val string) (string, error) { return uid, nil } - asUID, err := l.d.driver.LeaseUIDs(1) + asUID, err := l.n.engine.LeaseUIDs(1) if err != nil { return "", fmt.Errorf("error allocating UID: %w", err) } diff --git a/live_benchmark_test.go b/live_benchmark_test.go index 036862d..fd1572f 100644 --- a/live_benchmark_test.go +++ b/live_benchmark_test.go @@ -49,9 +49,9 @@ func BenchmarkDatabaseOperations(b *testing.B) { runtime.ReadMemStats(&ms) initialAlloc := ms.Alloc - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(b.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(b.TempDir())) require.NoError(b, err) - defer driver.Close() + defer engine.Close() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -60,7 +60,7 @@ func BenchmarkDatabaseOperations(b *testing.B) { dataFile := filepath.Join(dataFolder, "data.rdf") require.NoError(b, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) require.NoError(b, os.WriteFile(dataFile, []byte(SmallData), 0600)) - require.NoError(b, driver.Load(context.Background(), schemaFile, dataFile)) + require.NoError(b, engine.Load(context.Background(), schemaFile, dataFile)) } reportMemStats(b, initialAlloc) }) @@ -75,16 +75,16 @@ func BenchmarkDatabaseOperations(b *testing.B) { initialAlloc := ms.Alloc // Setup database with data once - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(b.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(b.TempDir())) require.NoError(b, err) - defer driver.Close() + defer engine.Close() dataFolder := b.TempDir() schemaFile := filepath.Join(dataFolder, "data.schema") dataFile := filepath.Join(dataFolder, "data.rdf") require.NoError(b, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) require.NoError(b, os.WriteFile(dataFile, []byte(SmallData), 0600)) - require.NoError(b, driver.Load(context.Background(), schemaFile, dataFile)) + require.NoError(b, engine.Load(context.Background(), schemaFile, dataFile)) const query = `{ caro(func: allofterms(name@en, "Marc Caro")) { @@ -112,7 +112,7 @@ func BenchmarkDatabaseOperations(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - resp, err := driver.GetDefaultDB().Query(context.Background(), query) + resp, err := engine.GetDefaultNamespace().Query(context.Background(), query) require.NoError(b, err) require.JSONEq(b, expected, string(resp.Json)) } diff --git a/live_test.go b/live_test.go index 62a2731..9b44a62 100644 --- a/live_test.go +++ b/live_test.go @@ -49,16 +49,16 @@ const ( func TestLiveLoaderSmall(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() dataFolder := t.TempDir() schemaFile := filepath.Join(dataFolder, "data.schema") dataFile := filepath.Join(dataFolder, "data.rdf") require.NoError(t, os.WriteFile(schemaFile, []byte(DbSchema), 0600)) require.NoError(t, os.WriteFile(dataFile, []byte(SmallData), 0600)) - require.NoError(t, driver.Load(context.Background(), schemaFile, dataFile)) + require.NoError(t, engine.Load(context.Background(), schemaFile, dataFile)) const query = `{ caro(func: allofterms(name@en, "Marc Caro")) { @@ -84,15 +84,15 @@ func TestLiveLoaderSmall(t *testing.T) { ] }` - resp, err := driver.GetDefaultDB().Query(context.Background(), query) + resp, err := engine.GetDefaultNamespace().Query(context.Background(), query) require.NoError(t, err) require.JSONEq(t, expected, string(resp.Json)) } func TestLiveLoader1Million(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() baseDir := t.TempDir() schResp, err := grab.Get(baseDir, oneMillionSchema) @@ -100,12 +100,12 @@ func TestLiveLoader1Million(t *testing.T) { dataResp, err := grab.Get(baseDir, oneMillionRDF) require.NoError(t, err) - require.NoError(t, driver.DropAll(context.Background())) - require.NoError(t, driver.Load(context.Background(), schResp.Filename, dataResp.Filename)) + require.NoError(t, engine.DropAll(context.Background())) + require.NoError(t, engine.Load(context.Background(), schResp.Filename, dataResp.Filename)) for _, tt := range common.OneMillionTCs { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) - resp, err := driver.GetDefaultDB().Query(ctx, tt.Query) + resp, err := engine.GetDefaultNamespace().Query(ctx, tt.Query) cancel() if ctx.Err() == context.DeadlineExceeded { diff --git a/vector_test.go b/vector_test.go index f944fa1..f5574f9 100644 --- a/vector_test.go +++ b/vector_test.go @@ -29,20 +29,20 @@ const ( ) func TestVectorDelete(t *testing.T) { - driver, err := modusdb.NewDriver(modusdb.NewDefaultConfig(t.TempDir())) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig(t.TempDir())) require.NoError(t, err) - defer driver.Close() + defer engine.Close() - require.NoError(t, driver.DropAll(context.Background())) - require.NoError(t, driver.GetDefaultDB().AlterSchema(context.Background(), + require.NoError(t, engine.DropAll(context.Background())) + require.NoError(t, engine.GetDefaultNamespace().AlterSchema(context.Background(), fmt.Sprintf(vectorSchemaWithIndex, "vtest", "4", "euclidean"))) // insert random vectors - assignIDs, err := driver.LeaseUIDs(numVectors + 1) + assignIDs, err := engine.LeaseUIDs(numVectors + 1) require.NoError(t, err) //nolint:gosec rdf, vectors := dgraphapi.GenerateRandomVectors(int(assignIDs.StartId)-10, int(assignIDs.EndId)-10, 10, "vtest") - _, err = driver.GetDefaultDB().Mutate(context.Background(), []*api.Mutation{{SetNquads: []byte(rdf)}}) + _, err = engine.GetDefaultNamespace().Mutate(context.Background(), []*api.Mutation{{SetNquads: []byte(rdf)}}) require.NoError(t, err) // check the count of the vectors inserted @@ -51,7 +51,7 @@ func TestVectorDelete(t *testing.T) { count(uid) } }` - resp, err := driver.GetDefaultDB().Query(context.Background(), q1) + resp, err := engine.GetDefaultNamespace().Query(context.Background(), q1) require.NoError(t, err) require.JSONEq(t, fmt.Sprintf(`{"vector":[{"count":%d}]}`, numVectors), string(resp.Json)) @@ -64,11 +64,11 @@ func TestVectorDelete(t *testing.T) { } }` - require.Equal(t, vectors, queryVectors(t, driver, vectorQuery)) + require.Equal(t, vectors, queryVectors(t, engine, vectorQuery)) triples := strings.Split(rdf, "\n") deleteTriple := func(idx int) string { - _, err := driver.GetDefaultDB().Mutate(context.Background(), []*api.Mutation{{ + _, err := engine.GetDefaultNamespace().Mutate(context.Background(), []*api.Mutation{{ DelNquads: []byte(triples[idx]), }}) require.NoError(t, err) @@ -80,7 +80,7 @@ func TestVectorDelete(t *testing.T) { } }`, uid[1:len(uid)-1]) - res, err := driver.GetDefaultDB().Query(context.Background(), q2) + res, err := engine.GetDefaultNamespace().Query(context.Background(), q2) require.NoError(t, err) require.JSONEq(t, `{"vector":[]}`, string(res.Json)) return triples[idx] @@ -96,17 +96,17 @@ func TestVectorDelete(t *testing.T) { for i := 0; i < len(triples)-2; i++ { triple := deleteTriple(i) vectorQuery := fmt.Sprintf(q3, strings.Split(triple, `"`)[1]) - respVectors := queryVectors(t, driver, vectorQuery) + respVectors := queryVectors(t, engine, vectorQuery) require.Len(t, respVectors, 1) require.Contains(t, vectors, respVectors[0]) } triple := deleteTriple(len(triples) - 2) - _ = queryVectors(t, driver, fmt.Sprintf(q3, strings.Split(triple, `"`)[1])) + _ = queryVectors(t, engine, fmt.Sprintf(q3, strings.Split(triple, `"`)[1])) } -func queryVectors(t *testing.T, driver *modusdb.Driver, query string) [][]float32 { - resp, err := driver.GetDefaultDB().Query(context.Background(), query) +func queryVectors(t *testing.T, engine *modusdb.Engine, query string) [][]float32 { + resp, err := engine.GetDefaultNamespace().Query(context.Background(), query) require.NoError(t, err) var data struct { diff --git a/zero.go b/zero.go index 33d3e44..7dc7bad 100644 --- a/zero.go +++ b/zero.go @@ -34,9 +34,9 @@ const ( zeroStateKey = "0-dgraph.modusdb.zero" ) -func (db *Driver) LeaseUIDs(numUIDs uint64) (*pb.AssignedIds, error) { +func (ns *Engine) LeaseUIDs(numUIDs uint64) (*pb.AssignedIds, error) { num := &pb.Num{Val: numUIDs, Type: pb.Num_UID} - return db.z.nextUIDs(num) + return ns.z.nextUIDs(num) } type zero struct { @@ -46,7 +46,7 @@ type zero struct { minLeasedTs uint64 maxLeasedTs uint64 - lastDB uint64 + lastNamespace uint64 } func newZero() (*zero, bool, error) { @@ -62,13 +62,13 @@ func newZero() (*zero, bool, error) { z.maxLeasedUID = initialUID z.minLeasedTs = initialTs z.maxLeasedTs = initialTs - z.lastDB = 0 + z.lastNamespace = 0 } else { z.minLeasedUID = zs.MaxUID z.maxLeasedUID = zs.MaxUID z.minLeasedTs = zs.MaxTxnTs z.maxLeasedTs = zs.MaxTxnTs - z.lastDB = zs.MaxNsID + z.lastNamespace = zs.MaxNsID } posting.Oracle().ProcessDelta(&pb.OracleDelta{MaxAssigned: z.minLeasedTs - 1}) worker.SetMaxUID(z.minLeasedUID - 1) @@ -133,12 +133,12 @@ func (z *zero) nextUIDs(num *pb.Num) (*pb.AssignedIds, error) { return resp, nil } -func (z *zero) nextDB() (uint64, error) { - z.lastDB++ +func (z *zero) nextNamespace() (uint64, error) { + z.lastNamespace++ if err := z.writeZeroState(); err != nil { return 0, fmt.Errorf("error leasing namespace ID: %w", err) } - return z.lastDB, nil + return z.lastNamespace, nil } func readZeroState() (*pb.MembershipState, error) { @@ -165,7 +165,7 @@ func readZeroState() (*pb.MembershipState, error) { } func (z *zero) writeZeroState() error { - zeroState := &pb.MembershipState{MaxUID: z.maxLeasedUID, MaxTxnTs: z.maxLeasedTs, MaxNsID: z.lastDB} + zeroState := &pb.MembershipState{MaxUID: z.maxLeasedUID, MaxTxnTs: z.maxLeasedTs, MaxNsID: z.lastNamespace} data, err := proto.Marshal(zeroState) if err != nil { return fmt.Errorf("error marshalling zero state: %w", err) From af542861a38c772b4364095d2eb56b8c4e323ede Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:51:51 -0800 Subject: [PATCH 08/10] update --- README.md | 2 +- api/structreflect/tagparser.go | 2 +- api_test.go | 10 +++++----- driver.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ae83cd2..b0e8dd1 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ import ( type User struct { Gid uint64 `json:"gid,omitempty"` - Id string `json:"id,omitempty" ns:"constraint=unique"` + Id string `json:"id,omitempty" db:"constraint=unique"` Name string `json:"name,omitempty"` Age int `json:"age,omitempty"` } diff --git a/api/structreflect/tagparser.go b/api/structreflect/tagparser.go index 0d94918..da9224f 100644 --- a/api/structreflect/tagparser.go +++ b/api/structreflect/tagparser.go @@ -26,7 +26,7 @@ func parseJsonTag(field reflect.StructField) (string, error) { } func parseDbTag(field reflect.StructField) *DbTag { - dbConstraintsTag := field.Tag.Get("ns") + dbConstraintsTag := field.Tag.Get("db") if dbConstraintsTag == "" { return nil } diff --git a/api_test.go b/api_test.go index 068089a..537a4e8 100644 --- a/api_test.go +++ b/api_test.go @@ -24,7 +24,7 @@ type User struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` Age int `json:"age,omitempty"` - ClerkId string `json:"clerk_id,omitempty" ns:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` } func TestFirstTimeUser(t *testing.T) { @@ -383,14 +383,14 @@ func TestQueryApiWithPaginiationAndSorting(t *testing.T) { type Project struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" ns:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` Branches []Branch `json:"branches,omitempty" readFrom:"type=Branch,field=proj"` } type Branch struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" ns:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` Proj Project `json:"proj,omitempty"` } @@ -722,7 +722,7 @@ type BadProject struct { type BadBranch struct { Gid uint64 `json:"gid,omitempty"` Name string `json:"name,omitempty"` - ClerkId string `json:"clerk_id,omitempty" ns:"constraint=unique"` + ClerkId string `json:"clerk_id,omitempty" db:"constraint=unique"` Proj BadProject `json:"proj,omitempty"` } @@ -764,7 +764,7 @@ func TestNestedObjectMutationWithBadType(t *testing.T) { type Document struct { Gid uint64 `json:"gid,omitempty"` Text string `json:"text,omitempty"` - TextVec []float32 `json:"textVec,omitempty" ns:"constraint=vector"` + TextVec []float32 `json:"textVec,omitempty" db:"constraint=vector"` } func TestVectorIndexSearchTyped(t *testing.T) { diff --git a/driver.go b/driver.go index fb8fe3c..9be39a3 100644 --- a/driver.go +++ b/driver.go @@ -85,7 +85,7 @@ func NewEngine(conf Config) (*Engine, error) { engine := &Engine{} engine.isOpen.Store(true) if err := engine.reset(); err != nil { - return nil, fmt.Errorf("error resetting ns: %w", err) + return nil, fmt.Errorf("error resetting db: %w", err) } x.UpdateHealthStatus(true) @@ -163,7 +163,7 @@ func (engine *Engine) DropAll(ctx context.Context) error { return fmt.Errorf("error applying mutation: %w", err) } if err := engine.reset(); err != nil { - return fmt.Errorf("error resetting ns: %w", err) + return fmt.Errorf("error resetting db: %w", err) } // TODO: insert drop record From fb65f7d97b812837316bc245086c6d73081641bd Mon Sep 17 00:00:00 2001 From: Jai Radhakrishnan <55522316+jairad26@users.noreply.github.com> Date: Wed, 15 Jan 2025 00:12:21 -0800 Subject: [PATCH 09/10] revs --- README.md | 6 +++--- driver.go => engine.go | 0 driver_test.go => engine_test.go | 0 db.go => namespace.go | 0 db_test.go => namespace_test.go | 0 5 files changed, 3 insertions(+), 3 deletions(-) rename driver.go => engine.go (100%) rename driver_test.go => engine_test.go (100%) rename db.go => namespace.go (100%) rename db_test.go => namespace_test.go (100%) diff --git a/README.md b/README.md index b0e8dd1..f392352 100644 --- a/README.md +++ b/README.md @@ -46,11 +46,11 @@ type User struct { } func main() { - ns, err := New(NewDefaultConfig("/tmp/modusdb")) + engine, err := modusdb.NewEngine(modusdb.NewDefaultConfig("/local/modusdb")) if err != nil { panic(err) } - defer ns.Close() + defer engine.Close() gid, user, err := modusdb.Upsert(ns, User{ Id: "123", @@ -86,7 +86,7 @@ like to get involved. Modus and its components are Copyright 2025 Hypermode Inc., and licensed under the terms of the Apache License, Version 2.0. See the [LICENSE](./LICENSE) file for a complete copy of the license. If you have any questions about modus licensing, or need an alternate license or other arrangement, -please contact us at hello@hypermode.com. +please contact us at . ## Acknowledgements diff --git a/driver.go b/engine.go similarity index 100% rename from driver.go rename to engine.go diff --git a/driver_test.go b/engine_test.go similarity index 100% rename from driver_test.go rename to engine_test.go diff --git a/db.go b/namespace.go similarity index 100% rename from db.go rename to namespace.go diff --git a/db_test.go b/namespace_test.go similarity index 100% rename from db_test.go rename to namespace_test.go From 833acddfa7ff75317932c419a7887b7abe50e7c1 Mon Sep 17 00:00:00 2001 From: Aman Mangal Date: Wed, 15 Jan 2025 13:45:35 +0530 Subject: [PATCH 10/10] Trigger Build