diff --git a/README.md b/README.md index fc96ca4..d770b9a 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,443 @@ func main() { } ``` +## Creating a Client + +The `NewClient` function takes a URI and optional configuration options. + +```go +client, err := mg.NewClient(uri) +if err != nil { + panic(err) +} +defer client.Close() +``` + +### URI Options + +modusGraph supports two URI schemes for managing graph databases: + +#### `file://` - Local File-Based Database + +Connects to a database stored locally on the filesystem. This mode doesn't require a separate +database server and is perfect for development, testing, or embedded applications. The directory +must exist before connecting. + +File-based databases do not support concurrent access from separate processes. + +```go +// Connect to a local file-based database +client, err := mg.NewClient("file:///path/to/data") +``` + +#### `dgraph://` - Remote Dgraph Server + +Connects to a Dgraph cluster. For more details on the Dgraph URI format, see the +[Dgraph Dgo documentation](https://github.com/hypermodeinc/dgo#connection-strings). + +```go +// Connect to a remote Dgraph server +client, err := mg.NewClient("dgraph://hostname:9080") +``` + +### Configuration Options + +modusGraph provides several configuration options that can be passed to the `NewClient` function: + +#### WithAutoSchema(bool) + +Enables or disables automatic schema management. When enabled, modusGraph will automatically create +and update the graph database schema based on the struct tags of objects you insert. + +```go +// Enable automatic schema management +client, err := mg.NewClient(uri, mg.WithAutoSchema(true)) +``` + +#### WithPoolSize(int) + +Sets the size of the connection pool for better performance under load. The default is 10 +connections. + +```go +// Set pool size to 20 connections +client, err := mg.NewClient(uri, mg.WithPoolSize(20)) +``` + +#### WithLogger(logr.Logger) + +Configures structured logging with custom verbosity levels. By default, logging is disabled. + +```go +// Set up a logger +logger := logr.New(logr.Discard()) +client, err := mg.NewClient(uri, mg.WithLogger(logger)) +``` + +You can combine multiple options: + +```go +// Using multiple configuration options +client, err := mg.NewClient(uri, + mg.WithAutoSchema(true), + mg.WithPoolSize(20), + mg.WithLogger(logger)) +``` + +## Defining Your Graph with Structs + +modusGraph uses Go structs to define your graph database schema. By adding `json` and `dgraph` tags +to your struct fields, you tell modusGraph how to store and index your data in the graph database. + +### Basic Structure + +Every struct that represents a node in your graph should include: + +```go +type MyNode struct { + // Your fields here with appropriate tags + + // These fields are required for Dgraph integration + UID string `json:"uid,omitempty"` + DType []string `json:"dgraph.type,omitempty"` +} +``` + +### `dgraph` Field Tags + +modusGraph uses struct tags to define how each field should be handled in the graph database: + +| Directive | Option | Description | Example | +| ----------- | -------- | ------------------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| **index** | exact | Creates an exact-match index for string fields | Name string `json:"name" dgraph:"index=exact"` | +| | hash | Creates a hash index (same as exact) | Code string `json:"code" dgraph:"index=hash"` | +| | term | Creates a term index for text search | Description string `json:"description" dgraph:"index=term"` | +| | fulltext | Creates a full-text search index | Content string `json:"content" dgraph:"index=fulltext"` | +| | int | Creates an index for integer fields | Age int `json:"age" dgraph:"index=int"` | +| | geo | Creates a geolocation index | Location `json:"location" dgraph:"index=geo"` | +| | day | Creates a day-based index for datetime fields | Created time.Time `json:"created" dgraph:"index=day"` | +| | year | Creates a year-based index for datetime fields | Birthday time.Time `json:"birthday" dgraph:"index=year"` | +| | month | Creates a month-based index for datetime fields | Hired time.Time `json:"hired" dgraph:"index=month"` | +| | hour | Creates an hour-based index for datetime fields | Login time.Time `json:"login" dgraph:"index=hour"` | +| | hnsw | Creates a vector similarity index | Vector \*dg.VectorFloat32 `json:"vector" dgraph:"index=hnsw(metric:cosine)"` | +| **type** | geo | Specifies a geolocation field | Location `json:"location" dgraph:"type=geo"` | +| | datetime | Specifies a datetime field | CreatedAt time.Time `json:"createdAt" dgraph:"type=datetime"` | +| | int | Specifies an integer field | Count int `json:"count" dgraph:"type=int"` | +| | float | Specifies a floating-point field | Price float64 `json:"price" dgraph:"type=float"` | +| | bool | Specifies a boolean field | Active bool `json:"active" dgraph:"type=bool"` | +| | password | Specifies a password field (stored securely) | Password string `json:"password" dgraph:"type=password"` | +| **count** | | Creates a count index | Visits int `json:"visits" dgraph:"count"` | +| **unique** | | Enforces uniqueness for the field (remote Dgraph only) | Email string `json:"email" dgraph:"index=hash unique"` | +| **upsert** | | Allows a field to be used in upsert operations (remote Dgraph only) | UserID string `json:"userID" dgraph:"index=hash upsert"` | +| **reverse** | | Creates a bidirectional edge | Friends []\*Person `json:"friends" dgraph:"reverse"` | +| **lang** | | Enables multi-language support for the field | Description string `json:"description" dgraph:"lang"` | + +### Relationships + +Relationships between nodes are defined using struct pointers or slices of struct pointers: + +```go +type Person struct { + Name string `json:"name,omitempty" dgraph:"index=exact"` + Friends []*Person `json:"friends,omitempty"` + Manager *Person `json:"manager,omitempty"` + + UID string `json:"uid,omitempty"` + DType []string `json:"dgraph.type,omitempty"` +} +``` + +### Reverse Edges + +Reverse edges allow efficient bidirectional traversal. When you query in the reverse direction, use +the tilde prefix in your JSON tag: + +```go +type Student struct { + Name string `json:"name,omitempty" dgraph:"index=exact"` + Takes_Class []*Class `json:"takes_class,omitempty" dgraph:"reverse"` + + UID string `json:"uid,omitempty"` + DType []string `json:"dgraph.type,omitempty"` +} + +type Class struct { + Name string `json:"name,omitempty" dgraph:"index=exact"` + Students []*Student `json:"~takes_class,omitempty"` // Reverse edge + + UID string `json:"uid,omitempty"` + DType []string `json:"dgraph.type,omitempty"` +} +``` + +Advanced querying is required to properly bind reverse edges in query results. See the +`TestReverseEdgeQuery` test in [query_test.go](./query_test.go) for an example. + +## Basic Operations + +modusGraph provides a simple API for common database operations. + +### Inserting Data + +To insert a new node into the database: + +```go +ctx := context.Background() + +// Create a new object +user := User{ + Name: "John Doe", + Email: "john@example.com", + Role: "Admin", +} + +// Insert it into the database +err := client.Insert(ctx, &user) +if err != nil { + log.Fatalf("Failed to create user: %v", err) +} + +// The UID field will be populated after insertion +fmt.Println("Created user with UID:", user.UID) +``` + +### Updating Data + +To update an existing node, first retrieve it, modify it, then save it back: + +```go +ctx := context.Background() + +// Get the existing object by UID +var user User +err := client.Get(ctx, &user, "0x1234") +if err != nil { + log.Fatalf("Failed to get user: %v", err) +} + +// Modify fields +user.Name = "Jane Doe" +user.Role = "Manager" + +// Save the changes +err = client.Update(ctx, &user) +if err != nil { + log.Fatalf("Failed to update user: %v", err) +} +``` + +### Deleting Data + +To delete one or more nodes from the database: + +```go +ctx := context.Background() + +// Delete by UID +err := client.Delete(ctx, []string{"0x1234", "0x5678"}) +if err != nil { + log.Fatalf("Failed to delete node: %v", err) +} +``` + +### Querying Data + +modusGraph provides a basic query API for retrieving data: + +```go +ctx := context.Background() + +// Basic query to get all users +var users []User +err := client.Query(ctx, User{}).Nodes(&users) +if err != nil { + log.Fatalf("Failed to query users: %v", err) +} + +// Query with filters +var adminUsers []User +err = client.Query(ctx, User{}). + Filter(`eq(role, "Admin")`). + Nodes(&adminUsers) +if err != nil { + log.Fatalf("Failed to query admin users: %v", err) +} + +// Query with pagination +var pagedUsers []User +err = client.Query(ctx, User{}). + Filter(`has(name)`). + Offset(10). + Limit(5). + Nodes(&pagedUsers) +if err != nil { + log.Fatalf("Failed to query paged users: %v", err) +} + +// Query with ordering +var sortedUsers []User +err = client.Query(ctx, User{}). + Order("name"). + Nodes(&sortedUsers) +if err != nil { + log.Fatalf("Failed to query sorted users: %v", err) +} +``` + +### Advanced Querying + +modusGraph is built on top of the [dgman](https://github.com/dolan-in/dgman) package, which provides +access to Dgraph's more powerful and complete query capabilities. For advanced use cases, you can +access the underlying Dgraph client directly and construct more sophisticated queries: + +```go +// Define a struct with vector field for similarity search +type Product struct { + Name string `json:"name,omitempty" dgraph:"index=term"` + Description string `json:"description,omitempty"` + Vector *dg.VectorFloat32 `json:"vector,omitempty" dgraph:"index=hnsw(metric:cosine)"` + + UID string `json:"uid,omitempty"` + DType []string `json:"dgraph.type,omitempty"` +} + +// Get similar products using vector similarity search +func getSimilarProducts(client mg.Client, embeddings []float32) (*Product, error) { + ctx := context.Background() + + // Convert vector to string format for query + vectorStr := fmt.Sprintf("%v", embeddings) + vectorStr = strings.Trim(strings.ReplaceAll(vectorStr, " ", ", "), "[]") + + // Create result variable + var result Product + + // Get access to the underlying Dgraph client + dgo, cleanup, err := client.DgraphClient() + if err != nil { + return nil, err + } + defer cleanup() + + // Construct query using similar_to function with a parameter for the vector + query := dg.NewQuery().Model(&result).RootFunc("similar_to(vector, 1, $vec)") + + // Execute query with variables + tx := dg.NewReadOnlyTxn(dgo) + err = tx.Query(query). + Vars("similar_to($vec: string)", map[string]string{"$vec": vectorStr}). + Scan() + + if err != nil { + return nil, err + } + + return &result, nil +} +``` + +This example demonstrates vector similarity search for finding semantically similar items - a +powerful feature in Dgraph. You can also access other advanced capabilities like full-text search +with language-specific analyzers, geolocation queries, and more. The ability to access the raw +Dgraph client gives you the full power of Dgraph's query language while still benefiting from +modusGraph's simplified client interface and schema management. + +## Schema Management + +modusGraph provides robust schema management features that simplify working with Dgraph's schema +system. + +### AutoSchema + +The AutoSchema feature automatically generates and updates the database schema based on your Go +struct definitions. When enabled, modusGraph will analyze the struct tags of objects you insert and +ensure the appropriate schema exists in the database. + +Enable AutoSchema when creating a client: + +```go +// Enable automatic schema management +client, err := mg.NewClient(uri, mg.WithAutoSchema(true)) +if err != nil { + log.Fatalf("Failed to create client: %v", err) +} + +// Now you can insert objects without manually creating the schema first +user := User{ + Name: "John Doe", + Email: "john@example.com", +} + +// The schema will be automatically created or updated as needed +err = client.Insert(ctx, &user) +``` + +With AutoSchema enabled, modusGraph will: + +1. Analyze the struct tags of objects being inserted +2. Generate the appropriate Dgraph schema based on these tags +3. Apply any necessary schema updates to the database +4. Handle type definitions for node types based on struct names + +This is particularly useful during development when your schema is evolving frequently. + +### Schema Operations + +For more control over schema management, modusGraph provides several methods in the Client +interface: + +#### UpdateSchema + +Manually update the schema based on one or more struct types: + +```go +// Update schema based on User and Post structs +err := client.UpdateSchema(ctx, User{}, Post{}) +if err != nil { + log.Fatalf("Failed to update schema: %v", err) +} +``` + +This is useful when you want to ensure the schema is created before inserting data, or when you need +to update the schema for new struct types. + +#### GetSchema + +Retrieve the current schema definition from the database: + +```go +// Get the current schema +schema, err := client.GetSchema(ctx) +if err != nil { + log.Fatalf("Failed to get schema: %v", err) +} + +fmt.Println("Current schema:") +fmt.Println(schema) +``` + +The returned schema is in Dgraph Schema Definition Language format. + +#### DropAll and DropData + +Reset the database completely or just clear the data: + +```go +// Remove all data but keep the schema +err := client.DropData(ctx) +if err != nil { + log.Fatalf("Failed to drop data: %v", err) +} + +// Or remove both schema and data +err = client.DropAll(ctx) +if err != nil { + log.Fatalf("Failed to drop all: %v", err) +} +``` + +These operations are useful for testing or when you need to reset your database state. + ## Limitations modusGraph has a few limitations to be aware of: @@ -155,8 +592,8 @@ us at . modusGraph (and its dependencies) are designed to work on POSIX-compliant operating systems, and are not guaranteed to work on Windows. -Tests at the top level folder (`go test .`) on Windows are maintained to pass, but other tests in -subfolders may not work as expected. +Tests at the top level folder (`go test .`) on Windows are maintained to pass on Windows, but other +tests in subfolders may not work as expected. Temporary folders created during tests may not be cleaned up properly on Windows. Users should periodically clean up these folders. The temporary folders are created in the Windows temp diff --git a/api.go b/api.go index ae60883..65f9173 100644 --- a/api.go +++ b/api.go @@ -16,6 +16,7 @@ import ( "github.com/hypermodeinc/modusgraph/api/structreflect" ) +// Deprecated: Use NewClient and client.Insert instead. func Create[T any](ctx context.Context, engine *Engine, object T, nsId ...uint64) (uint64, T, error) { engine.mutex.Lock() @@ -53,6 +54,7 @@ func Create[T any](ctx context.Context, engine *Engine, object T, return getByGid[T](ctx, ns, gid) } +// Deprecated func Upsert[T any](ctx context.Context, engine *Engine, object T, nsId ...uint64) (uint64, T, bool, error) { @@ -126,6 +128,7 @@ func Upsert[T any](ctx context.Context, engine *Engine, object T, return gid, object, wasFound, nil } +// Deprecated: Use NewClient and client.Get instead. func Get[T any, R UniqueField](ctx context.Context, engine *Engine, uniqueField R, nsId ...uint64) (uint64, T, error) { engine.mutex.Lock() @@ -159,6 +162,7 @@ func Get[T any, R UniqueField](ctx context.Context, engine *Engine, uniqueField return 0, obj, errors.New("invalid unique field type") } +// Deprecated: Use NewClient and client.Query instead. func Query[T any](ctx context.Context, engine *Engine, queryParams QueryParams, nsId ...uint64) ([]uint64, []T, error) { engine.mutex.Lock() @@ -174,6 +178,7 @@ func Query[T any](ctx context.Context, engine *Engine, queryParams QueryParams, return executeQuery[T](ctx, ns, queryParams, true) } +// Deprecated: Use NewClient and client.Delete instead. func Delete[T any, R UniqueField](ctx context.Context, engine *Engine, uniqueField R, nsId ...uint64) (uint64, T, error) { engine.mutex.Lock() diff --git a/client.go b/client.go index 4000879..1ac0778 100644 --- a/client.go +++ b/client.go @@ -20,23 +20,55 @@ import ( // Client provides an interface for Dgraph operations type Client interface { + // Insert adds a new object or slice of objects to the database. + // The object must be a pointer to a struct with appropriate dgraph tags. Insert(context.Context, any) error + + // Upsert inserts an object if it doesn't exist or updates it if it does. + // This operation requires a field with a unique directive in the dgraph tag. + // Note: This operation is not supported in file-based (local) mode. Upsert(context.Context, any) error + + // Update modifies an existing object in the database. + // The object must be a pointer to a struct and must have a UID field set. Update(context.Context, any) error + + // Get retrieves a single object by its UID and populates the provided object. + // The object parameter must be a pointer to a struct. Get(context.Context, any, string) error + + // Query creates a new query builder for retrieving data from the database. + // Returns a *dg.Query that can be further refined with filters, pagination, etc. Query(context.Context, any) *dg.Query + + // Delete removes objects with the specified UIDs from the database. Delete(context.Context, []string) error + + // Close releases all resources used by the client. + // It should be called when the client is no longer needed. Close() + // UpdateSchema ensures the database schema matches the provided object types. + // Pass one or more objects that will be used as templates for the schema. UpdateSchema(context.Context, ...any) error + + // GetSchema retrieves the current schema definition from the database. + // Returns a string containing the full schema in Dgraph Schema Definition Language. GetSchema(context.Context) (string, error) + + // DropAll removes the schema and all data from the database. DropAll(context.Context) error + + // DropData removes all data from the database but keeps the schema intact. DropData(context.Context) error + // QueryRaw executes a raw Dgraph query with optional query variables. // The `query` parameter is the Dgraph query string. // The `vars` parameter is a map of variable names to their values, used to parameterize the query. QueryRaw(context.Context, string, map[string]string) ([]byte, error) + // DgraphClient returns a gRPC Dgraph client from the connection pool and a cleanup function. + // The cleanup function must be called when finished with the client to return it to the pool. DgraphClient() (*dgo.Dgraph, func(), error) } @@ -126,6 +158,11 @@ func NewClient(uri string, opts ...ClientOpt) (Client, error) { opt(&options) } + // TODO: implement namespace support for v25 + if options.namespace != "" { + options.logger.Info("Warning, namespace is set, but it is not supported in this version") + } + client := client{ uri: uri, options: options, @@ -378,7 +415,13 @@ func (c client) QueryRaw(ctx context.Context, q string, vars map[string]string) // Close releases resources used by the client. func (c client) Close() { - c.pool.close() + // Add nil check to prevent panic if pool is nil + if c.pool != nil { + c.pool.close() + } + if c.engine != nil { + c.engine.Close() + } } // DgraphClient returns a Dgraph client from the pool and a cleanup function to put it back. diff --git a/client_test.go b/client_test.go index 93ea47c..1a5504a 100644 --- a/client_test.go +++ b/client_test.go @@ -208,3 +208,38 @@ func TestClientPoolStress(t *testing.T) { mg.Shutdown() } } + +func TestClientPoolMisuse(t *testing.T) { + testCases := []struct { + name string + uri string + skip bool + }{ + { + name: "ClientPoolMisuseWithFileURI", + uri: "file://" + GetTempDir(t), + }, + { + name: "ClientPoolMisuseWithDgraphURI", + uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), + skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.skip { + t.Skip("Skipping test as MODUSGRAPH_TEST_ADDR is not set") + } + + // Create a client with pool size 10 + client, err := mg.NewClient(tc.uri, mg.WithPoolSize(10)) + require.NoError(t, err) + client.Close() + client.Close() + }) + } + + // Shutdown at the end of the test to ensure the next test can start fresh + mg.Shutdown() +} diff --git a/query_test.go b/query_test.go index 660f404..09e6296 100644 --- a/query_test.go +++ b/query_test.go @@ -327,3 +327,107 @@ func TestVectorSimilaritySearch(t *testing.T) { }) } } + +type Student struct { + Name string `json:"name,omitempty" dgraph:"index=exact"` + + // The reverse directive tells Dgraph to maintain a reverse edge + Takes_Class []*Class `json:"takes_class,omitempty" dgraph:"reverse"` + + UID string `json:"uid,omitempty"` + DType []string `json:"dgraph.type,omitempty"` +} + +type Class struct { + Name string `json:"name,omitempty" dgraph:"index=exact"` + + // The tilde prefix tells modusGraph not to manage this field in the schema, + // but we still need it in the struct in order for results to scan correctly + Students []*Student `json:"~takes_class,omitempty"` + + UID string `json:"uid,omitempty"` + DType []string `json:"dgraph.type,omitempty"` +} + +func TestReverseEdgeQuery(t *testing.T) { + testCases := []struct { + name string + uri string + skip bool + }{ + { + name: "ReverseEdgeQueryWithFileURI", + uri: "file://" + GetTempDir(t), + }, + { + name: "ReverseEdgeQueryWithDgraphURI", + uri: "dgraph://" + os.Getenv("MODUSGRAPH_TEST_ADDR"), + skip: os.Getenv("MODUSGRAPH_TEST_ADDR") == "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + if tc.skip { + t.Skipf("Skipping %s: MODUSGRAPH_TEST_ADDR not set", tc.name) + return + } + + ctx := context.Background() + client, cleanup := CreateTestClient(t, tc.uri) + defer cleanup() + + class := Class{ + Name: "Math", + } + err := client.Insert(ctx, &class) + require.NoError(t, err) + require.Equal(t, "Math", class.Name) + + student := Student{ + Name: "Bob", + Takes_Class: []*Class{&class}, + } + err = client.Insert(ctx, &student) + require.NoError(t, err) + require.NotEmpty(t, student.UID) + + student = Student{ + Name: "Alice", + Takes_Class: []*Class{&class}, + } + err = client.Insert(ctx, &student) + require.NoError(t, err) + require.NotEmpty(t, student.UID) + + schema, err := client.GetSchema(ctx) + require.NoError(t, err) + require.Contains(t, schema, "type Student") + require.Contains(t, schema, "type Class") + + // We cannot use the 'contructor' style querying because graph + // querying uses the `expand(_all_)` operator, which does not + // support reverse edges. + var result []Class + query := dg.NewQuery().Model(&result).Query(` + { + name + uid + ~takes_class { + name + uid + } + }`) + dgo, cleanup, err := client.DgraphClient() + require.NoError(t, err) + defer cleanup() + tx := dg.NewReadOnlyTxn(dgo) + err = tx.Query(query).Scan() + require.NoError(t, err) + + require.Len(t, result, 1, "Should have found 1 class") + require.Equal(t, "Math", result[0].Name, "Class name should match") + require.Len(t, result[0].Students, 2, "Should have found 2 students") + }) + } +}