diff --git a/README.md b/README.md index fb1f965..e1b570a 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ The module structure exactly mirrors the Eclipse Store Java repository for famil - ✅ **`afs/blobstore`** - Complete Abstract File System blob storage backend - ✅ **`afs/aws/s3`** - Complete AWS S3 storage backend - ✅ **`afs/azure/storage`** - Complete Azure Storage backend -- 🚧 **`afs/googlecloud/firestore`** - Google Cloud Firestore integration (in progress) +- ✅ **`afs/googlecloud/firestore`** - Complete Google Cloud Firestore integration - ✅ **`gigamap/gigamap`** - Complete high-performance indexed collections with: - ✅ **Advanced indexing system** (bitmap, hash, unique indices) - ✅ **Full LINQ support** for querying (Eclipse Store compatible) @@ -125,7 +125,7 @@ The .NET implementation maintains the same module structure, interfaces, and des - ✅ **Blob storage** support for large object handling - ✅ **AWS S3** storage backend for cloud storage - ✅ **Azure Storage** backend for Microsoft cloud -- 🚧 **Google Cloud Firestore** integration (in progress) +- ✅ **Google Cloud Firestore** integration ## Architecture diff --git a/afs/blobstore/src/AfsStorageConnection.cs b/afs/blobstore/src/AfsStorageConnection.cs index cc44f46..5971bf1 100644 --- a/afs/blobstore/src/AfsStorageConnection.cs +++ b/afs/blobstore/src/AfsStorageConnection.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Reflection; using System.Threading.Tasks; using MessagePack; using NebulaStore.Storage; @@ -212,10 +213,126 @@ private static IBlobStoreConnector CreateConnector(IEmbeddedStorageConfiguration "blobstore" => new LocalBlobStoreConnector( configuration.AfsConnectionString ?? configuration.StorageDirectory, configuration.AfsUseCache), + "firestore" => CreateFirestoreConnector(configuration), + "azure.storage" => CreateAzureStorageConnector(configuration), + "s3" => CreateS3Connector(configuration), _ => throw new NotSupportedException($"AFS storage type '{configuration.AfsStorageType}' is not supported") }; } + /// + /// Creates a Google Cloud Firestore connector. + /// + /// The storage configuration + /// The Firestore connector + private static IBlobStoreConnector CreateFirestoreConnector(IEmbeddedStorageConfiguration configuration) + { + try + { + // Use reflection to avoid hard dependency on Google Cloud Firestore + var firestoreAssembly = System.Reflection.Assembly.LoadFrom("NebulaStore.Afs.GoogleCloud.Firestore.dll"); + var connectorType = firestoreAssembly.GetType("NebulaStore.Afs.GoogleCloud.Firestore.GoogleCloudFirestoreConnector"); + + if (connectorType == null) + throw new TypeLoadException("GoogleCloudFirestoreConnector type not found"); + + // Create FirestoreDb instance + var firestoreDbType = Type.GetType("Google.Cloud.Firestore.FirestoreDb, Google.Cloud.Firestore"); + if (firestoreDbType == null) + throw new TypeLoadException("Google.Cloud.Firestore.FirestoreDb type not found. Make sure Google.Cloud.Firestore package is installed."); + + var createMethod = firestoreDbType.GetMethod("Create", new[] { typeof(string) }); + if (createMethod == null) + throw new MethodAccessException("FirestoreDb.Create method not found"); + + var projectId = configuration.AfsConnectionString ?? throw new ArgumentException("Project ID must be specified in AfsConnectionString for Firestore storage"); + var firestoreDb = createMethod.Invoke(null, new object[] { projectId }); + + // Create connector + var factoryMethod = configuration.AfsUseCache + ? connectorType.GetMethod("Caching", new[] { firestoreDbType }) + : connectorType.GetMethod("New", new[] { firestoreDbType }); + + if (factoryMethod == null) + throw new MethodAccessException($"GoogleCloudFirestoreConnector factory method not found"); + + var connector = factoryMethod.Invoke(null, new[] { firestoreDb }); + return (IBlobStoreConnector)connector!; + } + catch (Exception ex) when (!(ex is ArgumentException)) + { + throw new NotSupportedException( + "Google Cloud Firestore connector could not be created. " + + "Make sure NebulaStore.Afs.GoogleCloud.Firestore and Google.Cloud.Firestore packages are installed.", ex); + } + } + + /// + /// Creates an Azure Storage connector. + /// + /// The storage configuration + /// The Azure Storage connector + private static IBlobStoreConnector CreateAzureStorageConnector(IEmbeddedStorageConfiguration configuration) + { + try + { + // Use reflection to avoid hard dependency on Azure Storage + var azureAssembly = System.Reflection.Assembly.LoadFrom("NebulaStore.Afs.Azure.Storage.dll"); + var connectorType = azureAssembly.GetType("NebulaStore.Afs.Azure.Storage.AzureStorageConnector"); + + if (connectorType == null) + throw new TypeLoadException("AzureStorageConnector type not found"); + + var connectionString = configuration.AfsConnectionString ?? throw new ArgumentException("Connection string must be specified in AfsConnectionString for Azure storage"); + + var factoryMethod = connectorType.GetMethod("FromConnectionString", new[] { typeof(string), typeof(bool) }); + if (factoryMethod == null) + throw new MethodAccessException("AzureStorageConnector.FromConnectionString method not found"); + + var connector = factoryMethod.Invoke(null, new object[] { connectionString, configuration.AfsUseCache }); + return (IBlobStoreConnector)connector!; + } + catch (Exception ex) when (!(ex is ArgumentException)) + { + throw new NotSupportedException( + "Azure Storage connector could not be created. " + + "Make sure NebulaStore.Afs.Azure.Storage and Azure.Storage.Blobs packages are installed.", ex); + } + } + + /// + /// Creates an AWS S3 connector. + /// + /// The storage configuration + /// The S3 connector + private static IBlobStoreConnector CreateS3Connector(IEmbeddedStorageConfiguration configuration) + { + try + { + // Use reflection to avoid hard dependency on AWS S3 + var s3Assembly = System.Reflection.Assembly.LoadFrom("NebulaStore.Afs.Aws.S3.dll"); + var connectorType = s3Assembly.GetType("NebulaStore.Afs.Aws.S3.AwsS3Connector"); + + if (connectorType == null) + throw new TypeLoadException("AwsS3Connector type not found"); + + var bucketName = configuration.AfsConnectionString ?? throw new ArgumentException("Bucket name must be specified in AfsConnectionString for S3 storage"); + + var factoryMethod = connectorType.GetMethod("FromBucketName", new[] { typeof(string), typeof(bool) }); + if (factoryMethod == null) + throw new MethodAccessException("AwsS3Connector.FromBucketName method not found"); + + var connector = factoryMethod.Invoke(null, new object[] { bucketName, configuration.AfsUseCache }); + return (IBlobStoreConnector)connector!; + } + catch (Exception ex) when (!(ex is ArgumentException)) + { + throw new NotSupportedException( + "AWS S3 connector could not be created. " + + "Make sure NebulaStore.Afs.Aws.S3 and AWSSDK.S3 packages are installed.", ex); + } + } + /// /// Registers type handlers for GigaMap serialization. /// This enables automatic persistence of GigaMap instances following Eclipse Store patterns. diff --git a/afs/googlecloud/firestore/examples/FirestoreExample.cs b/afs/googlecloud/firestore/examples/FirestoreExample.cs new file mode 100644 index 0000000..ffd4de1 --- /dev/null +++ b/afs/googlecloud/firestore/examples/FirestoreExample.cs @@ -0,0 +1,189 @@ +using System; +using Google.Cloud.Firestore; +using NebulaStore.Afs.GoogleCloud.Firestore; +using NebulaStore.Storage.Embedded; +using NebulaStore.Storage.EmbeddedConfiguration; + +namespace NebulaStore.Afs.GoogleCloud.Firestore.Examples; + +/// +/// Example demonstrating how to use NebulaStore with Google Cloud Firestore. +/// +public class FirestoreExample +{ + /// + /// Example data class for storage. + /// + public class Person + { + public string Name { get; set; } = string.Empty; + public int Age { get; set; } + public string Email { get; set; } = string.Empty; + } + + /// + /// Example using the configuration-based approach. + /// + public static void ConfigurationBasedExample() + { + Console.WriteLine("=== Configuration-Based Firestore Example ==="); + + // Create configuration for Firestore storage + var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("firestore-storage") + .UseFirestore("your-project-id", useCache: true) + .SetChannelCount(4) + .Build(); + + try + { + // Start storage with Firestore backend + using var storage = EmbeddedStorage.Foundation(config).Start(); + + // Create and store data + var root = storage.Root(); + if (root == null) + { + root = new Person + { + Name = "John Doe", + Age = 30, + Email = "john.doe@example.com" + }; + storage.SetRoot(root); + } + + // Modify and store + root.Age = 31; + storage.StoreRoot(); + + Console.WriteLine($"Stored person: {root.Name}, Age: {root.Age}"); + } + catch (Exception ex) + { + Console.WriteLine($"Error: {ex.Message}"); + Console.WriteLine("Make sure you have:"); + Console.WriteLine("1. A Google Cloud Project with Firestore enabled"); + Console.WriteLine("2. Proper authentication configured"); + Console.WriteLine("3. The Google.Cloud.Firestore package installed"); + } + } + + /// + /// Example using the convenience extension methods. + /// + public static void ConvenienceMethodExample() + { + Console.WriteLine("\n=== Convenience Method Firestore Example ==="); + + try + { + // Start with Firestore using convenience method + using var storage = EmbeddedStorageFirestoreExtensions.StartWithFirestore("your-project-id"); + + var root = storage.Root(); + if (root == null) + { + root = new Person + { + Name = "Jane Smith", + Age = 25, + Email = "jane.smith@example.com" + }; + storage.SetRoot(root); + } + + Console.WriteLine($"Retrieved person: {root.Name}, Age: {root.Age}"); + } + catch (Exception ex) + { + Console.WriteLine($"Error: {ex.Message}"); + } + } + + /// + /// Example using direct AFS file system operations. + /// + public static void DirectAfsExample() + { + Console.WriteLine("\n=== Direct AFS Firestore Example ==="); + + try + { + // Create Firestore connection + var firestore = FirestoreDb.Create("your-project-id"); + + // Create file system with Firestore backend + using var fileSystem = EmbeddedStorageFirestoreExtensions.CreateFirestoreFileSystem("your-project-id"); + + // Perform direct file operations + var path = NebulaStore.Afs.Blobstore.BlobStorePath.New("my-collection", "folder", "file.txt"); + var data = System.Text.Encoding.UTF8.GetBytes("Hello, Firestore!"); + + fileSystem.IoHandler.WriteData(path, data); + var readData = fileSystem.IoHandler.ReadData(path, 0, -1); + var content = System.Text.Encoding.UTF8.GetString(readData); + + Console.WriteLine($"Stored and retrieved: {content}"); + } + catch (Exception ex) + { + Console.WriteLine($"Error: {ex.Message}"); + } + } + + /// + /// Example with authentication setup. + /// + public static void AuthenticationExample() + { + Console.WriteLine("\n=== Authentication Setup Example ==="); + + try + { + // Option 1: Using service account key file + var firestoreWithKey = new FirestoreDbBuilder + { + ProjectId = "your-project-id", + CredentialsPath = "path/to/service-account-key.json" + }.Build(); + + using var connector1 = GoogleCloudFirestoreConnector.New(firestoreWithKey); + Console.WriteLine("Created connector with service account key"); + + // Option 2: Using environment variable for credentials + Environment.SetEnvironmentVariable("GOOGLE_APPLICATION_CREDENTIALS", "path/to/service-account-key.json"); + var firestoreWithEnv = FirestoreDb.Create("your-project-id"); + + using var connector2 = GoogleCloudFirestoreConnector.Caching(firestoreWithEnv); + Console.WriteLine("Created connector with environment credentials"); + + // Option 3: Using emulator for testing + Environment.SetEnvironmentVariable("FIRESTORE_EMULATOR_HOST", "localhost:8080"); + var firestoreEmulator = FirestoreDb.Create("test-project"); + + using var connector3 = GoogleCloudFirestoreConnector.New(firestoreEmulator); + Console.WriteLine("Created connector for emulator"); + } + catch (Exception ex) + { + Console.WriteLine($"Error: {ex.Message}"); + } + } + + /// + /// Main entry point for the examples. + /// + public static void Main(string[] args) + { + Console.WriteLine("NebulaStore Google Cloud Firestore Integration Examples"); + Console.WriteLine("====================================================="); + + ConfigurationBasedExample(); + ConvenienceMethodExample(); + DirectAfsExample(); + AuthenticationExample(); + + Console.WriteLine("\nFor more information, see the README.md file."); + } +} diff --git a/afs/tests/FirestoreIntegrationTest.cs b/afs/tests/FirestoreIntegrationTest.cs new file mode 100644 index 0000000..5adea80 --- /dev/null +++ b/afs/tests/FirestoreIntegrationTest.cs @@ -0,0 +1,118 @@ +using System; +using FluentAssertions; +using NebulaStore.Storage.EmbeddedConfiguration; +using Xunit; + +namespace NebulaStore.Afs.Tests; + +/// +/// Integration test to verify that Firestore storage type is properly registered in the AFS system. +/// This test doesn't require actual Firestore connectivity - it just tests the registration. +/// +public class FirestoreIntegrationTest +{ + [Fact] + public void AfsStorageConnection_WithFirestoreType_ShouldThrowNotSupportedExceptionWithoutFirestorePackage() + { + // Arrange + var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("test-storage") + .SetUseAfs(true) + .SetAfsStorageType("firestore") + .SetAfsConnectionString("test-project-id") + .Build(); + + // Act & Assert + // Since we don't have the actual Firestore package loaded in this test project, + // it should throw a NotSupportedException when trying to create the connector + var action = () => new NebulaStore.Afs.Blobstore.AfsStorageConnection( + config, + new NebulaStore.Storage.TypeHandlerRegistry()); + + action.Should().Throw() + .WithMessage("*Google Cloud Firestore connector could not be created*"); + } + + [Fact] + public void AfsStorageConnection_WithAzureStorageType_ShouldThrowNotSupportedExceptionWithoutAzurePackage() + { + // Arrange + var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("test-storage") + .SetUseAfs(true) + .SetAfsStorageType("azure.storage") + .SetAfsConnectionString("DefaultEndpointsProtocol=https;AccountName=test;AccountKey=test;EndpointSuffix=core.windows.net") + .Build(); + + // Act & Assert + // Since we don't have the actual Azure Storage package loaded in this test project, + // it should throw a NotSupportedException when trying to create the connector + var action = () => new NebulaStore.Afs.Blobstore.AfsStorageConnection( + config, + new NebulaStore.Storage.TypeHandlerRegistry()); + + action.Should().Throw() + .WithMessage("*Azure Storage connector could not be created*"); + } + + [Fact] + public void AfsStorageConnection_WithS3Type_ShouldThrowNotSupportedExceptionWithoutS3Package() + { + // Arrange + var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("test-storage") + .SetUseAfs(true) + .SetAfsStorageType("s3") + .SetAfsConnectionString("test-bucket") + .Build(); + + // Act & Assert + // Since we don't have the actual S3 package loaded in this test project, + // it should throw a NotSupportedException when trying to create the connector + var action = () => new NebulaStore.Afs.Blobstore.AfsStorageConnection( + config, + new NebulaStore.Storage.TypeHandlerRegistry()); + + action.Should().Throw() + .WithMessage("*AWS S3 connector could not be created*"); + } + + [Fact] + public void AfsStorageConnection_WithBlobstoreType_ShouldCreateSuccessfully() + { + // Arrange + var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("test-storage") + .SetUseAfs(true) + .SetAfsStorageType("blobstore") + .Build(); + + // Act & Assert + // This should work since blobstore is the default local implementation + using var connection = new NebulaStore.Afs.Blobstore.AfsStorageConnection( + config, + new NebulaStore.Storage.TypeHandlerRegistry()); + + connection.Should().NotBeNull(); + connection.IsActive.Should().BeTrue(); + } + + [Fact] + public void AfsStorageConnection_WithUnsupportedType_ShouldThrowNotSupportedException() + { + // Arrange + var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("test-storage") + .SetUseAfs(true) + .SetAfsStorageType("unsupported-type") + .Build(); + + // Act & Assert + var action = () => new NebulaStore.Afs.Blobstore.AfsStorageConnection( + config, + new NebulaStore.Storage.TypeHandlerRegistry()); + + action.Should().Throw() + .WithMessage("AFS storage type 'unsupported-type' is not supported"); + } +}