From a06d9051ac7d8f19ec77de3241f96870ea1a0130 Mon Sep 17 00:00:00 2001 From: prpeh Date: Thu, 2 Oct 2025 22:59:59 +0700 Subject: [PATCH] Add Redis AFS connector module - Implement RedisConnector using StackExchange.Redis - Port Eclipse Store Redis AFS module to .NET Core - Add BlobMetadata class for blob key/size tracking - Add RedisConfiguration with fluent API - Integrate with AfsStorageConnection factory - Add comprehensive tests for connector and configuration - Add README with usage examples and documentation - Add RedisExample.cs with multiple usage scenarios - Update .gitignore to exclude .DS_Store files The Redis connector provides: - Thread-safe operations with optional caching - Full CRUD operations for files and directories - Configurable connection, timeouts, and authentication - Compatible with NebulaStore embedded storage system - Follows Eclipse Store AFS patterns --- .gitignore | 7 +- afs/blobstore/src/AfsStorageConnection.cs | 38 + afs/redis/NebulaStore.Afs.Redis.csproj | 36 + afs/redis/README.md | 262 +++++++ afs/redis/src/BlobMetadata.cs | 56 ++ afs/redis/src/RedisConfiguration.cs | 215 ++++++ afs/redis/src/RedisConnector.cs | 667 ++++++++++++++++++ .../test/NebulaStore.Afs.Redis.Tests.csproj | 32 + afs/redis/test/RedisConfigurationTests.cs | 223 ++++++ afs/redis/test/RedisConnectorTests.cs | 312 ++++++++ examples/RedisExample.cs | 281 ++++++++ 11 files changed, 2128 insertions(+), 1 deletion(-) create mode 100644 afs/redis/NebulaStore.Afs.Redis.csproj create mode 100644 afs/redis/README.md create mode 100644 afs/redis/src/BlobMetadata.cs create mode 100644 afs/redis/src/RedisConfiguration.cs create mode 100644 afs/redis/src/RedisConnector.cs create mode 100644 afs/redis/test/NebulaStore.Afs.Redis.Tests.csproj create mode 100644 afs/redis/test/RedisConfigurationTests.cs create mode 100644 afs/redis/test/RedisConnectorTests.cs create mode 100644 examples/RedisExample.cs diff --git a/.gitignore b/.gitignore index 43f9aee..24cf88a 100644 --- a/.gitignore +++ b/.gitignore @@ -62,4 +62,9 @@ TestResults/ *.msgpack *-storage/ storage/data.msgpack -storage/root.msgpack \ No newline at end of file +storage/root.msgpack + +# macOS +.DS_Store +.AppleDouble +.LSOverride \ No newline at end of file diff --git a/afs/blobstore/src/AfsStorageConnection.cs b/afs/blobstore/src/AfsStorageConnection.cs index 5971bf1..2439ffd 100644 --- a/afs/blobstore/src/AfsStorageConnection.cs +++ b/afs/blobstore/src/AfsStorageConnection.cs @@ -216,6 +216,7 @@ private static IBlobStoreConnector CreateConnector(IEmbeddedStorageConfiguration "firestore" => CreateFirestoreConnector(configuration), "azure.storage" => CreateAzureStorageConnector(configuration), "s3" => CreateS3Connector(configuration), + "redis" => CreateRedisConnector(configuration), _ => throw new NotSupportedException($"AFS storage type '{configuration.AfsStorageType}' is not supported") }; } @@ -333,6 +334,43 @@ private static IBlobStoreConnector CreateS3Connector(IEmbeddedStorageConfigurati } } + /// + /// Creates a Redis connector. + /// + /// The storage configuration + /// The Redis connector + private static IBlobStoreConnector CreateRedisConnector(IEmbeddedStorageConfiguration configuration) + { + try + { + // Use reflection to avoid hard dependency on Redis + var redisAssembly = System.Reflection.Assembly.LoadFrom("NebulaStore.Afs.Redis.dll"); + var connectorType = redisAssembly.GetType("NebulaStore.Afs.Redis.RedisConnector"); + + if (connectorType == null) + throw new TypeLoadException("RedisConnector type not found"); + + var connectionString = configuration.AfsConnectionString ?? "localhost:6379"; + + // Use the factory method that takes connection string and database number + var factoryMethod = configuration.AfsUseCache + ? connectorType.GetMethod("Caching", new[] { typeof(string), typeof(int) }) + : connectorType.GetMethod("New", new[] { typeof(string), typeof(int) }); + + if (factoryMethod == null) + throw new MethodAccessException("RedisConnector factory method not found"); + + var connector = factoryMethod.Invoke(null, new object[] { connectionString, 0 }); + return (IBlobStoreConnector)connector!; + } + catch (Exception ex) when (!(ex is ArgumentException)) + { + throw new NotSupportedException( + "Redis connector could not be created. " + + "Make sure NebulaStore.Afs.Redis and StackExchange.Redis packages are installed.", ex); + } + } + /// /// Registers type handlers for GigaMap serialization. /// This enables automatic persistence of GigaMap instances following Eclipse Store patterns. diff --git a/afs/redis/NebulaStore.Afs.Redis.csproj b/afs/redis/NebulaStore.Afs.Redis.csproj new file mode 100644 index 0000000..8ab354d --- /dev/null +++ b/afs/redis/NebulaStore.Afs.Redis.csproj @@ -0,0 +1,36 @@ + + + + net9.0 + latest + enable + enable + true + NebulaStore.Afs.Redis + 1.0.0 + NebulaStore Contributors + Redis connector for NebulaStore Abstract File System (AFS) + NebulaStore;AFS;Redis;Storage + MIT + https://github.com/hadv/NebulaStore + git + false + false + + + + + + + + + + + + + + + + + + diff --git a/afs/redis/README.md b/afs/redis/README.md new file mode 100644 index 0000000..d974d44 --- /dev/null +++ b/afs/redis/README.md @@ -0,0 +1,262 @@ +# NebulaStore AFS Redis Connector + +Redis connector for NebulaStore Abstract File System (AFS). This module provides a Redis-based storage backend for NebulaStore, allowing you to store object graphs in Redis using the StackExchange.Redis client. + +## Overview + +The Redis AFS connector stores files as numbered blob entries in Redis. Each blob is stored as a separate Redis key with binary data as the value. This implementation follows the Eclipse Store AFS pattern and is compatible with the NebulaStore storage system. + +## Features + +- **Redis Storage**: Store NebulaStore data in Redis key-value store +- **Caching Support**: Optional in-memory caching for improved performance +- **Configurable**: Flexible configuration options for connection, timeouts, and database selection +- **Thread-Safe**: All operations are thread-safe +- **Compatible**: Works seamlessly with NebulaStore's embedded storage system + +## Installation + +```bash +dotnet add package NebulaStore.Afs.Redis +dotnet add package StackExchange.Redis +``` + +## Quick Start + +### Using with EmbeddedStorage + +```csharp +using NebulaStore.Storage.Embedded; +using NebulaStore.Storage.EmbeddedConfiguration; + +// Configure storage to use Redis +var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("storage") + .SetAfsStorageType("redis") + .SetAfsConnectionString("localhost:6379") + .SetAfsUseCache(true); + +// Start storage with Redis backend +using var storage = EmbeddedStorage.StartWithAfs(config); + +// Use storage normally +var root = storage.Root(); +root.SomeProperty = "value"; +storage.StoreRoot(); +``` + +### Direct AFS Usage + +```csharp +using StackExchange.Redis; +using NebulaStore.Afs.Redis; +using NebulaStore.Afs.Blobstore; + +// Create Redis connection +var redis = ConnectionMultiplexer.Connect("localhost:6379"); + +// Create connector +using var connector = RedisConnector.New(redis); + +// Create file system +using var fileSystem = BlobStoreFileSystem.New(connector); + +// Perform operations +var path = BlobStorePath.New("my-container", "folder", "file.dat"); +var data = System.Text.Encoding.UTF8.GetBytes("Hello, Redis!"); + +fileSystem.IoHandler.WriteData(path, data); +var readData = fileSystem.IoHandler.ReadData(path, 0, -1); +``` + +### With Caching + +```csharp +// Create connector with caching enabled +using var connector = RedisConnector.Caching("localhost:6379"); +using var fileSystem = BlobStoreFileSystem.New(connector); +``` + +### Using RedisConfiguration + +```csharp +using StackExchange.Redis; +using NebulaStore.Afs.Redis; + +// Create configuration +var config = RedisConfiguration.New() + .SetConnectionString("localhost:6379") + .SetDatabaseNumber(0) + .SetUseCache(true) + .SetCommandTimeout(TimeSpan.FromMinutes(1)) + .SetPassword("your-password") // Optional + .SetUseSsl(false); // Optional + +// Build StackExchange.Redis options +var options = config.ToConfigurationOptions(); +var redis = ConnectionMultiplexer.Connect(options); + +// Create connector +using var connector = RedisConnector.New(redis); +``` + +## Configuration Options + +### RedisConfiguration Properties + +- **ConnectionString**: Redis connection string (default: "localhost:6379") +- **DatabaseNumber**: Redis database number (default: 0) +- **UseCache**: Enable in-memory caching (default: true) +- **CommandTimeout**: Command execution timeout (default: 1 minute) +- **ConnectTimeout**: Connection timeout (default: 5 seconds) +- **SyncTimeout**: Synchronous operation timeout (default: 5 seconds) +- **AllowAdmin**: Allow admin operations (default: true) +- **AbortOnConnectFail**: Abort on connection failure (default: false) +- **Password**: Redis authentication password (optional) +- **UseSsl**: Enable SSL/TLS (default: false) + +### AFS Configuration + +When using with EmbeddedStorage, configure via `IEmbeddedStorageConfiguration`: + +```csharp +var config = EmbeddedStorageConfiguration.New() + .SetAfsStorageType("redis") + .SetAfsConnectionString("localhost:6379") + .SetAfsUseCache(true); +``` + +## Architecture + +### Storage Structure + +Files are stored in Redis using a hierarchical key structure: + +``` +container/path/to/file.0 +container/path/to/file.1 +container/path/to/file.2 +``` + +Each file is split into numbered blobs (suffixed with `.0`, `.1`, `.2`, etc.). This allows for efficient storage and retrieval of large files. + +### Key Features + +1. **Virtual Directories**: Directories are virtual in Redis - they don't need to be explicitly created +2. **Blob Numbering**: Files are split into numbered blobs for efficient storage +3. **Atomic Operations**: Uses Redis atomic operations for data consistency +4. **Caching**: Optional in-memory caching reduces Redis queries + +## Performance Considerations + +- **Caching**: Enable caching for read-heavy workloads +- **Connection Pooling**: StackExchange.Redis handles connection pooling automatically +- **Database Selection**: Use different database numbers to isolate data +- **Timeouts**: Adjust timeouts based on your network latency and data size + +## Requirements + +- .NET 9.0 or later +- StackExchange.Redis 2.8.16 or later +- Redis server 5.0 or later (recommended) + +## Thread Safety + +All operations in the RedisConnector are thread-safe. The connector uses: +- Lock-based synchronization for cache operations +- StackExchange.Redis's built-in thread-safe operations +- Atomic Redis commands where applicable + +## Error Handling + +The connector handles common Redis errors: +- Connection failures +- Timeout errors +- Key not found scenarios +- Data serialization issues + +## Limitations + +- Redis key size limits apply (512 MB per key) +- Memory constraints of Redis server +- Network latency affects performance +- Requires Redis server to be running and accessible + +## Examples + +### Complete Example with Custom Data + +```csharp +using StackExchange.Redis; +using NebulaStore.Afs.Redis; +using NebulaStore.Afs.Blobstore; +using NebulaStore.Storage.Embedded; + +// Define your data model +public class Customer +{ + public int Id { get; set; } + public string Name { get; set; } = ""; + public List Orders { get; set; } = new(); +} + +public class Order +{ + public int OrderId { get; set; } + public decimal Amount { get; set; } +} + +// Configure and use Redis storage +var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("redis-storage") + .SetAfsStorageType("redis") + .SetAfsConnectionString("localhost:6379") + .SetAfsUseCache(true); + +using var storage = EmbeddedStorage.StartWithAfs(config); + +// Create and store data +var root = storage.Root(); +root.Id = 1; +root.Name = "John Doe"; +root.Orders.Add(new Order { OrderId = 100, Amount = 99.99m }); + +storage.StoreRoot(); +``` + +## Troubleshooting + +### Connection Issues + +If you encounter connection issues: +1. Verify Redis server is running: `redis-cli ping` +2. Check connection string format +3. Verify network connectivity +4. Check firewall settings + +### Performance Issues + +For performance optimization: +1. Enable caching for read-heavy workloads +2. Adjust command timeouts +3. Use connection multiplexing +4. Monitor Redis memory usage + +## License + +This project is licensed under the MIT License. + +## Contributing + +Contributions are welcome! Please ensure: +- Code follows existing patterns +- Tests are included +- Documentation is updated + +## Related + +- [NebulaStore](../../README.md) +- [AFS Overview](../README.md) +- [Eclipse Store](https://github.com/eclipse-store/store) +- [StackExchange.Redis](https://github.com/StackExchange/StackExchange.Redis) + diff --git a/afs/redis/src/BlobMetadata.cs b/afs/redis/src/BlobMetadata.cs new file mode 100644 index 0000000..f2faddb --- /dev/null +++ b/afs/redis/src/BlobMetadata.cs @@ -0,0 +1,56 @@ +namespace NebulaStore.Afs.Redis; + +/// +/// Metadata for a blob stored in Redis. +/// Contains the key and size information for a blob. +/// +public sealed class BlobMetadata +{ + /// + /// Gets the Redis key for this blob. + /// + public string Key { get; } + + /// + /// Gets the size of the blob in bytes. + /// + public long Size { get; } + + /// + /// Initializes a new instance of the BlobMetadata class. + /// + /// The Redis key + /// The blob size in bytes + /// Thrown if key is null or empty + /// Thrown if size is negative + private BlobMetadata(string key, long size) + { + if (string.IsNullOrEmpty(key)) + throw new ArgumentException("Key cannot be null or empty", nameof(key)); + if (size < 0) + throw new ArgumentOutOfRangeException(nameof(size), "Size cannot be negative"); + + Key = key; + Size = size; + } + + /// + /// Creates a new BlobMetadata instance. + /// + /// The Redis key + /// The blob size in bytes + /// A new BlobMetadata instance + public static BlobMetadata New(string key, long size) + { + return new BlobMetadata(key, size); + } + + /// + /// Returns a string representation of this blob metadata. + /// + public override string ToString() + { + return $"BlobMetadata(Key={Key}, Size={Size})"; + } +} + diff --git a/afs/redis/src/RedisConfiguration.cs b/afs/redis/src/RedisConfiguration.cs new file mode 100644 index 0000000..fd52a6e --- /dev/null +++ b/afs/redis/src/RedisConfiguration.cs @@ -0,0 +1,215 @@ +namespace NebulaStore.Afs.Redis; + +/// +/// Configuration for Redis AFS connector. +/// +public class RedisConfiguration +{ + /// + /// Gets or sets the Redis connection string. + /// + public string ConnectionString { get; private set; } = "localhost:6379"; + + /// + /// Gets or sets the Redis database number. + /// + public int DatabaseNumber { get; private set; } = 0; + + /// + /// Gets or sets whether to use caching. + /// + public bool UseCache { get; private set; } = true; + + /// + /// Gets or sets the command timeout. + /// + public TimeSpan CommandTimeout { get; private set; } = TimeSpan.FromMinutes(1); + + /// + /// Gets or sets the connect timeout. + /// + public TimeSpan ConnectTimeout { get; private set; } = TimeSpan.FromSeconds(5); + + /// + /// Gets or sets the sync timeout. + /// + public TimeSpan SyncTimeout { get; private set; } = TimeSpan.FromSeconds(5); + + /// + /// Gets or sets whether to allow admin operations. + /// + public bool AllowAdmin { get; private set; } = true; + + /// + /// Gets or sets whether to abort on connect fail. + /// + public bool AbortOnConnectFail { get; private set; } = false; + + /// + /// Gets or sets the password for Redis authentication. + /// + public string? Password { get; private set; } + + /// + /// Gets or sets the SSL/TLS settings. + /// + public bool UseSsl { get; private set; } = false; + + /// + /// Creates a new RedisConfiguration instance. + /// + public static RedisConfiguration New() + { + return new RedisConfiguration(); + } + + /// + /// Sets the connection string. + /// + public RedisConfiguration SetConnectionString(string connectionString) + { + if (string.IsNullOrWhiteSpace(connectionString)) + throw new ArgumentException("Connection string cannot be null or empty", nameof(connectionString)); + + ConnectionString = connectionString; + return this; + } + + /// + /// Sets the database number. + /// + public RedisConfiguration SetDatabaseNumber(int databaseNumber) + { + if (databaseNumber < 0) + throw new ArgumentOutOfRangeException(nameof(databaseNumber), "Database number must be non-negative"); + + DatabaseNumber = databaseNumber; + return this; + } + + /// + /// Sets whether to use caching. + /// + public RedisConfiguration SetUseCache(bool useCache) + { + UseCache = useCache; + return this; + } + + /// + /// Sets the command timeout. + /// + public RedisConfiguration SetCommandTimeout(TimeSpan timeout) + { + if (timeout <= TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(timeout), "Timeout must be positive"); + + CommandTimeout = timeout; + return this; + } + + /// + /// Sets the connect timeout. + /// + public RedisConfiguration SetConnectTimeout(TimeSpan timeout) + { + if (timeout <= TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(timeout), "Timeout must be positive"); + + ConnectTimeout = timeout; + return this; + } + + /// + /// Sets the sync timeout. + /// + public RedisConfiguration SetSyncTimeout(TimeSpan timeout) + { + if (timeout <= TimeSpan.Zero) + throw new ArgumentOutOfRangeException(nameof(timeout), "Timeout must be positive"); + + SyncTimeout = timeout; + return this; + } + + /// + /// Sets whether to allow admin operations. + /// + public RedisConfiguration SetAllowAdmin(bool allowAdmin) + { + AllowAdmin = allowAdmin; + return this; + } + + /// + /// Sets whether to abort on connect fail. + /// + public RedisConfiguration SetAbortOnConnectFail(bool abortOnConnectFail) + { + AbortOnConnectFail = abortOnConnectFail; + return this; + } + + /// + /// Sets the password for Redis authentication. + /// + public RedisConfiguration SetPassword(string? password) + { + Password = password; + return this; + } + + /// + /// Sets whether to use SSL/TLS. + /// + public RedisConfiguration SetUseSsl(bool useSsl) + { + UseSsl = useSsl; + return this; + } + + /// + /// Validates the configuration. + /// + public void Validate() + { + if (string.IsNullOrWhiteSpace(ConnectionString)) + throw new InvalidOperationException("Connection string must be set"); + + if (DatabaseNumber < 0) + throw new InvalidOperationException("Database number must be non-negative"); + + if (CommandTimeout <= TimeSpan.Zero) + throw new InvalidOperationException("Command timeout must be positive"); + + if (ConnectTimeout <= TimeSpan.Zero) + throw new InvalidOperationException("Connect timeout must be positive"); + + if (SyncTimeout <= TimeSpan.Zero) + throw new InvalidOperationException("Sync timeout must be positive"); + } + + /// + /// Builds a StackExchange.Redis ConfigurationOptions from this configuration. + /// + public StackExchange.Redis.ConfigurationOptions ToConfigurationOptions() + { + Validate(); + + var options = StackExchange.Redis.ConfigurationOptions.Parse(ConnectionString); + options.DefaultDatabase = DatabaseNumber; + options.ConnectTimeout = (int)ConnectTimeout.TotalMilliseconds; + options.SyncTimeout = (int)SyncTimeout.TotalMilliseconds; + options.AllowAdmin = AllowAdmin; + options.AbortOnConnectFail = AbortOnConnectFail; + options.Ssl = UseSsl; + + if (!string.IsNullOrWhiteSpace(Password)) + { + options.Password = Password; + } + + return options; + } +} + diff --git a/afs/redis/src/RedisConnector.cs b/afs/redis/src/RedisConnector.cs new file mode 100644 index 0000000..8a77db2 --- /dev/null +++ b/afs/redis/src/RedisConnector.cs @@ -0,0 +1,667 @@ +using System.Text.RegularExpressions; +using StackExchange.Redis; +using NebulaStore.Afs.Blobstore; +using NebulaStore.Afs.Blobstore.Types; + +namespace NebulaStore.Afs.Redis; + +/// +/// Redis implementation of IBlobStoreConnector. +/// Stores blobs as key-value pairs in Redis using the StackExchange.Redis client. +/// +/// +/// This connector stores files as numbered blob entries in Redis. +/// Each blob is stored as a separate Redis key with binary data as the value. +/// +/// First create a Redis connection: +/// +/// var redis = ConnectionMultiplexer.Connect("localhost:6379"); +/// var connector = RedisConnector.New(redis); +/// var fileSystem = BlobStoreFileSystem.New(connector); +/// +/// +public class RedisConnector : BlobStoreConnectorBase +{ + private readonly IConnectionMultiplexer _redis; + private readonly IDatabase _database; + private readonly bool _useCache; + private readonly Dictionary _directoryExistsCache = new(); + private readonly Dictionary _fileExistsCache = new(); + private readonly Dictionary _fileSizeCache = new(); + private readonly object _cacheLock = new(); + private readonly TimeSpan _commandTimeout; + + /// + /// Initializes a new instance of the RedisConnector class. + /// + /// The Redis connection multiplexer + /// Whether to enable caching + /// The Redis database number (default: 0) + /// Command timeout (default: 1 minute) + private RedisConnector( + IConnectionMultiplexer redis, + bool useCache = false, + int databaseNumber = 0, + TimeSpan? commandTimeout = null) + { + _redis = redis ?? throw new ArgumentNullException(nameof(redis)); + _database = _redis.GetDatabase(databaseNumber); + _useCache = useCache; + _commandTimeout = commandTimeout ?? TimeSpan.FromMinutes(1); + } + + /// + /// Creates a new Redis connector. + /// + /// The Redis connection multiplexer + /// The Redis database number (default: 0) + /// A new Redis connector + public static RedisConnector New(IConnectionMultiplexer redis, int databaseNumber = 0) + { + return new RedisConnector(redis, useCache: false, databaseNumber: databaseNumber); + } + + /// + /// Creates a new Redis connector from a connection string. + /// + /// The Redis connection string + /// The Redis database number (default: 0) + /// A new Redis connector + public static RedisConnector New(string connectionString, int databaseNumber = 0) + { + var redis = ConnectionMultiplexer.Connect(connectionString); + return new RedisConnector(redis, useCache: false, databaseNumber: databaseNumber); + } + + /// + /// Creates a new Redis connector with caching enabled. + /// + /// The Redis connection multiplexer + /// The Redis database number (default: 0) + /// A new Redis connector with caching + public static RedisConnector Caching(IConnectionMultiplexer redis, int databaseNumber = 0) + { + return new RedisConnector(redis, useCache: true, databaseNumber: databaseNumber); + } + + /// + /// Creates a new Redis connector with caching from a connection string. + /// + /// The Redis connection string + /// The Redis database number (default: 0) + /// A new Redis connector with caching + public static RedisConnector Caching(string connectionString, int databaseNumber = 0) + { + var redis = ConnectionMultiplexer.Connect(connectionString); + return new RedisConnector(redis, useCache: true, databaseNumber: databaseNumber); + } + + /// + /// Gets all blobs for a file. + /// + private List GetBlobs(BlobStorePath file) + { + var prefix = ToBlobKeyPrefixWithContainer(file); + var pattern = BlobKeyRegex(prefix); + var regex = new Regex(pattern); + + var server = _redis.GetServer(_redis.GetEndPoints().First()); + var keys = server.Keys(pattern: prefix + "*"); + + var blobs = new List(); + foreach (var key in keys) + { + var keyStr = key.ToString(); + if (regex.IsMatch(keyStr)) + { + var length = _database.StringLength(key); + blobs.Add(BlobMetadata.New(keyStr, length)); + } + } + + // Sort by blob number + blobs.Sort((a, b) => ExtractBlobNumber(a.Key).CompareTo(ExtractBlobNumber(b.Key))); + return blobs; + } + + /// + /// Gets child keys for a directory. + /// + private List GetChildKeys(BlobStorePath directory) + { + var prefix = ToChildKeysPrefixWithContainer(directory); + var pattern = ChildKeysRegexWithContainer(directory); + var regex = new Regex(pattern); + + var server = _redis.GetServer(_redis.GetEndPoints().First()); + var keys = server.Keys(pattern: prefix + "*"); + + var childKeys = new List(); + foreach (var key in keys) + { + var keyStr = key.ToString(); + if (regex.IsMatch(keyStr)) + { + childKeys.Add(keyStr); + } + } + + return childKeys; + } + + /// + /// Extracts the blob number from a blob key. + /// + private static long ExtractBlobNumber(string key) + { + var lastDot = key.LastIndexOf(NumberSuffixSeparatorChar); + if (lastDot >= 0 && lastDot < key.Length - 1) + { + if (long.TryParse(key.Substring(lastDot + 1), out var number)) + { + return number; + } + } + return 0; + } + + /// + /// Gets the next blob number for a file. + /// + private long GetNextBlobNumber(BlobStorePath file) + { + var blobs = GetBlobs(file); + if (blobs.Count == 0) + { + return 0; + } + + var maxNumber = blobs.Max(b => ExtractBlobNumber(b.Key)); + return maxNumber + 1; + } + + /// + /// Creates a blob key prefix with container. + /// + private static string ToBlobKeyPrefixWithContainer(BlobStorePath file) + { + return string.Join(BlobStorePath.Separator, file.PathElements) + NumberSuffixSeparator; + } + + /// + /// Creates a blob key with container. + /// + private static string ToBlobKeyWithContainer(BlobStorePath file, long number) + { + return ToBlobKeyPrefixWithContainer(file) + number.ToString(); + } + + /// + /// Creates a regex pattern for blob keys. + /// + private static string BlobKeyRegex(string prefix) + { + return "^" + Regex.Escape(prefix) + "[0-9]+$"; + } + + /// + /// Creates a child keys prefix with container. + /// + private static string ToChildKeysPrefixWithContainer(BlobStorePath directory) + { + return string.Join(BlobStorePath.Separator, directory.PathElements) + BlobStorePath.Separator; + } + + /// + /// Creates a regex pattern for child keys. + /// + private static string ChildKeysRegexWithContainer(BlobStorePath directory) + { + var prefix = ToChildKeysPrefixWithContainer(directory); + return "^" + Regex.Escape(prefix) + "[^" + Regex.Escape(BlobStorePath.Separator) + "]+"; + } + + /// + /// Invalidates cache entries for a file. + /// + private void InvalidateCache(string key) + { + if (_useCache) + { + lock (_cacheLock) + { + _fileExistsCache.Remove(key); + _fileSizeCache.Remove(key); + } + } + } + + public override long GetFileSize(BlobStorePath file) + { + EnsureNotDisposed(); + + if (_useCache) + { + lock (_cacheLock) + { + if (_fileSizeCache.TryGetValue(file.ToString(), out var cachedSize)) + { + return cachedSize; + } + } + } + + var blobs = GetBlobs(file); + var totalSize = blobs.Sum(b => b.Size); + + if (_useCache) + { + lock (_cacheLock) + { + _fileSizeCache[file.ToString()] = totalSize; + } + } + + return totalSize; + } + + public override bool DirectoryExists(BlobStorePath directory) + { + EnsureNotDisposed(); + + if (_useCache) + { + lock (_cacheLock) + { + if (_directoryExistsCache.TryGetValue(directory.ToString(), out var cachedExists)) + { + return cachedExists; + } + } + } + + // In Redis, directories are virtual - they exist if there are any keys with that prefix + var result = true; + + if (_useCache) + { + lock (_cacheLock) + { + _directoryExistsCache[directory.ToString()] = result; + } + } + + return result; + } + + public override bool FileExists(BlobStorePath file) + { + EnsureNotDisposed(); + + if (_useCache) + { + lock (_cacheLock) + { + if (_fileExistsCache.TryGetValue(file.ToString(), out var cachedExists)) + { + return cachedExists; + } + } + } + + var blobs = GetBlobs(file); + var exists = blobs.Count > 0; + + if (_useCache) + { + lock (_cacheLock) + { + _fileExistsCache[file.ToString()] = exists; + } + } + + return exists; + } + + public override void VisitChildren(BlobStorePath directory, IBlobStorePathVisitor visitor) + { + EnsureNotDisposed(); + + var childKeys = GetChildKeys(directory); + var directoryNames = new HashSet(); + var fileNames = new HashSet(); + + foreach (var key in childKeys) + { + if (IsDirectoryKey(key)) + { + var dirName = DirectoryNameOfKey(key); + directoryNames.Add(dirName); + } + else + { + var fileName = FileNameOfKey(key); + fileNames.Add(fileName); + } + } + + foreach (var dirName in directoryNames) + { + visitor.VisitDirectory(directory, dirName); + } + + foreach (var fileName in fileNames) + { + visitor.VisitFile(directory, fileName); + } + } + + /// + /// Extracts the directory name from a directory key. + /// + private static string DirectoryNameOfKey(string key) + { + var lastSeparator = -1; + for (int i = key.Length - 2; i >= 0; i--) + { + if (key[i] == BlobStorePath.SeparatorChar) + { + lastSeparator = i; + break; + } + } + return key.Substring(lastSeparator + 1, key.Length - lastSeparator - 2); + } + + /// + /// Extracts the file name from a blob key. + /// + private static string FileNameOfKey(string key) + { + var lastSeparator = key.LastIndexOf(BlobStorePath.SeparatorChar); + var lastDot = key.LastIndexOf(NumberSuffixSeparatorChar); + return key.Substring(lastSeparator + 1, lastDot - lastSeparator - 1); + } + + public override bool IsEmpty(BlobStorePath directory) + { + EnsureNotDisposed(); + var childKeys = GetChildKeys(directory); + return childKeys.Count == 0; + } + + public override bool CreateDirectory(BlobStorePath directory) + { + EnsureNotDisposed(); + + // In Redis, directories are virtual - they don't need to be created + if (_useCache) + { + lock (_cacheLock) + { + _directoryExistsCache[directory.ToString()] = true; + } + } + + return true; + } + + public override bool CreateFile(BlobStorePath file) + { + EnsureNotDisposed(); + + // Files are created on first write + return true; + } + + public override bool DeleteFile(BlobStorePath file) + { + EnsureNotDisposed(); + + var blobs = GetBlobs(file); + if (blobs.Count == 0) + { + return false; + } + + var keys = blobs.Select(b => (RedisKey)b.Key).ToArray(); + var deleted = _database.KeyDelete(keys); + + InvalidateCache(file.ToString()); + + return deleted == blobs.Count; + } + + public override byte[] ReadData(BlobStorePath file, long offset, long length) + { + EnsureNotDisposed(); + + if (length == 0) + { + return Array.Empty(); + } + + var blobs = GetBlobs(file); + var totalSize = blobs.Sum(b => b.Size); + var actualLength = length > 0 ? length : totalSize - offset; + + if (actualLength <= 0) + { + return Array.Empty(); + } + + var result = new byte[actualLength]; + var resultOffset = 0; + var remaining = actualLength; + var skipped = 0L; + + foreach (var blob in blobs) + { + if (remaining <= 0) + { + break; + } + + if (skipped + blob.Size <= offset) + { + skipped += blob.Size; + continue; + } + + var blobOffset = skipped < offset ? offset - skipped : 0; + var amount = Math.Min(blob.Size - blobOffset, remaining); + + var data = _database.StringGetRange(blob.Key, blobOffset, blobOffset + amount - 1); + if (data.HasValue) + { + var bytes = (byte[])data!; + Array.Copy(bytes, 0, result, resultOffset, bytes.Length); + resultOffset += bytes.Length; + remaining -= bytes.Length; + } + + skipped += blob.Size; + } + + return result; + } + + public override long ReadData(BlobStorePath file, byte[] targetBuffer, long offset, long length) + { + EnsureNotDisposed(); + + if (length == 0) + { + return 0; + } + + var data = ReadData(file, offset, length); + Array.Copy(data, 0, targetBuffer, 0, data.Length); + return data.Length; + } + + public override long WriteData(BlobStorePath file, IEnumerable sourceBuffers) + { + EnsureNotDisposed(); + + // Calculate total size + var totalSize = sourceBuffers.Sum(b => b.Length); + if (totalSize == 0) + { + return 0; + } + + // Combine all buffers into one + var combinedData = new byte[totalSize]; + var offset = 0; + foreach (var buffer in sourceBuffers) + { + Array.Copy(buffer, 0, combinedData, offset, buffer.Length); + offset += buffer.Length; + } + + // Get next blob number + var nextBlobNumber = GetNextBlobNumber(file); + var blobKey = ToBlobKeyWithContainer(file, nextBlobNumber); + + // Write to Redis + var success = _database.StringSet(blobKey, combinedData); + if (!success) + { + throw new IOException($"Failed to write data to Redis key: {blobKey}"); + } + + // Update cache + if (_useCache) + { + lock (_cacheLock) + { + _fileExistsCache[file.ToString()] = true; + if (_fileSizeCache.TryGetValue(file.ToString(), out var currentSize)) + { + _fileSizeCache[file.ToString()] = currentSize + totalSize; + } + else + { + _fileSizeCache[file.ToString()] = totalSize; + } + } + } + + return totalSize; + } + + public override void MoveFile(BlobStorePath sourceFile, BlobStorePath targetFile) + { + EnsureNotDisposed(); + + // Copy the file + CopyFile(sourceFile, targetFile, 0, -1); + + // Delete the source + DeleteFile(sourceFile); + + // Update cache + if (_useCache) + { + lock (_cacheLock) + { + _fileExistsCache[sourceFile.ToString()] = false; + _fileExistsCache[targetFile.ToString()] = true; + + if (_fileSizeCache.TryGetValue(sourceFile.ToString(), out var size)) + { + _fileSizeCache.Remove(sourceFile.ToString()); + _fileSizeCache[targetFile.ToString()] = size; + } + } + } + } + + public override long CopyFile(BlobStorePath sourceFile, BlobStorePath targetFile, long offset, long length) + { + EnsureNotDisposed(); + + var data = ReadData(sourceFile, offset, length); + return WriteData(targetFile, new[] { data }); + } + + public override void TruncateFile(BlobStorePath file, long newLength) + { + EnsureNotDisposed(); + + if (newLength == 0) + { + DeleteFile(file); + return; + } + + var blobs = GetBlobs(file); + var currentOffset = 0L; + BlobMetadata? targetBlob = null; + var blobIndex = 0; + + // Find the blob that contains the truncation point + for (int i = 0; i < blobs.Count; i++) + { + var blob = blobs[i]; + var blobStart = currentOffset; + var blobEnd = currentOffset + blob.Size - 1; + + if (blobStart <= newLength && blobEnd >= newLength) + { + targetBlob = blob; + blobIndex = i; + break; + } + + currentOffset += blob.Size; + } + + if (targetBlob == null) + { + throw new ArgumentException("New length exceeds file length"); + } + + var blobStart2 = currentOffset; + + // Delete all blobs after the target blob + if (blobIndex < blobs.Count - 1) + { + var keysToDelete = blobs.Skip(blobIndex + 1).Select(b => (RedisKey)b.Key).ToArray(); + _database.KeyDelete(keysToDelete); + } + + // If truncation point is at the start of the blob, delete it too + if (blobStart2 == newLength) + { + _database.KeyDelete(targetBlob.Key); + } + // If truncation point is in the middle of the blob, truncate it + else if (blobStart2 + targetBlob.Size > newLength) + { + var newBlobLength = newLength - blobStart2; + var data = _database.StringGetRange(targetBlob.Key, 0, newBlobLength - 1); + _database.KeyDelete(targetBlob.Key); + _database.StringSet(targetBlob.Key, data); + } + + // Update cache + if (_useCache) + { + lock (_cacheLock) + { + _fileSizeCache[file.ToString()] = newLength; + } + } + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + // Note: We don't dispose the Redis connection as it may be shared + // The caller is responsible for disposing the IConnectionMultiplexer + } + } +} diff --git a/afs/redis/test/NebulaStore.Afs.Redis.Tests.csproj b/afs/redis/test/NebulaStore.Afs.Redis.Tests.csproj new file mode 100644 index 0000000..3bb81a9 --- /dev/null +++ b/afs/redis/test/NebulaStore.Afs.Redis.Tests.csproj @@ -0,0 +1,32 @@ + + + + net9.0 + latest + enable + enable + false + true + + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + diff --git a/afs/redis/test/RedisConfigurationTests.cs b/afs/redis/test/RedisConfigurationTests.cs new file mode 100644 index 0000000..06b186a --- /dev/null +++ b/afs/redis/test/RedisConfigurationTests.cs @@ -0,0 +1,223 @@ +using Xunit; +using NebulaStore.Afs.Redis; + +namespace NebulaStore.Afs.Redis.Tests; + +/// +/// Tests for RedisConfiguration. +/// +public class RedisConfigurationTests +{ + [Fact] + public void New_CreatesDefaultConfiguration() + { + var config = RedisConfiguration.New(); + + Assert.NotNull(config); + Assert.Equal("localhost:6379", config.ConnectionString); + Assert.Equal(0, config.DatabaseNumber); + Assert.True(config.UseCache); + Assert.Equal(TimeSpan.FromMinutes(1), config.CommandTimeout); + } + + [Fact] + public void SetConnectionString_SetsValue() + { + var config = RedisConfiguration.New() + .SetConnectionString("redis.example.com:6379"); + + Assert.Equal("redis.example.com:6379", config.ConnectionString); + } + + [Fact] + public void SetConnectionString_WithNullOrEmpty_ThrowsException() + { + var config = RedisConfiguration.New(); + + Assert.Throws(() => config.SetConnectionString("")); + Assert.Throws(() => config.SetConnectionString(null!)); + } + + [Fact] + public void SetDatabaseNumber_SetsValue() + { + var config = RedisConfiguration.New() + .SetDatabaseNumber(5); + + Assert.Equal(5, config.DatabaseNumber); + } + + [Fact] + public void SetDatabaseNumber_WithNegative_ThrowsException() + { + var config = RedisConfiguration.New(); + + Assert.Throws(() => config.SetDatabaseNumber(-1)); + } + + [Fact] + public void SetUseCache_SetsValue() + { + var config = RedisConfiguration.New() + .SetUseCache(false); + + Assert.False(config.UseCache); + } + + [Fact] + public void SetCommandTimeout_SetsValue() + { + var timeout = TimeSpan.FromSeconds(30); + var config = RedisConfiguration.New() + .SetCommandTimeout(timeout); + + Assert.Equal(timeout, config.CommandTimeout); + } + + [Fact] + public void SetCommandTimeout_WithZeroOrNegative_ThrowsException() + { + var config = RedisConfiguration.New(); + + Assert.Throws(() => config.SetCommandTimeout(TimeSpan.Zero)); + Assert.Throws(() => config.SetCommandTimeout(TimeSpan.FromSeconds(-1))); + } + + [Fact] + public void SetConnectTimeout_SetsValue() + { + var timeout = TimeSpan.FromSeconds(10); + var config = RedisConfiguration.New() + .SetConnectTimeout(timeout); + + Assert.Equal(timeout, config.ConnectTimeout); + } + + [Fact] + public void SetSyncTimeout_SetsValue() + { + var timeout = TimeSpan.FromSeconds(10); + var config = RedisConfiguration.New() + .SetSyncTimeout(timeout); + + Assert.Equal(timeout, config.SyncTimeout); + } + + [Fact] + public void SetAllowAdmin_SetsValue() + { + var config = RedisConfiguration.New() + .SetAllowAdmin(false); + + Assert.False(config.AllowAdmin); + } + + [Fact] + public void SetAbortOnConnectFail_SetsValue() + { + var config = RedisConfiguration.New() + .SetAbortOnConnectFail(true); + + Assert.True(config.AbortOnConnectFail); + } + + [Fact] + public void SetPassword_SetsValue() + { + var config = RedisConfiguration.New() + .SetPassword("secret123"); + + Assert.Equal("secret123", config.Password); + } + + [Fact] + public void SetUseSsl_SetsValue() + { + var config = RedisConfiguration.New() + .SetUseSsl(true); + + Assert.True(config.UseSsl); + } + + [Fact] + public void Validate_WithValidConfiguration_DoesNotThrow() + { + var config = RedisConfiguration.New() + .SetConnectionString("localhost:6379") + .SetDatabaseNumber(0) + .SetCommandTimeout(TimeSpan.FromMinutes(1)); + + var exception = Record.Exception(() => config.Validate()); + + Assert.Null(exception); + } + + [Fact] + public void Validate_WithEmptyConnectionString_ThrowsException() + { + var config = RedisConfiguration.New(); + // Use reflection to set invalid state + var field = typeof(RedisConfiguration).GetField("k__BackingField", + System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance); + field?.SetValue(config, ""); + + Assert.Throws(() => config.Validate()); + } + + [Fact] + public void ToConfigurationOptions_CreatesValidOptions() + { + var config = RedisConfiguration.New() + .SetConnectionString("localhost:6379") + .SetDatabaseNumber(1) + .SetAllowAdmin(true) + .SetUseSsl(false); + + var options = config.ToConfigurationOptions(); + + Assert.NotNull(options); + Assert.Equal(1, options.DefaultDatabase); + Assert.True(options.AllowAdmin); + Assert.False(options.Ssl); + } + + [Fact] + public void ToConfigurationOptions_WithPassword_SetsPassword() + { + var config = RedisConfiguration.New() + .SetConnectionString("localhost:6379") + .SetPassword("secret123"); + + var options = config.ToConfigurationOptions(); + + Assert.Equal("secret123", options.Password); + } + + [Fact] + public void FluentInterface_AllowsChaining() + { + var config = RedisConfiguration.New() + .SetConnectionString("redis.example.com:6379") + .SetDatabaseNumber(2) + .SetUseCache(false) + .SetCommandTimeout(TimeSpan.FromSeconds(30)) + .SetConnectTimeout(TimeSpan.FromSeconds(10)) + .SetSyncTimeout(TimeSpan.FromSeconds(10)) + .SetAllowAdmin(false) + .SetAbortOnConnectFail(true) + .SetPassword("password") + .SetUseSsl(true); + + Assert.Equal("redis.example.com:6379", config.ConnectionString); + Assert.Equal(2, config.DatabaseNumber); + Assert.False(config.UseCache); + Assert.Equal(TimeSpan.FromSeconds(30), config.CommandTimeout); + Assert.Equal(TimeSpan.FromSeconds(10), config.ConnectTimeout); + Assert.Equal(TimeSpan.FromSeconds(10), config.SyncTimeout); + Assert.False(config.AllowAdmin); + Assert.True(config.AbortOnConnectFail); + Assert.Equal("password", config.Password); + Assert.True(config.UseSsl); + } +} + diff --git a/afs/redis/test/RedisConnectorTests.cs b/afs/redis/test/RedisConnectorTests.cs new file mode 100644 index 0000000..7ee38ad --- /dev/null +++ b/afs/redis/test/RedisConnectorTests.cs @@ -0,0 +1,312 @@ +using Xunit; +using StackExchange.Redis; +using NebulaStore.Afs.Redis; +using NebulaStore.Afs.Blobstore; + +namespace NebulaStore.Afs.Redis.Tests; + +/// +/// Tests for RedisConnector. +/// Note: These tests require a running Redis server on localhost:6379 +/// +public class RedisConnectorTests : IDisposable +{ + private readonly IConnectionMultiplexer? _redis; + private readonly RedisConnector? _connector; + private readonly bool _redisAvailable; + + public RedisConnectorTests() + { + try + { + // Try to connect to Redis + _redis = ConnectionMultiplexer.Connect("localhost:6379"); + _connector = RedisConnector.New(_redis, databaseNumber: 15); // Use DB 15 for tests + _redisAvailable = true; + + // Clean up test database + var server = _redis.GetServer(_redis.GetEndPoints().First()); + var db = _redis.GetDatabase(15); + server.FlushDatabase(15); + } + catch + { + _redisAvailable = false; + } + } + + [Fact] + public void Constructor_WithValidConnection_CreatesConnector() + { + if (!_redisAvailable) + { + // Skip test if Redis is not available + return; + } + + Assert.NotNull(_connector); + } + + [Fact] + public void CreateDirectory_CreatesVirtualDirectory() + { + if (!_redisAvailable || _connector == null) + return; + + var directory = BlobStorePath.New("test-container", "dir1"); + var result = _connector.CreateDirectory(directory); + + Assert.True(result); + Assert.True(_connector.DirectoryExists(directory)); + } + + [Fact] + public void CreateFile_CreatesVirtualFile() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "dir1", "file1.dat"); + var result = _connector.CreateFile(file); + + Assert.True(result); + } + + [Fact] + public void WriteData_WritesDataToRedis() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "test.dat"); + var data = System.Text.Encoding.UTF8.GetBytes("Hello, Redis!"); + + var bytesWritten = _connector.WriteData(file, new[] { data }); + + Assert.Equal(data.Length, bytesWritten); + Assert.True(_connector.FileExists(file)); + } + + [Fact] + public void ReadData_ReadsDataFromRedis() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "test.dat"); + var originalData = System.Text.Encoding.UTF8.GetBytes("Hello, Redis!"); + + _connector.WriteData(file, new[] { originalData }); + var readData = _connector.ReadData(file, 0, -1); + + Assert.Equal(originalData, readData); + } + + [Fact] + public void ReadData_WithOffset_ReadsPartialData() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "test.dat"); + var originalData = System.Text.Encoding.UTF8.GetBytes("Hello, Redis!"); + + _connector.WriteData(file, new[] { originalData }); + var readData = _connector.ReadData(file, 7, 5); // Read "Redis" + + var expected = System.Text.Encoding.UTF8.GetBytes("Redis"); + Assert.Equal(expected, readData); + } + + [Fact] + public void GetFileSize_ReturnsCorrectSize() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "test.dat"); + var data = System.Text.Encoding.UTF8.GetBytes("Hello, Redis!"); + + _connector.WriteData(file, new[] { data }); + var size = _connector.GetFileSize(file); + + Assert.Equal(data.Length, size); + } + + [Fact] + public void FileExists_ReturnsTrueForExistingFile() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "test.dat"); + var data = System.Text.Encoding.UTF8.GetBytes("Test"); + + _connector.WriteData(file, new[] { data }); + + Assert.True(_connector.FileExists(file)); + } + + [Fact] + public void FileExists_ReturnsFalseForNonExistingFile() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "nonexistent.dat"); + + Assert.False(_connector.FileExists(file)); + } + + [Fact] + public void DeleteFile_DeletesFile() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "test.dat"); + var data = System.Text.Encoding.UTF8.GetBytes("Test"); + + _connector.WriteData(file, new[] { data }); + Assert.True(_connector.FileExists(file)); + + var deleted = _connector.DeleteFile(file); + + Assert.True(deleted); + Assert.False(_connector.FileExists(file)); + } + + [Fact] + public void DeleteFile_ReturnsFalseForNonExistingFile() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "nonexistent.dat"); + + var deleted = _connector.DeleteFile(file); + + Assert.False(deleted); + } + + [Fact] + public void WriteData_MultipleBuffers_CombinesData() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "multi.dat"); + var buffer1 = System.Text.Encoding.UTF8.GetBytes("Hello, "); + var buffer2 = System.Text.Encoding.UTF8.GetBytes("Redis!"); + + var bytesWritten = _connector.WriteData(file, new[] { buffer1, buffer2 }); + + Assert.Equal(buffer1.Length + buffer2.Length, bytesWritten); + + var readData = _connector.ReadData(file, 0, -1); + var expected = System.Text.Encoding.UTF8.GetBytes("Hello, Redis!"); + Assert.Equal(expected, readData); + } + + [Fact] + public void CopyFile_CopiesFileData() + { + if (!_redisAvailable || _connector == null) + return; + + var sourceFile = BlobStorePath.New("test-container", "data", "source.dat"); + var targetFile = BlobStorePath.New("test-container", "data", "target.dat"); + var data = System.Text.Encoding.UTF8.GetBytes("Copy me!"); + + _connector.WriteData(sourceFile, new[] { data }); + var bytesCopied = _connector.CopyFile(sourceFile, targetFile, 0, -1); + + Assert.Equal(data.Length, bytesCopied); + Assert.True(_connector.FileExists(targetFile)); + + var copiedData = _connector.ReadData(targetFile, 0, -1); + Assert.Equal(data, copiedData); + } + + [Fact] + public void MoveFile_MovesFileData() + { + if (!_redisAvailable || _connector == null) + return; + + var sourceFile = BlobStorePath.New("test-container", "data", "source.dat"); + var targetFile = BlobStorePath.New("test-container", "data", "target.dat"); + var data = System.Text.Encoding.UTF8.GetBytes("Move me!"); + + _connector.WriteData(sourceFile, new[] { data }); + _connector.MoveFile(sourceFile, targetFile); + + Assert.False(_connector.FileExists(sourceFile)); + Assert.True(_connector.FileExists(targetFile)); + + var movedData = _connector.ReadData(targetFile, 0, -1); + Assert.Equal(data, movedData); + } + + [Fact] + public void TruncateFile_TruncatesFileToZero() + { + if (!_redisAvailable || _connector == null) + return; + + var file = BlobStorePath.New("test-container", "data", "truncate.dat"); + var data = System.Text.Encoding.UTF8.GetBytes("Truncate me!"); + + _connector.WriteData(file, new[] { data }); + _connector.TruncateFile(file, 0); + + Assert.False(_connector.FileExists(file)); + } + + [Fact] + public void IsEmpty_ReturnsTrueForEmptyDirectory() + { + if (!_redisAvailable || _connector == null) + return; + + var directory = BlobStorePath.New("test-container", "empty-dir"); + + Assert.True(_connector.IsEmpty(directory)); + } + + [Fact] + public void IsEmpty_ReturnsFalseForNonEmptyDirectory() + { + if (!_redisAvailable || _connector == null) + return; + + var directory = BlobStorePath.New("test-container", "non-empty-dir"); + var file = BlobStorePath.New("test-container", "non-empty-dir", "file.dat"); + var data = System.Text.Encoding.UTF8.GetBytes("Data"); + + _connector.WriteData(file, new[] { data }); + + Assert.False(_connector.IsEmpty(directory)); + } + + public void Dispose() + { + if (_redisAvailable && _redis != null) + { + // Clean up test database + try + { + var server = _redis.GetServer(_redis.GetEndPoints().First()); + server.FlushDatabase(15); + } + catch + { + // Ignore cleanup errors + } + + _connector?.Dispose(); + _redis?.Dispose(); + } + } +} + diff --git a/examples/RedisExample.cs b/examples/RedisExample.cs new file mode 100644 index 0000000..1f1b495 --- /dev/null +++ b/examples/RedisExample.cs @@ -0,0 +1,281 @@ +using System; +using System.Collections.Generic; +using StackExchange.Redis; +using NebulaStore.Afs.Redis; +using NebulaStore.Afs.Blobstore; +using NebulaStore.Storage.Embedded; +using NebulaStore.Storage.EmbeddedConfiguration; + +namespace NebulaStore.Examples; + +/// +/// Examples demonstrating Redis AFS connector usage with NebulaStore. +/// +public static class RedisExample +{ + // Example data models + public class Product + { + public int Id { get; set; } + public string Name { get; set; } = ""; + public decimal Price { get; set; } + public List Tags { get; set; } = new(); + } + + public class Inventory + { + public List Products { get; set; } = new(); + public DateTime LastUpdated { get; set; } + } + + /// + /// Basic example using Redis with EmbeddedStorage. + /// + public static void BasicRedisStorageExample() + { + Console.WriteLine("=== Basic Redis Storage Example ===\n"); + + // Configure storage to use Redis + var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("redis-storage") + .SetAfsStorageType("redis") + .SetAfsConnectionString("localhost:6379") + .SetAfsUseCache(true); + + // Start storage with Redis backend + using var storage = EmbeddedStorage.StartWithAfs(config); + + // Initialize or get root object + var inventory = storage.Root(); + if (inventory.Products.Count == 0) + { + Console.WriteLine("Initializing inventory..."); + inventory.Products.Add(new Product + { + Id = 1, + Name = "Laptop", + Price = 999.99m, + Tags = new List { "electronics", "computers" } + }); + inventory.Products.Add(new Product + { + Id = 2, + Name = "Mouse", + Price = 29.99m, + Tags = new List { "electronics", "accessories" } + }); + inventory.LastUpdated = DateTime.UtcNow; + } + + // Store the root object + storage.StoreRoot(); + Console.WriteLine($"Stored {inventory.Products.Count} products"); + Console.WriteLine($"Last updated: {inventory.LastUpdated}"); + + // Display products + foreach (var product in inventory.Products) + { + Console.WriteLine($" - {product.Name}: ${product.Price} (Tags: {string.Join(", ", product.Tags)})"); + } + } + + /// + /// Example using direct AFS API with Redis. + /// + public static void DirectRedisAfsExample() + { + Console.WriteLine("\n=== Direct Redis AFS Example ===\n"); + + // Create Redis connection + var redis = ConnectionMultiplexer.Connect("localhost:6379"); + Console.WriteLine("Connected to Redis"); + + // Create connector with caching + using var connector = RedisConnector.Caching(redis); + using var fileSystem = BlobStoreFileSystem.New(connector); + + // Create a path + var path = BlobStorePath.New("products", "data", "product-1.dat"); + + // Write data + var productData = System.Text.Encoding.UTF8.GetBytes("Product: Laptop, Price: $999.99"); + var bytesWritten = fileSystem.IoHandler.WriteData(path, productData); + Console.WriteLine($"Written {bytesWritten} bytes to {path}"); + + // Read data back + var readData = fileSystem.IoHandler.ReadData(path, 0, -1); + var content = System.Text.Encoding.UTF8.GetString(readData); + Console.WriteLine($"Read back: {content}"); + + // Check file size + var fileSize = fileSystem.IoHandler.GetFileSize(path); + Console.WriteLine($"File size: {fileSize} bytes"); + + // Delete file + var deleted = fileSystem.IoHandler.DeleteFile(path); + Console.WriteLine($"File deleted: {deleted}"); + } + + /// + /// Example using RedisConfiguration for advanced setup. + /// + public static void AdvancedRedisConfigurationExample() + { + Console.WriteLine("\n=== Advanced Redis Configuration Example ===\n"); + + // Create detailed configuration + var redisConfig = RedisConfiguration.New() + .SetConnectionString("localhost:6379") + .SetDatabaseNumber(1) // Use database 1 instead of default 0 + .SetUseCache(true) + .SetCommandTimeout(TimeSpan.FromSeconds(30)) + .SetConnectTimeout(TimeSpan.FromSeconds(10)) + .SetAllowAdmin(true); + + Console.WriteLine("Redis Configuration:"); + Console.WriteLine($" Connection: {redisConfig.ConnectionString}"); + Console.WriteLine($" Database: {redisConfig.DatabaseNumber}"); + Console.WriteLine($" Cache: {redisConfig.UseCache}"); + Console.WriteLine($" Command Timeout: {redisConfig.CommandTimeout}"); + + // Build StackExchange.Redis options + var options = redisConfig.ToConfigurationOptions(); + var redis = ConnectionMultiplexer.Connect(options); + + // Create connector + using var connector = RedisConnector.New(redis, redisConfig.DatabaseNumber); + using var fileSystem = BlobStoreFileSystem.New(connector); + + // Test operations + var testPath = BlobStorePath.New("test-container", "advanced", "test.dat"); + var testData = new byte[1024]; // 1KB of test data + new Random().NextBytes(testData); + + var bytesWritten = fileSystem.IoHandler.WriteData(testPath, testData); + Console.WriteLine($"\nWritten {bytesWritten} bytes to Redis database {redisConfig.DatabaseNumber}"); + + var readData = fileSystem.IoHandler.ReadData(testPath, 0, -1); + Console.WriteLine($"Read {readData.Length} bytes back"); + Console.WriteLine($"Data integrity: {testData.SequenceEqual(readData)}"); + + // Cleanup + fileSystem.IoHandler.DeleteFile(testPath); + Console.WriteLine("Test file deleted"); + } + + /// + /// Example demonstrating Redis with multiple storage operations. + /// + public static void MultipleOperationsExample() + { + Console.WriteLine("\n=== Multiple Operations Example ===\n"); + + var config = EmbeddedStorageConfiguration.New() + .SetStorageDirectory("redis-multi-storage") + .SetAfsStorageType("redis") + .SetAfsConnectionString("localhost:6379") + .SetAfsUseCache(true); + + using var storage = EmbeddedStorage.StartWithAfs(config); + + var inventory = storage.Root(); + + // Add multiple products + Console.WriteLine("Adding products..."); + for (int i = 1; i <= 5; i++) + { + inventory.Products.Add(new Product + { + Id = i, + Name = $"Product {i}", + Price = 10.00m * i, + Tags = new List { "category-" + (i % 3), "tag-" + i } + }); + } + inventory.LastUpdated = DateTime.UtcNow; + + // Store all + storage.StoreRoot(); + Console.WriteLine($"Stored {inventory.Products.Count} products"); + + // Update a product + Console.WriteLine("\nUpdating product..."); + var productToUpdate = inventory.Products[0]; + productToUpdate.Price = 99.99m; + productToUpdate.Tags.Add("updated"); + storage.Store(productToUpdate); + Console.WriteLine($"Updated {productToUpdate.Name} to ${productToUpdate.Price}"); + + // Remove a product + Console.WriteLine("\nRemoving product..."); + var productToRemove = inventory.Products[^1]; + inventory.Products.Remove(productToRemove); + storage.StoreRoot(); + Console.WriteLine($"Removed {productToRemove.Name}, now have {inventory.Products.Count} products"); + + // Display final state + Console.WriteLine("\nFinal inventory:"); + foreach (var product in inventory.Products) + { + Console.WriteLine($" - {product.Name}: ${product.Price}"); + } + } + + /// + /// Example showing Redis connection with authentication. + /// + public static void RedisWithAuthenticationExample() + { + Console.WriteLine("\n=== Redis with Authentication Example ===\n"); + + // Note: This example assumes you have a Redis server with authentication enabled + var redisConfig = RedisConfiguration.New() + .SetConnectionString("localhost:6379") + .SetPassword("your-redis-password") // Set your Redis password + .SetUseSsl(false) // Set to true if using SSL/TLS + .SetDatabaseNumber(0) + .SetUseCache(true); + + try + { + var options = redisConfig.ToConfigurationOptions(); + Console.WriteLine("Attempting to connect to Redis with authentication..."); + + // Note: This will fail if Redis is not configured with the password + // Uncomment to test with actual Redis authentication + // var redis = ConnectionMultiplexer.Connect(options); + // using var connector = RedisConnector.New(redis); + // Console.WriteLine("Successfully connected to Redis with authentication"); + + Console.WriteLine("(Example code - requires Redis with authentication configured)"); + } + catch (Exception ex) + { + Console.WriteLine($"Connection failed: {ex.Message}"); + } + } + + /// + /// Run all Redis examples. + /// + public static void RunAllExamples() + { + try + { + BasicRedisStorageExample(); + DirectRedisAfsExample(); + AdvancedRedisConfigurationExample(); + MultipleOperationsExample(); + RedisWithAuthenticationExample(); + + Console.WriteLine("\n=== All Redis Examples Completed ==="); + } + catch (Exception ex) + { + Console.WriteLine($"\nError running examples: {ex.Message}"); + Console.WriteLine("Make sure Redis server is running on localhost:6379"); + Console.WriteLine("You can start Redis with: redis-server"); + } + } +} +