Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions NebulaStore.sln
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,15 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NebulaStore.Afs.Aws.S3", "a
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NebulaStore.Afs.Aws.S3.Tests", "afs\aws\s3\test\NebulaStore.Afs.Aws.S3.Tests.csproj", "{L1M2N3O4-P5Q6-7890-RSTU-VW1234567890}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "azure", "azure", "{M2N3O4P5-Q6R7-8901-STUV-WX2345678901}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NebulaStore.Afs.Azure.Storage", "afs\azure\storage\NebulaStore.Afs.Azure.Storage.csproj", "{N3O4P5Q6-R7S8-9012-TUVW-XY3456789012}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NebulaStore.Afs.Azure.Storage.Tests", "afs\azure\storage\test\NebulaStore.Afs.Azure.Storage.Tests.csproj", "{Q7R8S9T0-U1V2-3456-WXYZ-AB4567890123}"
EndProject



Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -92,6 +101,17 @@ Global
{L1M2N3O4-P5Q6-7890-RSTU-VW1234567890}.Debug|Any CPU.Build.0 = Debug|Any CPU
{L1M2N3O4-P5Q6-7890-RSTU-VW1234567890}.Release|Any CPU.ActiveCfg = Release|Any CPU
{L1M2N3O4-P5Q6-7890-RSTU-VW1234567890}.Release|Any CPU.Build.0 = Release|Any CPU
{N3O4P5Q6-R7S8-9012-TUVW-XY3456789012}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{N3O4P5Q6-R7S8-9012-TUVW-XY3456789012}.Debug|Any CPU.Build.0 = Debug|Any CPU
{N3O4P5Q6-R7S8-9012-TUVW-XY3456789012}.Release|Any CPU.ActiveCfg = Release|Any CPU
{N3O4P5Q6-R7S8-9012-TUVW-XY3456789012}.Release|Any CPU.Build.0 = Release|Any CPU
{Q7R8S9T0-U1V2-3456-WXYZ-AB4567890123}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{Q7R8S9T0-U1V2-3456-WXYZ-AB4567890123}.Debug|Any CPU.Build.0 = Debug|Any CPU
{Q7R8S9T0-U1V2-3456-WXYZ-AB4567890123}.Release|Any CPU.ActiveCfg = Release|Any CPU
{Q7R8S9T0-U1V2-3456-WXYZ-AB4567890123}.Release|Any CPU.Build.0 = Release|Any CPU



EndGlobalSection
GlobalSection(NestedProjects) = preSolution
{A1B2C3D4-E5F6-7890-ABCD-EF1234567890} = {D0E78B5A-0D39-4CE8-A836-5FC8C7D60478}
Expand All @@ -105,5 +125,10 @@ Global
{J9K0L1M2-N3O4-5678-PQRS-TU9012345678} = {F6G7H8I9-J0K1-2345-MNOP-QR6789012345}
{K0L1M2N3-O4P5-6789-QRST-UV0123456789} = {J9K0L1M2-N3O4-5678-PQRS-TU9012345678}
{L1M2N3O4-P5Q6-7890-RSTU-VW1234567890} = {J9K0L1M2-N3O4-5678-PQRS-TU9012345678}
{M2N3O4P5-Q6R7-8901-STUV-WX2345678901} = {F6G7H8I9-J0K1-2345-MNOP-QR6789012345}
{N3O4P5Q6-R7S8-9012-TUVW-XY3456789012} = {M2N3O4P5-Q6R7-8901-STUV-WX2345678901}
{Q7R8S9T0-U1V2-3456-WXYZ-AB4567890123} = {M2N3O4P5-Q6R7-8901-STUV-WX2345678901}


EndGlobalSection
EndGlobal
36 changes: 36 additions & 0 deletions afs/azure/storage/NebulaStore.Afs.Azure.Storage.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net9.0</TargetFramework>
<LangVersion>latest</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
<PackageId>NebulaStore.Afs.Azure.Storage</PackageId>
<PackageVersion>1.0.0</PackageVersion>
<Authors>NebulaStore Contributors</Authors>
<Description>Azure Blob Storage connector for NebulaStore Abstract File System (AFS)</Description>
<PackageTags>NebulaStore;AFS;Azure;BlobStorage;Storage</PackageTags>
<PackageLicenseExpression>MIT</PackageLicenseExpression>
<RepositoryUrl>https://github.com/hadv/NebulaStore</RepositoryUrl>
<RepositoryType>git</RepositoryType>
<GenerateAssemblyInfo>false</GenerateAssemblyInfo>
<GenerateTargetFrameworkAttribute>false</GenerateTargetFrameworkAttribute>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\..\blobstore\NebulaStore.Afs.Blobstore.csproj" />
</ItemGroup>

<ItemGroup>
<PackageReference Include="Azure.Storage.Blobs" Version="12.19.1" />
<PackageReference Include="Azure.Identity" Version="1.12.0" />
</ItemGroup>

<ItemGroup>
<Compile Remove="test/**" />
<EmbeddedResource Remove="test/**" />
<None Remove="test/**" />
</ItemGroup>

</Project>
177 changes: 177 additions & 0 deletions afs/azure/storage/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
# NebulaStore Azure Blob Storage Connector

Azure Blob Storage implementation for NebulaStore Abstract File System (AFS).

## Overview

This connector enables NebulaStore to use Azure Blob Storage as a backend storage system. It implements the `IBlobStoreConnector` interface and provides seamless integration with Azure's cloud storage services.

## Features

- **Azure Blob Storage Integration**: Store data directly in Azure Blob Storage containers
- **Automatic Blob Management**: Files are automatically split into numbered blobs for optimal performance
- **Container Validation**: Enforces Azure container naming rules and validation
- **Caching Support**: Optional caching for improved performance
- **Connection String Support**: Multiple authentication methods including connection strings and managed identity
- **Large File Support**: Handles files larger than Azure's single blob limits through automatic chunking

## Installation

Add the NuGet package to your project:

```bash
dotnet add package NebulaStore.Afs.Azure.Storage
```

## Quick Start

### Using with EmbeddedStorage

```csharp
using NebulaStore.Storage.Embedded;
using NebulaStore.Storage.EmbeddedConfiguration;

var config = EmbeddedStorageConfiguration.New()
.SetStorageDirectory("my-container")
.SetUseAfs(true)
.SetAfsStorageType("azure.storage")
.SetAfsConnectionString("DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net")
.SetAfsUseCache(true)
.Build();

using var storage = EmbeddedStorage.StartWithAfs(config);

// Use storage normally
var root = storage.Root<MyDataClass>();
root.SomeProperty = "value";
storage.StoreRoot();
```

### Direct AFS Usage

```csharp
using Azure.Storage.Blobs;
using NebulaStore.Afs.Azure.Storage;
using NebulaStore.Afs.Blobstore;

// Create Azure Blob Service client
var connectionString = "DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net";
var blobServiceClient = new BlobServiceClient(connectionString);

// Create connector
using var connector = AzureStorageConnector.New(blobServiceClient);

// Create file system
using var fileSystem = BlobStoreFileSystem.New(connector);

// Perform operations
var path = BlobStorePath.New("my-container", "folder", "file.txt");
var data = System.Text.Encoding.UTF8.GetBytes("Hello, Azure!");

fileSystem.IoHandler.WriteData(path, new[] { data });
var readData = fileSystem.IoHandler.ReadData(path, 0, -1);
```

## Configuration

### Connection String

```csharp
var config = EmbeddedStorageConfiguration.New()
.SetAfsConnectionString("DefaultEndpointsProtocol=https;AccountName=myaccount;AccountKey=mykey;EndpointSuffix=core.windows.net")
.Build();
```

### Managed Identity

```csharp
using Azure.Identity;
using Azure.Storage.Blobs;

var credential = new DefaultAzureCredential();
var blobServiceClient = new BlobServiceClient(new Uri("https://myaccount.blob.core.windows.net"), credential);
var connector = AzureStorageConnector.New(blobServiceClient);
```

### Advanced Configuration

```csharp
var azureConfig = AzureStorageConfiguration.New()
.SetConnectionString(connectionString)
.SetUseCache(true)
.SetMaxBlobSize(100 * 1024 * 1024) // 100MB per blob
.SetRetryOptions(new BlobRequestOptions { MaximumExecutionTime = TimeSpan.FromMinutes(5) });

var connector = AzureStorageConnector.New(azureConfig);
```

## Container Naming Rules

Azure Blob Storage containers must follow specific naming rules:

- Container names must be between 3 and 63 characters long
- Container names can contain only lowercase letters, numbers, and dashes (-)
- Container names must begin with a lowercase letter or number
- Container names cannot end with a dash (-)
- Container names cannot have consecutive dashes (--)

## Performance Considerations

- **Blob Size**: Files larger than 100MB are automatically split into multiple blobs
- **Caching**: Enable caching for frequently accessed metadata
- **Parallel Operations**: The connector supports concurrent read/write operations
- **Regional Placement**: Choose Azure regions close to your application for optimal performance

## Error Handling

The connector includes comprehensive error handling for common Azure storage scenarios:

```csharp
try
{
using var storage = EmbeddedStorage.StartWithAfs(config);
// ... operations
}
catch (Azure.RequestFailedException ex) when (ex.Status == 404)
{
// Container or blob not found
Console.WriteLine($"Resource not found: {ex.Message}");
}
catch (Azure.RequestFailedException ex) when (ex.Status == 403)
{
// Access denied
Console.WriteLine($"Access denied: {ex.Message}");
}
catch (ArgumentException ex)
{
// Invalid container name or configuration
Console.WriteLine($"Configuration error: {ex.Message}");
}
```

## Examples

See the [examples](../../../examples/) directory for complete working examples:

- `AzureStorageExample.cs` - Basic Azure storage usage
- `AzureStorageAdvancedExample.cs` - Advanced configuration and error handling
- `AzureStoragePerformanceExample.cs` - Performance optimization techniques

## Testing

The connector includes comprehensive tests that can run against:

- **Azurite Emulator**: Local development and testing
- **Azure Storage Account**: Integration testing with real Azure services

```bash
# Run tests with Azurite emulator
dotnet test

# Run integration tests (requires Azure storage account)
dotnet test --configuration Release --logger "console;verbosity=detailed"
```

## License

This project is licensed under the MIT License - see the [LICENSE](../../../LICENSE) file for details.
Loading
Loading