Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions src/AmtocBots.Api/AmtocBots.Api.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
<Project Sdk="Microsoft.NET.Sdk.Web">

<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<RootNamespace>AmtocBots.Api</RootNamespace>
<AssemblyName>AmtocBots.Api</AssemblyName>
</PropertyGroup>

<ItemGroup>
<!-- EF Core + PostgreSQL + pgvector -->
<PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.*" />
<PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.2.*" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.*">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
</PackageReference>

<!-- Auth -->
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.*" />

<!-- Docker management -->
<PackageReference Include="Docker.DotNet" Version="3.125.*" />

<!-- Redis -->
<PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="10.0.*" />
<PackageReference Include="StackExchange.Redis" Version="2.8.*" />

<!-- Cron scheduling -->
<PackageReference Include="NCrontab" Version="3.3.*" />

<!-- Password/token hashing -->
<PackageReference Include="BCrypt.Net-Next" Version="4.0.*" />

<!-- JSON5 support for OpenClaw configs -->
<PackageReference Include="Hjson" Version="3.1.*" />
</ItemGroup>

</Project>
72 changes: 72 additions & 0 deletions src/AmtocBots.Api/BackgroundServices/MetricsPollingService.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
using AmtocBots.Api.Data;
using AmtocBots.Api.Hubs;
using AmtocBots.Api.Services.Docker;
using Microsoft.AspNetCore.SignalR;
using Microsoft.EntityFrameworkCore;

namespace AmtocBots.Api.BackgroundServices;

public sealed class MetricsPollingService(
IServiceScopeFactory scopeFactory,
IHubContext<InstanceHub> hub,
IDockerService docker,
ILogger<MetricsPollingService> log) : BackgroundService
{
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
log.LogInformation("Metrics polling service started");

while (!stoppingToken.IsCancellationRequested)
{
try
{
await PollAndBroadcastAsync(stoppingToken);
}
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
{
log.LogWarning(ex, "Error during metrics polling");
}

await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
}
}

private async Task PollAndBroadcastAsync(CancellationToken ct)
{
using var scope = scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();

var running = await db.Instances
.Where(i => i.ContainerId != null && i.Status == "running")
.Select(i => new { i.Id, ContainerId = i.ContainerId! })
.ToListAsync(ct);

if (running.Count == 0) return;

var stats = await docker.GetAllManagedStatsAsync(
running.Select(r => (r.Id, r.ContainerId)), ct);

foreach (var stat in stats)
{
await hub.Clients.Group($"instance:{stat.InstanceId}")
.SendAsync("StatusUpdate", stat, ct);
}

// Sync status changes back to DB
var statusMap = stats.ToDictionary(s => s.InstanceId, s => s.Status);
foreach (var inst in running)
{
if (statusMap.TryGetValue(inst.Id, out var newStatus))
{
var entity = await db.Instances.FindAsync([inst.Id], ct);
if (entity is not null && entity.Status != newStatus)
{
entity.Status = newStatus;
entity.UpdatedAt = DateTimeOffset.UtcNow;
}
}
}

await db.SaveChangesAsync(ct);
}
}
69 changes: 69 additions & 0 deletions src/AmtocBots.Api/BackgroundServices/ModelSwitchScheduler.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
using AmtocBots.Api.Data;
using AmtocBots.Api.Services.Models;
using Microsoft.EntityFrameworkCore;
using NCrontab;

namespace AmtocBots.Api.BackgroundServices;

public sealed class ModelSwitchScheduler(
IServiceScopeFactory scopeFactory,
ILogger<ModelSwitchScheduler> log) : BackgroundService
{
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
log.LogInformation("Model switch scheduler started");

while (!stoppingToken.IsCancellationRequested)
{
try
{
await EvaluateAsync(stoppingToken);
}
catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
{
log.LogWarning(ex, "Error in model switch scheduler");
}

await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
}
}

private async Task EvaluateAsync(CancellationToken ct)
{
using var scope = scopeFactory.CreateScope();
var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
var switcher = scope.ServiceProvider.GetRequiredService<IModelSwitchingService>();

var now = DateTime.UtcNow;
var cronRules = await db.SwitchRules
.Where(r => r.IsActive && r.RuleType == "cron" && r.CronExpression != null)
.ToListAsync(ct);

foreach (var rule in cronRules)
{
try
{
var schedule = CrontabSchedule.Parse(rule.CronExpression!, new CrontabSchedule.ParseOptions { IncludingSeconds = false });
var next = schedule.GetNextOccurrence(now.AddMinutes(-1));
if (next <= now)
{
log.LogInformation("Cron rule triggered for instance {Id}: switch to {Model}", rule.InstanceId, rule.TargetModel);
await switcher.SwitchModelAsync(rule.InstanceId, rule.TargetModel, ct);
}
}
catch (Exception ex)
{
log.LogWarning(ex, "Failed to evaluate cron rule {Id}", rule.Id);
}
}

// Also evaluate threshold rules for all running instances
var runningIds = await db.Instances
.Where(i => i.Status == "running")
.Select(i => i.Id)
.ToListAsync(ct);

foreach (var id in runningIds)
await switcher.EvaluateThresholdRulesAsync(id, ct);
}
}
56 changes: 56 additions & 0 deletions src/AmtocBots.Api/BackgroundServices/QueueRetryWorker.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
using AmtocBots.Api.Services.OpenClaw;
using AmtocBots.Api.Services.Queue;

namespace AmtocBots.Api.BackgroundServices;

public sealed class QueueRetryWorker(
RedisMessageQueueService queue,
IOpenClawClient openClaw,
ILogger<QueueRetryWorker> log) : BackgroundService
{
private static readonly TimeSpan[] Backoffs = [
TimeSpan.FromSeconds(5),
TimeSpan.FromSeconds(15),
TimeSpan.FromSeconds(30),
TimeSpan.FromMinutes(2),
TimeSpan.FromMinutes(5),
];

protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
log.LogInformation("Queue retry worker started");

while (!stoppingToken.IsCancellationRequested)
{
// Move any ready delayed messages back to main queue
await queue.FlushReadyDelayedAsync(stoppingToken);

var msg = await queue.DequeueAsync(stoppingToken);
if (msg is null)
{
await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
continue;
}

try
{
await openClaw.RunAgentAsync(msg.BaseUrl, msg.BearerToken,
new AgentRequest(msg.Description, msg.Model), stoppingToken);
log.LogDebug("Retried queued message for instance {Id}", msg.InstanceId);
}
catch (HttpRequestException ex) when ((int?)ex.StatusCode == 429)
{
var delay = msg.RetryCount < Backoffs.Length
? Backoffs[msg.RetryCount]
: TimeSpan.FromMinutes(10);

log.LogWarning("Rate limited on instance {Id}, requeuing with {Delay}s delay", msg.InstanceId, delay.TotalSeconds);
await queue.RequeueWithDelayAsync(msg, delay, stoppingToken);
}
catch (Exception ex)
{
log.LogError(ex, "Failed to send queued message for instance {Id}", msg.InstanceId);
}
}
}
}
10 changes: 10 additions & 0 deletions src/AmtocBots.Api/Configuration/DockerOptions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
namespace AmtocBots.Api.Configuration;

public sealed class DockerOptions
{
public string SocketPath { get; set; } = "/var/run/docker.sock";
public string OpenClawNetwork { get; set; } = "amtocbots_openclaw";
public string OpenClawImage { get; set; } = "ghcr.io/openclaw/openclaw:latest";
public int PortRangeStart { get; set; } = 18789;
public int PortRangeEnd { get; set; } = 19789;
}
7 changes: 7 additions & 0 deletions src/AmtocBots.Api/Configuration/EncryptionOptions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace AmtocBots.Api.Configuration;

public sealed class EncryptionOptions
{
/// <summary>Base64-encoded 32-byte AES-256 key. Generate with: openssl rand -base64 32</summary>
public string Key { get; set; } = string.Empty;
}
7 changes: 7 additions & 0 deletions src/AmtocBots.Api/Configuration/KeycloakOptions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
namespace AmtocBots.Api.Configuration;

public sealed class KeycloakOptions
{
public string Authority { get; set; } = string.Empty;
public string Audience { get; set; } = "amtocbots-api";
}
6 changes: 6 additions & 0 deletions src/AmtocBots.Api/Configuration/OllamaOptions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
namespace AmtocBots.Api.Configuration;

public sealed class OllamaOptions
{
public string BaseUrl { get; set; } = "http://localhost:11434";
}
Loading