AmtocBots Manager — Full Implementation (Phases 2–9)

CLAUDE.md

@@ -62,10 +62,10 @@ dotnet ef database update          # apply migrations
 ### Frontend (Angular 21)
 ```bash
 cd src/AmtocBots.Web
-npm install
-ng serve                           # dev server on :4200 (proxies /api to :8080)
+npm install --legacy-peer-deps     # required due to peer dep conflicts
+ng serve                           # dev server on :4200 (proxies /api and /hubs to :8080)
 ng build                           # production build
-ng test                            # unit tests
+ng test                            # unit tests (Karma/Jasmine)
 ng generate component features/X/components/Y --standalone
 ```
 
@@ -102,7 +102,8 @@ OpenClaw containers are **not** in `docker-compose.yml` — they are spawned dyn
 - **Control flow** — `@if`, `@for`, `@switch` (not `*ngIf`/`*ngFor`)
 - **`inject()`** — use in constructors and `computed`/`effect` bodies, not constructor params
 - **Lazy routes** — all features loaded via `loadChildren` with `loadComponent` for leaf routes
-- **Feature stores** — one `@Injectable({ providedIn: 'root' })` signal store per feature
+- **Feature stores** — one `@Injectable({ providedIn: 'root' })` signal store per feature; use `takeUntilDestroyed()` for cleanup and `firstValueFrom()` to bridge observables to promises; apply optimistic updates before API calls
+- **Path aliases** — `@core/*`, `@shared/*`, `@features/*`, `@env/*` (configured in `tsconfig.json`)
 
 ## Auth Flow
 
@@ -131,12 +132,24 @@ Channel tokens in `channel_configs.config_json` are **AES-256 encrypted at rest*
 
 ## Cloudflare Tunnel Setup
 
+Caddy does **not** handle TLS — Cloudflare Tunnel terminates HTTPS externally.
+
 1. `cloudflared tunnel login`
 2. `cloudflared tunnel create amtocbots-manager`
-3. Copy credentials JSON → `infra/cloudflare/credentials.json` (gitignored)
+3. Set `TUNNEL_TOKEN` in `.env` (token-based auth; credentials JSON not used)
 4. Set tunnel ID in `infra/cloudflare/config.yml`
 5. Add CNAME DNS records in Cloudflare dashboard
 
 ## Environment
 
 Copy `.env.example` → `.env` and fill in all values. The override file `docker-compose.override.yml` is auto-included by `docker compose` for dev (exposes ports, uses `start-dev` for Keycloak).
+
+Dev port exposure via `docker-compose.override.yml`: API `:8080`, Angular `:4200`, PostgreSQL `:5432`, Keycloak admin `:8180`.
+
+## API Endpoint Patterns (.NET)
+
+The API mixes two styles — prefer minimal endpoints for new routes:
+- **Minimal endpoints** (`src/AmtocBots.Api/Endpoints/`) — extension methods registered via `MapGroup()` chains in `Program.cs`
+- **Controllers** (`src/AmtocBots.Api/Controllers/`) — older pattern, still present
+
+Background services use NCrontab for cron expression parsing (`ModelSwitchScheduler`).

docker-compose.yml

@@ -35,11 +35,10 @@ services:
   cloudflared:
     image: cloudflare/cloudflared:latest
     restart: unless-stopped
-    command: tunnel --config /etc/cloudflared/config.yml run
+    command: tunnel run
+    environment:
+      TUNNEL_TOKEN: ${CLOUDFLARE_TUNNEL_TOKEN}
     networks: [internal]
-    volumes:
-      - ./infra/cloudflare/config.yml:/etc/cloudflared/config.yml:ro
-      - ./infra/cloudflare/credentials.json:/etc/cloudflared/credentials.json:ro
     depends_on:
       - caddy
 
@@ -66,7 +65,7 @@ services:
       ASPNETCORE_ENVIRONMENT: Production
       ASPNETCORE_URLS: http://+:8080
       ConnectionStrings__Default: "Host=manager-db;Port=5432;Database=amtocbots;Username=amtocbots;Password=${MANAGER_DB_PASSWORD}"
-      Redis__ConnectionString: "redis:6379"
+      Redis__ConnectionString: "redis:6379,password=${REDIS_PASSWORD}"
       Keycloak__Authority: "https://auth.amtocbot.com/realms/amtocbots"
       Keycloak__Audience: "amtocbots-api"
       Docker__SocketPath: "/var/run/docker.sock"

infra/caddy/Caddyfile

@@ -1,8 +1,8 @@
 # AmtocBots Caddy reverse proxy configuration
-# Caddy automatically provisions TLS certificates via Let's Encrypt
+# TLS is terminated by Cloudflare tunnel — Caddy serves plain HTTP internally
 
 # ── Manager PWA ──────────────────────────────────────────────────────────────
-manager.amtocbot.com {
+http://manager.amtocbot.com {
     # Angular PWA — static files
     reverse_proxy /api/* manager-api:8080
     reverse_proxy /hubs/* manager-api:8080 {
@@ -14,7 +14,7 @@ manager.amtocbot.com {
 }
 
 # ── Keycloak (auth) ───────────────────────────────────────────────────────────
-auth.amtocbot.com {
+http://auth.amtocbot.com {
     reverse_proxy keycloak:8080 {
         header_up X-Forwarded-Proto {scheme}
         header_up X-Forwarded-Host {host}

infra/keycloak/Dockerfile

@@ -1,18 +1,15 @@
-FROM quay.io/keycloak/keycloak:26 AS builder
+FROM alpine:3.20 AS downloader
+ARG MAGIC_LINK_VERSION=0.57
+RUN wget -qO /keycloak-magic-link.jar \
+    "https://repo1.maven.org/maven2/io/phasetwo/keycloak/keycloak-magic-link/${MAGIC_LINK_VERSION}/keycloak-magic-link-${MAGIC_LINK_VERSION}.jar"
 
-# Download the magic-link extension (p2-inc/keycloak-magic-link)
-# Check https://github.com/p2-inc/keycloak-magic-link/releases for latest version
-ARG MAGIC_LINK_VERSION=2.1.0
-ADD https://github.com/p2-inc/keycloak-magic-link/releases/download/${MAGIC_LINK_VERSION}/keycloak-magic-link-${MAGIC_LINK_VERSION}.jar \
-    /opt/keycloak/providers/keycloak-magic-link.jar
-
-# Build optimized Keycloak image
+FROM quay.io/keycloak/keycloak:26.2 AS builder
+COPY --from=downloader /keycloak-magic-link.jar /opt/keycloak/providers/keycloak-magic-link.jar
 RUN /opt/keycloak/bin/kc.sh build \
     --db=postgres \
     --health-enabled=true \
-    --features=passkeys,magic-link
+    --features=passkeys
 
-FROM quay.io/keycloak/keycloak:26
+FROM quay.io/keycloak/keycloak:26.2
 COPY --from=builder /opt/keycloak /opt/keycloak
-
 ENTRYPOINT ["/opt/keycloak/bin/kc.sh"]

infra/keycloak/realm-export.json

@@ -148,14 +148,14 @@
     "cibaInterval": "5",
     "parRequestUriLifespan": "60",
     "webAuthnPolicyRpEntityName": "AmtocBots",
-    "webAuthnPolicySignatureAlgorithms": ["ES256"],
+    "webAuthnPolicySignatureAlgorithms": "ES256",
     "webAuthnPolicyRpId": "amtocbot.com",
     "webAuthnPolicyAttestationConveyancePreference": "not specified",
     "webAuthnPolicyAuthenticatorAttachment": "not specified",
     "webAuthnPolicyRequireResidentKey": "not specified",
     "webAuthnPolicyUserVerificationRequirement": "not specified",
     "webAuthnPolicyPasswordlessRpEntityName": "AmtocBots",
-    "webAuthnPolicyPasswordlessSignatureAlgorithms": ["ES256"],
+    "webAuthnPolicyPasswordlessSignatureAlgorithms": "ES256",
     "webAuthnPolicyPasswordlessRpId": "amtocbot.com",
     "webAuthnPolicyPasswordlessUserVerificationRequirement": "required"
   }

src/AmtocBots.Api/AmtocBots.Api.csproj

@@ -0,0 +1,40 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <RootNamespace>AmtocBots.Api</RootNamespace>
+    <AssemblyName>AmtocBots.Api</AssemblyName>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <!-- EF Core + PostgreSQL + pgvector -->
+    <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="9.0.*" />
+    <PackageReference Include="Pgvector.EntityFrameworkCore" Version="0.2.*" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.*">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+    </PackageReference>
+
+    <!-- Auth -->
+    <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.*" />
+
+    <!-- Docker management -->
+    <PackageReference Include="Docker.DotNet" Version="3.125.*" />
+
+    <!-- Redis -->
+    <PackageReference Include="Microsoft.Extensions.Caching.StackExchangeRedis" Version="10.0.*" />
+    <PackageReference Include="StackExchange.Redis" Version="2.8.*" />
+
+    <!-- Cron scheduling -->
+    <PackageReference Include="NCrontab" Version="3.3.*" />
+
+    <!-- Password/token hashing -->
+    <PackageReference Include="BCrypt.Net-Next" Version="4.0.*" />
+
+    <!-- JSON5 support for OpenClaw configs -->
+    <PackageReference Include="Hjson" Version="3.0.*" />
+  </ItemGroup>
+
+</Project>

src/AmtocBots.Api/BackgroundServices/MetricsPollingService.cs

@@ -0,0 +1,72 @@
+using AmtocBots.Api.Data;
+using AmtocBots.Api.Hubs;
+using AmtocBots.Api.Services.Docker;
+using Microsoft.AspNetCore.SignalR;
+using Microsoft.EntityFrameworkCore;
+
+namespace AmtocBots.Api.BackgroundServices;
+
+public sealed class MetricsPollingService(
+    IServiceScopeFactory scopeFactory,
+    IHubContext<InstanceHub> hub,
+    IDockerService docker,
+    ILogger<MetricsPollingService> log) : BackgroundService
+{
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        log.LogInformation("Metrics polling service started");
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await PollAndBroadcastAsync(stoppingToken);
+            }
+            catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
+            {
+                log.LogWarning(ex, "Error during metrics polling");
+            }
+
+            await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken);
+        }
+    }
+
+    private async Task PollAndBroadcastAsync(CancellationToken ct)
+    {
+        using var scope = scopeFactory.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+
+        var running = await db.Instances
+            .Where(i => i.ContainerId != null && i.Status == "running")
+            .Select(i => new { i.Id, ContainerId = i.ContainerId! })
+            .ToListAsync(ct);
+
+        if (running.Count == 0) return;
+
+        var stats = await docker.GetAllManagedStatsAsync(
+            running.Select(r => (r.Id, r.ContainerId)), ct);
+
+        foreach (var stat in stats)
+        {
+            await hub.Clients.Group($"instance:{stat.InstanceId}")
+                .SendAsync("StatusUpdate", stat, ct);
+        }
+
+        // Sync status changes back to DB
+        var statusMap = stats.ToDictionary(s => s.InstanceId, s => s.Status);
+        foreach (var inst in running)
+        {
+            if (statusMap.TryGetValue(inst.Id, out var newStatus))
+            {
+                var entity = await db.Instances.FindAsync([inst.Id], ct);
+                if (entity is not null && entity.Status != newStatus)
+                {
+                    entity.Status = newStatus;
+                    entity.UpdatedAt = DateTimeOffset.UtcNow;
+                }
+            }
+        }
+
+        await db.SaveChangesAsync(ct);
+    }
+}

src/AmtocBots.Api/BackgroundServices/ModelSwitchScheduler.cs

@@ -0,0 +1,69 @@
+using AmtocBots.Api.Data;
+using AmtocBots.Api.Services.Models;
+using Microsoft.EntityFrameworkCore;
+using NCrontab;
+
+namespace AmtocBots.Api.BackgroundServices;
+
+public sealed class ModelSwitchScheduler(
+    IServiceScopeFactory scopeFactory,
+    ILogger<ModelSwitchScheduler> log) : BackgroundService
+{
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        log.LogInformation("Model switch scheduler started");
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            try
+            {
+                await EvaluateAsync(stoppingToken);
+            }
+            catch (Exception ex) when (!stoppingToken.IsCancellationRequested)
+            {
+                log.LogWarning(ex, "Error in model switch scheduler");
+            }
+
+            await Task.Delay(TimeSpan.FromMinutes(1), stoppingToken);
+        }
+    }
+
+    private async Task EvaluateAsync(CancellationToken ct)
+    {
+        using var scope = scopeFactory.CreateScope();
+        var db = scope.ServiceProvider.GetRequiredService<AppDbContext>();
+        var switcher = scope.ServiceProvider.GetRequiredService<IModelSwitchingService>();
+
+        var now = DateTime.UtcNow;
+        var cronRules = await db.SwitchRules
+            .Where(r => r.IsActive && r.RuleType == "cron" && r.CronExpression != null)
+            .ToListAsync(ct);
+
+        foreach (var rule in cronRules)
+        {
+            try
+            {
+                var schedule = CrontabSchedule.Parse(rule.CronExpression!, new CrontabSchedule.ParseOptions { IncludingSeconds = false });
+                var next = schedule.GetNextOccurrence(now.AddMinutes(-1));
+                if (next <= now)
+                {
+                    log.LogInformation("Cron rule triggered for instance {Id}: switch to {Model}", rule.InstanceId, rule.TargetModel);
+                    await switcher.SwitchModelAsync(rule.InstanceId, rule.TargetModel, ct);
+                }
+            }
+            catch (Exception ex)
+            {
+                log.LogWarning(ex, "Failed to evaluate cron rule {Id}", rule.Id);
+            }
+        }
+
+        // Also evaluate threshold rules for all running instances
+        var runningIds = await db.Instances
+            .Where(i => i.Status == "running")
+            .Select(i => i.Id)
+            .ToListAsync(ct);
+
+        foreach (var id in runningIds)
+            await switcher.EvaluateThresholdRulesAsync(id, ct);
+    }
+}

src/AmtocBots.Api/BackgroundServices/QueueRetryWorker.cs

@@ -0,0 +1,56 @@
+using AmtocBots.Api.Services.OpenClaw;
+using AmtocBots.Api.Services.Queue;
+
+namespace AmtocBots.Api.BackgroundServices;
+
+public sealed class QueueRetryWorker(
+    RedisMessageQueueService queue,
+    IOpenClawClient openClaw,
+    ILogger<QueueRetryWorker> log) : BackgroundService
+{
+    private static readonly TimeSpan[] Backoffs = [
+        TimeSpan.FromSeconds(5),
+        TimeSpan.FromSeconds(15),
+        TimeSpan.FromSeconds(30),
+        TimeSpan.FromMinutes(2),
+        TimeSpan.FromMinutes(5),
+    ];
+
+    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
+    {
+        log.LogInformation("Queue retry worker started");
+
+        while (!stoppingToken.IsCancellationRequested)
+        {
+            // Move any ready delayed messages back to main queue
+            await queue.FlushReadyDelayedAsync(stoppingToken);
+
+            var msg = await queue.DequeueAsync(stoppingToken);
+            if (msg is null)
+            {
+                await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
+                continue;
+            }
+
+            try
+            {
+                await openClaw.RunAgentAsync(msg.BaseUrl, msg.BearerToken,
+                    new AgentRequest(msg.Description, msg.Model), stoppingToken);
+                log.LogDebug("Retried queued message for instance {Id}", msg.InstanceId);
+            }
+            catch (HttpRequestException ex) when ((int?)ex.StatusCode == 429)
+            {
+                var delay = msg.RetryCount < Backoffs.Length
+                    ? Backoffs[msg.RetryCount]
+                    : TimeSpan.FromMinutes(10);
+
+                log.LogWarning("Rate limited on instance {Id}, requeuing with {Delay}s delay", msg.InstanceId, delay.TotalSeconds);
+                await queue.RequeueWithDelayAsync(msg, delay, stoppingToken);
+            }
+            catch (Exception ex)
+            {
+                log.LogError(ex, "Failed to send queued message for instance {Id}", msg.InstanceId);
+            }
+        }
+    }
+}

src/AmtocBots.Api/Configuration/DockerOptions.cs

@@ -0,0 +1,10 @@
+namespace AmtocBots.Api.Configuration;
+
+public sealed class DockerOptions
+{
+    public string SocketPath { get; set; } = "/var/run/docker.sock";
+    public string OpenClawNetwork { get; set; } = "amtocbots_openclaw";
+    public string OpenClawImage { get; set; } = "ghcr.io/openclaw/openclaw:latest";
+    public int PortRangeStart { get; set; } = 18789;
+    public int PortRangeEnd { get; set; } = 19789;
+}

src/AmtocBots.Api/Configuration/EncryptionOptions.cs

@@ -0,0 +1,7 @@
+namespace AmtocBots.Api.Configuration;
+
+public sealed class EncryptionOptions
+{
+    /// <summary>Base64-encoded 32-byte AES-256 key. Generate with: openssl rand -base64 32</summary>
+    public string Key { get; set; } = string.Empty;
+}