diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
index e23838246..deeec6418 100644
--- a/.devcontainer/Dockerfile
+++ b/.devcontainer/Dockerfile
@@ -4,3 +4,9 @@ FROM mcr.microsoft.com/devcontainers/dotnet:1-9.0-bookworm
RUN rm -f /etc/apt/sources.list.d/yarn.list 2>/dev/null; \
rm -f /etc/apt/keyrings/yarn.gpg 2>/dev/null; \
apt-get update || true
+
+# Install socat for Keycloak port-forwarding bridge.
+# Docker-in-Docker proxy ports aren't auto-forwarded by VS Code Dev Containers;
+# socat provides a userspace bridge that VS Code detects and forwards to the host.
+RUN apt-get install -y --no-install-recommends socat \
+ && rm -rf /var/lib/apt/lists/*
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 989596249..7025d1cdb 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -1,7 +1,7 @@
{
"name": "JIM Development Environment",
"build": {
- "dockerfile": "Dockerfile"
+ "dockerfile": "Dockerfile"
},
// Features - pre-built development tools
"features": {
@@ -103,7 +103,8 @@
"forwardPorts": [
5432, // PostgreSQL
5200, // JIM.Web (HTTP) - also serves API at /api/
- 5201 // JIM.Web (HTTPS) - also serves API at /api/
+ 5201, // JIM.Web (HTTPS) - also serves API at /api/
+ 8181 // Keycloak IdP (OIDC + admin console)
],
// Port labels and auto-forward behaviour
"portsAttributes": {
@@ -118,6 +119,10 @@
"5201": {
"label": "JIM Web + API (HTTPS)",
"onAutoForward": "silent"
+ },
+ "8181": {
+ "label": "Keycloak IdP",
+ "onAutoForward": "silent"
}
},
// Container user
@@ -140,10 +145,10 @@
"capAdd": [
"SYS_PTRACE"
],
- // Improved build performance
+ // Improved build performance and integration test directory snapshot storage
"hostRequirements": {
"cpus": 8,
"memory": "32gb",
- "storage": "64gb"
+ "storage": "120gb"
}
}
diff --git a/.devcontainer/jim-aliases.sh b/.devcontainer/jim-aliases.sh
index b81dec3e7..403dfd50c 100644
--- a/.devcontainer/jim-aliases.sh
+++ b/.devcontainer/jim-aliases.sh
@@ -26,6 +26,11 @@ Database Management:
jim-db-logs - View database logs
jim-postgres-tune - Auto-tune PostgreSQL for current devcontainer specs
+Identity Provider (Keycloak):
+ jim-keycloak - Start Keycloak IdP (for local F5 debugging)
+ jim-keycloak-stop - Stop Keycloak IdP
+ jim-keycloak-logs - View Keycloak logs
+
Docker Stack Management (auto-kills local JIM processes):
jim-stack - Start Docker stack
jim-stack-logs - View Docker stack logs
@@ -168,6 +173,32 @@ jim-db-logs() {
docker compose $(_jim_db_compose) logs -f
}
+# Standalone Keycloak IdP (local debugging workflow - use with jim-db + F5)
+# Starts the bundled Keycloak from docker-compose.override.yml without the full stack.
+jim-keycloak() {
+ docker compose -f docker-compose.yml -f docker-compose.override.yml up -d jim.keycloak
+ _jim_keycloak_bridge
+}
+jim-keycloak-stop() {
+ pkill -f 'socat.*TCP:127.0.0.1:8180' 2>/dev/null || true
+ docker compose -f docker-compose.yml -f docker-compose.override.yml stop jim.keycloak
+ docker compose -f docker-compose.yml -f docker-compose.override.yml rm -f jim.keycloak
+}
+jim-keycloak-logs() {
+ docker compose -f docker-compose.yml -f docker-compose.override.yml logs -f jim.keycloak
+}
+
+# Start a userspace port forwarder for Keycloak so VS Code can forward it.
+# Docker-in-Docker proxy ports aren't forwarded by VS Code Dev Containers
+# unless they were present at devcontainer build time. This socat bridge
+# runs as the vscode user, which VS Code detects and forwards to the host.
+_jim_keycloak_bridge() {
+ pkill -f 'socat.*TCP:127.0.0.1:8180' 2>/dev/null || true
+ if command -v socat &>/dev/null; then
+ socat TCP-LISTEN:8181,fork,reuseaddr,bind=0.0.0.0 TCP:127.0.0.1:8180 &
+ fi
+}
+
# Kill any locally-running JIM .NET processes (jim-web, jim-worker, jim-scheduler)
# so they don't hold ports that Docker containers need to bind
_jim_kill_local() {
@@ -181,12 +212,13 @@ _jim_kill_local() {
}
# Clear any previous aliases before defining functions (zsh cannot redefine alias as function)
-unalias jim-stack jim-stack-logs jim-stack-down jim-restart jim-build jim-build-web jim-build-worker jim-build-scheduler jim-cleanup jim-reset jim-db jim-db-stop jim-db-logs 2>/dev/null || true
+unalias jim-stack jim-stack-logs jim-stack-down jim-restart jim-build jim-build-web jim-build-worker jim-build-scheduler jim-cleanup jim-reset jim-db jim-db-stop jim-db-logs jim-keycloak jim-keycloak-stop jim-keycloak-logs 2>/dev/null || true
# Docker stack management
jim-stack() {
_jim_kill_local
docker compose $(_jim_compose) up -d
+ _jim_keycloak_bridge
}
jim-stack-logs() {
docker compose $(_jim_compose) logs -f
@@ -199,6 +231,7 @@ jim-stack-down() {
jim-restart() {
_jim_kill_local
docker compose $(_jim_compose) down && docker compose $(_jim_compose) up -d --force-recreate
+ _jim_keycloak_bridge
}
# Docker builds (rebuild and start services)
@@ -209,6 +242,7 @@ _jim_version_suffix() {
jim-build() {
_jim_kill_local
VERSION_SUFFIX="$(_jim_version_suffix)" docker compose $(_jim_compose) up -d --build
+ _jim_keycloak_bridge
}
jim-build-web() {
_jim_kill_local
@@ -244,15 +278,15 @@ jim-cleanup() {
df -h / | tail -1
}
-# Reset (preserves Samba AD snapshot images — they take a long time to build)
+# Reset (preserves Samba AD and OpenLDAP snapshot images — they take a long time to build)
jim-reset() {
docker compose $(_jim_compose) down --volumes
docker compose -f docker-compose.integration-tests.yml --profile scenario2 --profile scenario8 down --volumes --remove-orphans 2>/dev/null || true
docker rm -f samba-ad-primary samba-ad-source samba-ad-target sqlserver-hris-a oracle-hris-b postgres-target openldap-test mysql-test 2>/dev/null || true
- docker image prune -af --filter "label!=jim.samba.snapshot-hash" --filter "label!=jim.samba.build-hash" 2>/dev/null || true
+ docker image prune -af --filter "label!=jim.samba.snapshot-hash" --filter "label!=jim.samba.build-hash" --filter "label!=jim.openldap.snapshot-hash" --filter "label!=jim.openldap.build-hash" 2>/dev/null || true
docker volume ls --format "{{.Name}}" | grep jim-integration | xargs -r docker volume rm 2>/dev/null || true
docker volume rm -f jim-db-volume jim-logs-volume 2>/dev/null || true
- echo "JIM reset complete. Containers, images, and volumes removed (snapshots preserved). Run jim-build to rebuild."
+ echo "JIM reset complete. Containers, images, and volumes removed (Samba AD & OpenLDAP snapshots preserved). Run jim-build to rebuild."
}
# Structurizr diagram export
@@ -281,19 +315,20 @@ jim-diagrams() {
# Remove any stale container
docker rm -f "${container_name}" 2>/dev/null
- # Start Structurizr Lite (adrs mount resolves the symlink inside the container)
- echo "Starting Structurizr Lite on port ${port}..."
+ # Start Structurizr Local (adrs mount resolves the symlink inside the container)
+ # Note: structurizr/lite is deprecated; using structurizr/structurizr with 'local' command
+ echo "Starting Structurizr Local on port ${port}..."
docker run -d --name "${container_name}" \
-p "${port}:8080" \
-v "${structurizr_dir}:/usr/local/structurizr" \
-v "${repo_root}/docs/adrs:/usr/local/structurizr/adrs" \
- structurizr/lite > /dev/null
+ structurizr/structurizr local > /dev/null
- # Wait for Structurizr Lite to be ready
- echo "Waiting for Structurizr Lite to start..."
+ # Wait for Structurizr Local to be ready
+ echo "Waiting for Structurizr Local to start..."
local attempts=0
while [ $attempts -lt 30 ]; do
- if curl -sf "http://localhost:${port}/workspace/diagrams" > /dev/null 2>&1; then
+ if curl -sf "http://localhost:${port}/workspace/1/diagrams" > /dev/null 2>&1; then
break
fi
attempts=$((attempts + 1))
@@ -301,12 +336,12 @@ jim-diagrams() {
done
if [ $attempts -ge 30 ]; then
- echo "ERROR: Structurizr Lite failed to start within 60 seconds."
+ echo "ERROR: Structurizr Local failed to start within 60 seconds."
docker rm -f "${container_name}" > /dev/null 2>&1
return 1
fi
- echo "Structurizr Lite is ready."
+ echo "Structurizr Local is ready."
# Remove old images (light, dark, and legacy root-level)
rm -f "${repo_root}/docs/diagrams/images"/jim-structurizr-1-*.svg
@@ -315,12 +350,12 @@ jim-diagrams() {
# Export diagrams (light + dark)
node "${structurizr_dir}/export-diagrams.js" \
- "http://localhost:${port}/workspace/diagrams" \
+ "http://localhost:${port}/workspace/1/diagrams" \
"${repo_root}/docs/diagrams/images"
local export_rc=$?
# Cleanup
- echo "Stopping Structurizr Lite..."
+ echo "Stopping Structurizr Local..."
docker rm -f "${container_name}" > /dev/null 2>&1
if [ $export_rc -eq 0 ]; then
diff --git a/.devcontainer/keycloak/jim-realm.json b/.devcontainer/keycloak/jim-realm.json
new file mode 100644
index 000000000..981c7762c
--- /dev/null
+++ b/.devcontainer/keycloak/jim-realm.json
@@ -0,0 +1,2011 @@
+{
+ "realm": "jim",
+ "displayName": "JIM Development",
+ "notBefore": 0,
+ "defaultSignatureAlgorithm": "RS256",
+ "revokeRefreshToken": false,
+ "refreshTokenMaxReuse": 0,
+ "accessTokenLifespan": 300,
+ "accessTokenLifespanForImplicitFlow": 900,
+ "ssoSessionIdleTimeout": 1800,
+ "ssoSessionMaxLifespan": 36000,
+ "ssoSessionIdleTimeoutRememberMe": 0,
+ "ssoSessionMaxLifespanRememberMe": 0,
+ "offlineSessionIdleTimeout": 2592000,
+ "offlineSessionMaxLifespanEnabled": false,
+ "offlineSessionMaxLifespan": 5184000,
+ "clientSessionIdleTimeout": 0,
+ "clientSessionMaxLifespan": 0,
+ "clientOfflineSessionIdleTimeout": 0,
+ "clientOfflineSessionMaxLifespan": 0,
+ "accessCodeLifespan": 60,
+ "accessCodeLifespanUserAction": 300,
+ "accessCodeLifespanLogin": 1800,
+ "actionTokenGeneratedByAdminLifespan": 43200,
+ "actionTokenGeneratedByUserLifespan": 300,
+ "oauth2DeviceCodeLifespan": 600,
+ "oauth2DevicePollingInterval": 5,
+ "enabled": true,
+ "sslRequired": "none",
+ "registrationAllowed": false,
+ "registrationEmailAsUsername": false,
+ "rememberMe": false,
+ "verifyEmail": false,
+ "loginWithEmailAllowed": true,
+ "duplicateEmailsAllowed": false,
+ "resetPasswordAllowed": false,
+ "editUsernameAllowed": false,
+ "bruteForceProtected": false,
+ "permanentLockout": false,
+ "maxTemporaryLockouts": 0,
+ "bruteForceStrategy": "MULTIPLE",
+ "maxFailureWaitSeconds": 900,
+ "minimumQuickLoginWaitSeconds": 60,
+ "waitIncrementSeconds": 60,
+ "quickLoginCheckMilliSeconds": 1000,
+ "maxDeltaTimeSeconds": 43200,
+ "failureFactor": 30,
+ "groups": [],
+ "defaultRole": {
+ "name": "default-roles-jim",
+ "description": "${role_default-roles}",
+ "composite": true,
+ "clientRole": false,
+ "containerId": "28efb455-97d1-4c7c-bd33-c1eefd0e3810"
+ },
+ "requiredCredentials": [
+ "password"
+ ],
+ "otpPolicyType": "totp",
+ "otpPolicyAlgorithm": "HmacSHA1",
+ "otpPolicyInitialCounter": 0,
+ "otpPolicyDigits": 6,
+ "otpPolicyLookAheadWindow": 1,
+ "otpPolicyPeriod": 30,
+ "otpPolicyCodeReusable": false,
+ "otpSupportedApplications": [
+ "totpAppFreeOTPName",
+ "totpAppGoogleName",
+ "totpAppMicrosoftAuthenticatorName"
+ ],
+ "localizationTexts": {},
+ "webAuthnPolicyRpEntityName": "keycloak",
+ "webAuthnPolicySignatureAlgorithms": [
+ "ES256",
+ "RS256"
+ ],
+ "webAuthnPolicyRpId": "",
+ "webAuthnPolicyAttestationConveyancePreference": "not specified",
+ "webAuthnPolicyAuthenticatorAttachment": "not specified",
+ "webAuthnPolicyRequireResidentKey": "not specified",
+ "webAuthnPolicyUserVerificationRequirement": "not specified",
+ "webAuthnPolicyCreateTimeout": 0,
+ "webAuthnPolicyAvoidSameAuthenticatorRegister": false,
+ "webAuthnPolicyAcceptableAaguids": [],
+ "webAuthnPolicyExtraOrigins": [],
+ "webAuthnPolicyPasswordlessRpEntityName": "keycloak",
+ "webAuthnPolicyPasswordlessSignatureAlgorithms": [
+ "ES256",
+ "RS256"
+ ],
+ "webAuthnPolicyPasswordlessRpId": "",
+ "webAuthnPolicyPasswordlessAttestationConveyancePreference": "not specified",
+ "webAuthnPolicyPasswordlessAuthenticatorAttachment": "not specified",
+ "webAuthnPolicyPasswordlessRequireResidentKey": "not specified",
+ "webAuthnPolicyPasswordlessUserVerificationRequirement": "not specified",
+ "webAuthnPolicyPasswordlessCreateTimeout": 0,
+ "webAuthnPolicyPasswordlessAvoidSameAuthenticatorRegister": false,
+ "webAuthnPolicyPasswordlessAcceptableAaguids": [],
+ "webAuthnPolicyPasswordlessExtraOrigins": [],
+ "scopeMappings": [
+ {
+ "clientScope": "offline_access",
+ "roles": [
+ "offline_access"
+ ]
+ }
+ ],
+ "clientScopeMappings": {
+ "account": [
+ {
+ "client": "account-console",
+ "roles": [
+ "manage-account",
+ "view-groups"
+ ]
+ }
+ ]
+ },
+ "clients": [
+ {
+ "clientId": "account",
+ "name": "${client_account}",
+ "rootUrl": "${authBaseUrl}",
+ "baseUrl": "/realms/jim/account/",
+ "surrogateAuthRequired": false,
+ "enabled": true,
+ "alwaysDisplayInConsole": false,
+ "clientAuthenticatorType": "client-secret",
+ "redirectUris": [
+ "/realms/jim/account/*"
+ ],
+ "webOrigins": [],
+ "notBefore": 0,
+ "bearerOnly": false,
+ "consentRequired": false,
+ "standardFlowEnabled": true,
+ "implicitFlowEnabled": false,
+ "directAccessGrantsEnabled": false,
+ "serviceAccountsEnabled": false,
+ "publicClient": true,
+ "frontchannelLogout": false,
+ "protocol": "openid-connect",
+ "attributes": {
+ "realm_client": "false",
+ "post.logout.redirect.uris": "+"
+ },
+ "authenticationFlowBindingOverrides": {},
+ "fullScopeAllowed": false,
+ "nodeReRegistrationTimeout": 0,
+ "defaultClientScopes": [
+ "web-origins",
+ "acr",
+ "profile",
+ "roles",
+ "basic",
+ "email"
+ ],
+ "optionalClientScopes": [
+ "address",
+ "phone",
+ "offline_access",
+ "microprofile-jwt"
+ ]
+ },
+ {
+ "clientId": "account-console",
+ "name": "${client_account-console}",
+ "rootUrl": "${authBaseUrl}",
+ "baseUrl": "/realms/jim/account/",
+ "surrogateAuthRequired": false,
+ "enabled": true,
+ "alwaysDisplayInConsole": false,
+ "clientAuthenticatorType": "client-secret",
+ "redirectUris": [
+ "/realms/jim/account/*"
+ ],
+ "webOrigins": [],
+ "notBefore": 0,
+ "bearerOnly": false,
+ "consentRequired": false,
+ "standardFlowEnabled": true,
+ "implicitFlowEnabled": false,
+ "directAccessGrantsEnabled": false,
+ "serviceAccountsEnabled": false,
+ "publicClient": true,
+ "frontchannelLogout": false,
+ "protocol": "openid-connect",
+ "attributes": {
+ "realm_client": "false",
+ "post.logout.redirect.uris": "+",
+ "pkce.code.challenge.method": "S256"
+ },
+ "authenticationFlowBindingOverrides": {},
+ "fullScopeAllowed": false,
+ "nodeReRegistrationTimeout": 0,
+ "protocolMappers": [
+ {
+ "name": "audience resolve",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-audience-resolve-mapper",
+ "consentRequired": false,
+ "config": {}
+ }
+ ],
+ "defaultClientScopes": [
+ "web-origins",
+ "acr",
+ "profile",
+ "roles",
+ "basic",
+ "email"
+ ],
+ "optionalClientScopes": [
+ "address",
+ "phone",
+ "offline_access",
+ "microprofile-jwt"
+ ]
+ },
+ {
+ "clientId": "admin-cli",
+ "name": "${client_admin-cli}",
+ "surrogateAuthRequired": false,
+ "enabled": true,
+ "alwaysDisplayInConsole": false,
+ "clientAuthenticatorType": "client-secret",
+ "redirectUris": [],
+ "webOrigins": [],
+ "notBefore": 0,
+ "bearerOnly": false,
+ "consentRequired": false,
+ "standardFlowEnabled": false,
+ "implicitFlowEnabled": false,
+ "directAccessGrantsEnabled": true,
+ "serviceAccountsEnabled": false,
+ "publicClient": true,
+ "frontchannelLogout": false,
+ "protocol": "openid-connect",
+ "attributes": {
+ "realm_client": "false",
+ "client.use.lightweight.access.token.enabled": "true"
+ },
+ "authenticationFlowBindingOverrides": {},
+ "fullScopeAllowed": true,
+ "nodeReRegistrationTimeout": 0,
+ "defaultClientScopes": [
+ "web-origins",
+ "acr",
+ "profile",
+ "roles",
+ "basic",
+ "email"
+ ],
+ "optionalClientScopes": [
+ "address",
+ "phone",
+ "offline_access",
+ "microprofile-jwt"
+ ]
+ },
+ {
+ "clientId": "broker",
+ "name": "${client_broker}",
+ "surrogateAuthRequired": false,
+ "enabled": true,
+ "alwaysDisplayInConsole": false,
+ "clientAuthenticatorType": "client-secret",
+ "redirectUris": [],
+ "webOrigins": [],
+ "notBefore": 0,
+ "bearerOnly": true,
+ "consentRequired": false,
+ "standardFlowEnabled": true,
+ "implicitFlowEnabled": false,
+ "directAccessGrantsEnabled": false,
+ "serviceAccountsEnabled": false,
+ "publicClient": false,
+ "frontchannelLogout": false,
+ "protocol": "openid-connect",
+ "attributes": {
+ "realm_client": "true"
+ },
+ "authenticationFlowBindingOverrides": {},
+ "fullScopeAllowed": false,
+ "nodeReRegistrationTimeout": 0,
+ "defaultClientScopes": [
+ "web-origins",
+ "acr",
+ "profile",
+ "roles",
+ "basic",
+ "email"
+ ],
+ "optionalClientScopes": [
+ "address",
+ "phone",
+ "offline_access",
+ "microprofile-jwt"
+ ]
+ },
+ {
+ "clientId": "jim-powershell",
+ "name": "JIM PowerShell Module",
+ "description": "Public client for JIM PowerShell module — uses PKCE with loopback redirect",
+ "surrogateAuthRequired": false,
+ "enabled": true,
+ "alwaysDisplayInConsole": false,
+ "clientAuthenticatorType": "client-secret",
+ "redirectUris": [
+ "http://localhost:8408/*",
+ "http://localhost:8409/*",
+ "http://localhost:8402/*",
+ "http://localhost:8403/*",
+ "http://localhost:8400/*",
+ "http://localhost:8401/*",
+ "http://localhost:8406/*",
+ "http://localhost:8407/*",
+ "http://localhost:8404/*",
+ "http://localhost:8405/*"
+ ],
+ "webOrigins": [
+ "+"
+ ],
+ "notBefore": 0,
+ "bearerOnly": false,
+ "consentRequired": false,
+ "standardFlowEnabled": true,
+ "implicitFlowEnabled": false,
+ "directAccessGrantsEnabled": false,
+ "serviceAccountsEnabled": false,
+ "publicClient": true,
+ "frontchannelLogout": false,
+ "protocol": "openid-connect",
+ "attributes": {
+ "realm_client": "false",
+ "backchannel.logout.session.required": "true",
+ "pkce.code.challenge.method": "S256",
+ "backchannel.logout.revoke.offline.tokens": "false"
+ },
+ "authenticationFlowBindingOverrides": {},
+ "fullScopeAllowed": true,
+ "nodeReRegistrationTimeout": -1,
+ "defaultClientScopes": [
+ "web-origins",
+ "acr",
+ "profile",
+ "roles",
+ "basic",
+ "email"
+ ],
+ "optionalClientScopes": [
+ "address",
+ "phone",
+ "offline_access",
+ "microprofile-jwt",
+ "jim-api"
+ ]
+ },
+ {
+ "clientId": "jim-web",
+ "name": "JIM Web Application",
+ "description": "JIM web UI and API — confidential client with PKCE",
+ "surrogateAuthRequired": false,
+ "enabled": true,
+ "alwaysDisplayInConsole": false,
+ "clientAuthenticatorType": "client-secret",
+ "secret": "jim-dev-secret",
+ "redirectUris": [
+ "https://localhost:7000/*",
+ "http://localhost:5200/*"
+ ],
+ "webOrigins": [
+ "https://localhost:7000",
+ "http://localhost:5200"
+ ],
+ "notBefore": 0,
+ "bearerOnly": false,
+ "consentRequired": false,
+ "standardFlowEnabled": true,
+ "implicitFlowEnabled": false,
+ "directAccessGrantsEnabled": false,
+ "serviceAccountsEnabled": false,
+ "publicClient": false,
+ "frontchannelLogout": false,
+ "protocol": "openid-connect",
+ "attributes": {
+ "realm_client": "false",
+ "backchannel.logout.session.required": "true",
+ "post.logout.redirect.uris": "http://localhost:5200/*##https://localhost:7000/*",
+ "pkce.code.challenge.method": "S256",
+ "backchannel.logout.revoke.offline.tokens": "false"
+ },
+ "authenticationFlowBindingOverrides": {},
+ "fullScopeAllowed": true,
+ "nodeReRegistrationTimeout": -1,
+ "defaultClientScopes": [
+ "web-origins",
+ "acr",
+ "profile",
+ "roles",
+ "basic",
+ "email"
+ ],
+ "optionalClientScopes": [
+ "address",
+ "phone",
+ "offline_access",
+ "microprofile-jwt",
+ "jim-api"
+ ]
+ },
+ {
+ "clientId": "realm-management",
+ "name": "${client_realm-management}",
+ "surrogateAuthRequired": false,
+ "enabled": true,
+ "alwaysDisplayInConsole": false,
+ "clientAuthenticatorType": "client-secret",
+ "redirectUris": [],
+ "webOrigins": [],
+ "notBefore": 0,
+ "bearerOnly": true,
+ "consentRequired": false,
+ "standardFlowEnabled": true,
+ "implicitFlowEnabled": false,
+ "directAccessGrantsEnabled": false,
+ "serviceAccountsEnabled": false,
+ "publicClient": false,
+ "frontchannelLogout": false,
+ "protocol": "openid-connect",
+ "attributes": {
+ "realm_client": "true"
+ },
+ "authenticationFlowBindingOverrides": {},
+ "fullScopeAllowed": false,
+ "nodeReRegistrationTimeout": 0,
+ "defaultClientScopes": [
+ "web-origins",
+ "acr",
+ "profile",
+ "roles",
+ "basic",
+ "email"
+ ],
+ "optionalClientScopes": [
+ "address",
+ "phone",
+ "offline_access",
+ "microprofile-jwt"
+ ]
+ },
+ {
+ "clientId": "security-admin-console",
+ "name": "${client_security-admin-console}",
+ "rootUrl": "${authAdminUrl}",
+ "baseUrl": "/admin/jim/console/",
+ "surrogateAuthRequired": false,
+ "enabled": true,
+ "alwaysDisplayInConsole": false,
+ "clientAuthenticatorType": "client-secret",
+ "redirectUris": [
+ "/admin/jim/console/*"
+ ],
+ "webOrigins": [
+ "+"
+ ],
+ "notBefore": 0,
+ "bearerOnly": false,
+ "consentRequired": false,
+ "standardFlowEnabled": true,
+ "implicitFlowEnabled": false,
+ "directAccessGrantsEnabled": false,
+ "serviceAccountsEnabled": false,
+ "publicClient": true,
+ "frontchannelLogout": false,
+ "protocol": "openid-connect",
+ "attributes": {
+ "realm_client": "false",
+ "client.use.lightweight.access.token.enabled": "true",
+ "post.logout.redirect.uris": "+",
+ "pkce.code.challenge.method": "S256"
+ },
+ "authenticationFlowBindingOverrides": {},
+ "fullScopeAllowed": true,
+ "nodeReRegistrationTimeout": 0,
+ "protocolMappers": [
+ {
+ "name": "locale",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "locale",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "locale",
+ "jsonType.label": "String"
+ }
+ }
+ ],
+ "defaultClientScopes": [
+ "web-origins",
+ "acr",
+ "profile",
+ "roles",
+ "basic",
+ "email"
+ ],
+ "optionalClientScopes": [
+ "address",
+ "phone",
+ "offline_access",
+ "microprofile-jwt"
+ ]
+ }
+ ],
+ "clientScopes": [
+ {
+ "name": "email",
+ "description": "OpenID Connect built-in scope: email",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "true",
+ "consent.screen.text": "${emailScopeConsentText}",
+ "display.on.consent.screen": "true"
+ },
+ "protocolMappers": [
+ {
+ "name": "email verified",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-property-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "emailVerified",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "email_verified",
+ "jsonType.label": "boolean"
+ }
+ },
+ {
+ "name": "email",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "email",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "email",
+ "jsonType.label": "String"
+ }
+ }
+ ]
+ },
+ {
+ "name": "jim-api",
+ "description": "JIM API access scope",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "true",
+ "consent.screen.text": "Access JIM API",
+ "display.on.consent.screen": "true"
+ },
+ "protocolMappers": [
+ {
+ "name": "jim-api-audience",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-audience-mapper",
+ "consentRequired": false,
+ "config": {
+ "included.client.audience": "jim-web",
+ "id.token.claim": "false",
+ "lightweight.claim": "false",
+ "access.token.claim": "true",
+ "userinfo.token.claim": "false"
+ }
+ }
+ ]
+ },
+ {
+ "name": "offline_access",
+ "description": "OpenID Connect built-in scope: offline_access",
+ "protocol": "openid-connect",
+ "attributes": {
+ "consent.screen.text": "${offlineAccessScopeConsentText}",
+ "display.on.consent.screen": "true"
+ }
+ },
+ {
+ "name": "web-origins",
+ "description": "OpenID Connect scope for add allowed web origins to the access token",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "false",
+ "consent.screen.text": "",
+ "display.on.consent.screen": "false"
+ },
+ "protocolMappers": [
+ {
+ "name": "allowed web origins",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-allowed-origins-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "access.token.claim": "true"
+ }
+ }
+ ]
+ },
+ {
+ "name": "phone",
+ "description": "OpenID Connect built-in scope: phone",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "true",
+ "consent.screen.text": "${phoneScopeConsentText}",
+ "display.on.consent.screen": "true"
+ },
+ "protocolMappers": [
+ {
+ "name": "phone number",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "phoneNumber",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "phone_number",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "phone number verified",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "phoneNumberVerified",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "phone_number_verified",
+ "jsonType.label": "boolean"
+ }
+ }
+ ]
+ },
+ {
+ "name": "acr",
+ "description": "OpenID Connect scope for add acr (authentication context class reference) to the token",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "false",
+ "display.on.consent.screen": "false"
+ },
+ "protocolMappers": [
+ {
+ "name": "acr loa level",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-acr-mapper",
+ "consentRequired": false,
+ "config": {
+ "id.token.claim": "true",
+ "introspection.token.claim": "true",
+ "access.token.claim": "true"
+ }
+ }
+ ]
+ },
+ {
+ "name": "microprofile-jwt",
+ "description": "Microprofile - JWT built-in scope",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "true",
+ "display.on.consent.screen": "false"
+ },
+ "protocolMappers": [
+ {
+ "name": "groups",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-realm-role-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "multivalued": "true",
+ "user.attribute": "foo",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "groups",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "upn",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "username",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "upn",
+ "jsonType.label": "String"
+ }
+ }
+ ]
+ },
+ {
+ "name": "profile",
+ "description": "OpenID Connect built-in scope: profile",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "true",
+ "consent.screen.text": "${profileScopeConsentText}",
+ "display.on.consent.screen": "true"
+ },
+ "protocolMappers": [
+ {
+ "name": "middle name",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "middleName",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "middle_name",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "picture",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "picture",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "picture",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "nickname",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "nickname",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "nickname",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "profile",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "profile",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "profile",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "zoneinfo",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "zoneinfo",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "zoneinfo",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "locale",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "locale",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "locale",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "family name",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "lastName",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "family_name",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "given name",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "firstName",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "given_name",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "username",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "username",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "preferred_username",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "website",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "website",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "website",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "full name",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-full-name-mapper",
+ "consentRequired": false,
+ "config": {
+ "id.token.claim": "true",
+ "introspection.token.claim": "true",
+ "access.token.claim": "true",
+ "userinfo.token.claim": "true"
+ }
+ },
+ {
+ "name": "gender",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "gender",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "gender",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "birthdate",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "birthdate",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "birthdate",
+ "jsonType.label": "String"
+ }
+ },
+ {
+ "name": "updated at",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-attribute-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "userinfo.token.claim": "true",
+ "user.attribute": "updatedAt",
+ "id.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "updated_at",
+ "jsonType.label": "long"
+ }
+ }
+ ]
+ },
+ {
+ "name": "basic",
+ "description": "OpenID Connect scope for add all basic claims to the token",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "false",
+ "display.on.consent.screen": "false"
+ },
+ "protocolMappers": [
+ {
+ "name": "auth_time",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usersessionmodel-note-mapper",
+ "consentRequired": false,
+ "config": {
+ "user.session.note": "AUTH_TIME",
+ "id.token.claim": "true",
+ "introspection.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "auth_time",
+ "jsonType.label": "long"
+ }
+ },
+ {
+ "name": "sub",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-sub-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "access.token.claim": "true"
+ }
+ }
+ ]
+ },
+ {
+ "name": "roles",
+ "description": "OpenID Connect scope for add user roles to the access token",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "false",
+ "consent.screen.text": "${rolesScopeConsentText}",
+ "display.on.consent.screen": "true"
+ },
+ "protocolMappers": [
+ {
+ "name": "realm roles",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-realm-role-mapper",
+ "consentRequired": false,
+ "config": {
+ "user.attribute": "foo",
+ "introspection.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "realm_access.roles",
+ "jsonType.label": "String",
+ "multivalued": "true"
+ }
+ },
+ {
+ "name": "audience resolve",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-audience-resolve-mapper",
+ "consentRequired": false,
+ "config": {
+ "introspection.token.claim": "true",
+ "access.token.claim": "true"
+ }
+ },
+ {
+ "name": "client roles",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-usermodel-client-role-mapper",
+ "consentRequired": false,
+ "config": {
+ "user.attribute": "foo",
+ "introspection.token.claim": "true",
+ "access.token.claim": "true",
+ "claim.name": "resource_access.${client_id}.roles",
+ "jsonType.label": "String",
+ "multivalued": "true"
+ }
+ }
+ ]
+ },
+ {
+ "name": "address",
+ "description": "OpenID Connect built-in scope: address",
+ "protocol": "openid-connect",
+ "attributes": {
+ "include.in.token.scope": "true",
+ "consent.screen.text": "${addressScopeConsentText}",
+ "display.on.consent.screen": "true"
+ },
+ "protocolMappers": [
+ {
+ "name": "address",
+ "protocol": "openid-connect",
+ "protocolMapper": "oidc-address-mapper",
+ "consentRequired": false,
+ "config": {
+ "user.attribute.formatted": "formatted",
+ "user.attribute.country": "country",
+ "introspection.token.claim": "true",
+ "user.attribute.postal_code": "postal_code",
+ "userinfo.token.claim": "true",
+ "user.attribute.street": "street",
+ "id.token.claim": "true",
+ "user.attribute.region": "region",
+ "access.token.claim": "true",
+ "user.attribute.locality": "locality"
+ }
+ }
+ ]
+ }
+ ],
+ "defaultDefaultClientScopes": [
+ "profile",
+ "email",
+ "roles",
+ "web-origins",
+ "acr",
+ "basic"
+ ],
+ "defaultOptionalClientScopes": [
+ "offline_access",
+ "address",
+ "phone",
+ "microprofile-jwt",
+ "jim-api"
+ ],
+ "browserSecurityHeaders": {
+ "contentSecurityPolicyReportOnly": "",
+ "xContentTypeOptions": "nosniff",
+ "referrerPolicy": "no-referrer",
+ "xRobotsTag": "none",
+ "xFrameOptions": "SAMEORIGIN",
+ "contentSecurityPolicy": "frame-src 'self'; frame-ancestors 'self'; object-src 'none';",
+ "xXSSProtection": "1; mode=block",
+ "strictTransportSecurity": "max-age=31536000; includeSubDomains"
+ },
+ "smtpServer": {},
+ "eventsEnabled": false,
+ "eventsListeners": [
+ "jboss-logging"
+ ],
+ "enabledEventTypes": [],
+ "adminEventsEnabled": false,
+ "adminEventsDetailsEnabled": false,
+ "identityProviders": [],
+ "identityProviderMappers": [],
+ "components": {
+ "org.keycloak.services.clientregistration.policy.ClientRegistrationPolicy": [
+ {
+ "name": "Allowed Protocol Mapper Types",
+ "providerId": "allowed-protocol-mappers",
+ "subType": "authenticated",
+ "subComponents": {},
+ "config": {
+ "allowed-protocol-mapper-types": [
+ "saml-role-list-mapper",
+ "oidc-usermodel-attribute-mapper",
+ "oidc-sha256-pairwise-sub-mapper",
+ "saml-user-property-mapper",
+ "oidc-usermodel-property-mapper",
+ "oidc-address-mapper",
+ "oidc-full-name-mapper",
+ "saml-user-attribute-mapper"
+ ]
+ }
+ },
+ {
+ "name": "Trusted Hosts",
+ "providerId": "trusted-hosts",
+ "subType": "anonymous",
+ "subComponents": {},
+ "config": {
+ "host-sending-registration-request-must-match": [
+ "true"
+ ],
+ "client-uris-must-match": [
+ "true"
+ ]
+ }
+ },
+ {
+ "name": "Allowed Protocol Mapper Types",
+ "providerId": "allowed-protocol-mappers",
+ "subType": "anonymous",
+ "subComponents": {},
+ "config": {
+ "allowed-protocol-mapper-types": [
+ "saml-user-property-mapper",
+ "oidc-sha256-pairwise-sub-mapper",
+ "saml-role-list-mapper",
+ "oidc-full-name-mapper",
+ "saml-user-attribute-mapper",
+ "oidc-usermodel-property-mapper",
+ "oidc-address-mapper",
+ "oidc-usermodel-attribute-mapper"
+ ]
+ }
+ },
+ {
+ "name": "Max Clients Limit",
+ "providerId": "max-clients",
+ "subType": "anonymous",
+ "subComponents": {},
+ "config": {
+ "max-clients": [
+ "200"
+ ]
+ }
+ },
+ {
+ "name": "Allowed Client Scopes",
+ "providerId": "allowed-client-templates",
+ "subType": "authenticated",
+ "subComponents": {},
+ "config": {
+ "allow-default-scopes": [
+ "true"
+ ]
+ }
+ },
+ {
+ "name": "Full Scope Disabled",
+ "providerId": "scope",
+ "subType": "anonymous",
+ "subComponents": {},
+ "config": {}
+ },
+ {
+ "name": "Consent Required",
+ "providerId": "consent-required",
+ "subType": "anonymous",
+ "subComponents": {},
+ "config": {}
+ },
+ {
+ "name": "Allowed Client Scopes",
+ "providerId": "allowed-client-templates",
+ "subType": "anonymous",
+ "subComponents": {},
+ "config": {
+ "allow-default-scopes": [
+ "true"
+ ]
+ }
+ }
+ ],
+ "org.keycloak.keys.KeyProvider": [
+ {
+ "name": "rsa-generated",
+ "providerId": "rsa-generated",
+ "subComponents": {},
+ "config": {
+ "priority": [
+ "100"
+ ]
+ }
+ },
+ {
+ "name": "rsa-enc-generated",
+ "providerId": "rsa-enc-generated",
+ "subComponents": {},
+ "config": {
+ "priority": [
+ "100"
+ ],
+ "algorithm": [
+ "RSA-OAEP"
+ ]
+ }
+ },
+ {
+ "name": "hmac-generated-hs512",
+ "providerId": "hmac-generated",
+ "subComponents": {},
+ "config": {
+ "priority": [
+ "100"
+ ],
+ "algorithm": [
+ "HS512"
+ ]
+ }
+ },
+ {
+ "name": "aes-generated",
+ "providerId": "aes-generated",
+ "subComponents": {},
+ "config": {
+ "priority": [
+ "100"
+ ]
+ }
+ }
+ ]
+ },
+ "internationalizationEnabled": false,
+ "supportedLocales": [],
+ "authenticationFlows": [
+ {
+ "alias": "Account verification options",
+ "description": "Method with which to verity the existing account",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "idp-email-verification",
+ "authenticatorFlow": false,
+ "requirement": "ALTERNATIVE",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "ALTERNATIVE",
+ "priority": 20,
+ "autheticatorFlow": true,
+ "flowAlias": "Verify Existing Account by Re-authentication",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "Browser - Conditional OTP",
+ "description": "Flow to determine if the OTP is required for the authentication",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "conditional-user-configured",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "auth-otp-form",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "Browser - Conditional Organization",
+ "description": "Flow to determine if the organization identity-first login is to be used",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "conditional-user-configured",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "organization",
+ "authenticatorFlow": false,
+ "requirement": "ALTERNATIVE",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "Direct Grant - Conditional OTP",
+ "description": "Flow to determine if the OTP is required for the authentication",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "conditional-user-configured",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "direct-grant-validate-otp",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "First Broker Login - Conditional Organization",
+ "description": "Flow to determine if the authenticator that adds organization members is to be used",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "conditional-user-configured",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "idp-add-organization-member",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "First broker login - Conditional OTP",
+ "description": "Flow to determine if the OTP is required for the authentication",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "conditional-user-configured",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "auth-otp-form",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "Handle Existing Account",
+ "description": "Handle what to do if there is existing account with same email/username like authenticated identity provider",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "idp-confirm-link",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": true,
+ "flowAlias": "Account verification options",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "Organization",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticatorFlow": true,
+ "requirement": "CONDITIONAL",
+ "priority": 10,
+ "autheticatorFlow": true,
+ "flowAlias": "Browser - Conditional Organization",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "Reset - Conditional OTP",
+ "description": "Flow to determine if the OTP should be reset or not. Set to REQUIRED to force.",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "conditional-user-configured",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "reset-otp",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "User creation or linking",
+ "description": "Flow for the existing/non-existing user alternatives",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticatorConfig": "create unique user config",
+ "authenticator": "idp-create-user-if-unique",
+ "authenticatorFlow": false,
+ "requirement": "ALTERNATIVE",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "ALTERNATIVE",
+ "priority": 20,
+ "autheticatorFlow": true,
+ "flowAlias": "Handle Existing Account",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "Verify Existing Account by Re-authentication",
+ "description": "Reauthentication of existing account",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "idp-username-password-form",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "CONDITIONAL",
+ "priority": 20,
+ "autheticatorFlow": true,
+ "flowAlias": "First broker login - Conditional OTP",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "browser",
+ "description": "Browser based authentication",
+ "providerId": "basic-flow",
+ "topLevel": true,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "auth-cookie",
+ "authenticatorFlow": false,
+ "requirement": "ALTERNATIVE",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "auth-spnego",
+ "authenticatorFlow": false,
+ "requirement": "DISABLED",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "identity-provider-redirector",
+ "authenticatorFlow": false,
+ "requirement": "ALTERNATIVE",
+ "priority": 25,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "ALTERNATIVE",
+ "priority": 26,
+ "autheticatorFlow": true,
+ "flowAlias": "Organization",
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "ALTERNATIVE",
+ "priority": 30,
+ "autheticatorFlow": true,
+ "flowAlias": "forms",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "clients",
+ "description": "Base authentication for clients",
+ "providerId": "client-flow",
+ "topLevel": true,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "client-secret",
+ "authenticatorFlow": false,
+ "requirement": "ALTERNATIVE",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "client-jwt",
+ "authenticatorFlow": false,
+ "requirement": "ALTERNATIVE",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "client-secret-jwt",
+ "authenticatorFlow": false,
+ "requirement": "ALTERNATIVE",
+ "priority": 30,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "client-x509",
+ "authenticatorFlow": false,
+ "requirement": "ALTERNATIVE",
+ "priority": 40,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "direct grant",
+ "description": "OpenID Connect Resource Owner Grant",
+ "providerId": "basic-flow",
+ "topLevel": true,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "direct-grant-validate-username",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "direct-grant-validate-password",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "CONDITIONAL",
+ "priority": 30,
+ "autheticatorFlow": true,
+ "flowAlias": "Direct Grant - Conditional OTP",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "docker auth",
+ "description": "Used by Docker clients to authenticate against the IDP",
+ "providerId": "basic-flow",
+ "topLevel": true,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "docker-http-basic-authenticator",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "first broker login",
+ "description": "Actions taken after first broker login with identity provider account, which is not yet linked to any Keycloak account",
+ "providerId": "basic-flow",
+ "topLevel": true,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticatorConfig": "review profile config",
+ "authenticator": "idp-review-profile",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": true,
+ "flowAlias": "User creation or linking",
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "CONDITIONAL",
+ "priority": 50,
+ "autheticatorFlow": true,
+ "flowAlias": "First Broker Login - Conditional Organization",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "forms",
+ "description": "Username, password, otp and other auth forms.",
+ "providerId": "basic-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "auth-username-password-form",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "CONDITIONAL",
+ "priority": 20,
+ "autheticatorFlow": true,
+ "flowAlias": "Browser - Conditional OTP",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "registration",
+ "description": "Registration flow",
+ "providerId": "basic-flow",
+ "topLevel": true,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "registration-page-form",
+ "authenticatorFlow": true,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": true,
+ "flowAlias": "registration form",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "registration form",
+ "description": "Registration form",
+ "providerId": "form-flow",
+ "topLevel": false,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "registration-user-creation",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "registration-password-action",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 50,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "registration-recaptcha-action",
+ "authenticatorFlow": false,
+ "requirement": "DISABLED",
+ "priority": 60,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "registration-terms-and-conditions",
+ "authenticatorFlow": false,
+ "requirement": "DISABLED",
+ "priority": 70,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "reset credentials",
+ "description": "Reset credentials for a user if they forgot their password or something",
+ "providerId": "basic-flow",
+ "topLevel": true,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "reset-credentials-choose-user",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "reset-credential-email",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 20,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticator": "reset-password",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 30,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ },
+ {
+ "authenticatorFlow": true,
+ "requirement": "CONDITIONAL",
+ "priority": 40,
+ "autheticatorFlow": true,
+ "flowAlias": "Reset - Conditional OTP",
+ "userSetupAllowed": false
+ }
+ ]
+ },
+ {
+ "alias": "saml ecp",
+ "description": "SAML ECP Profile Authentication Flow",
+ "providerId": "basic-flow",
+ "topLevel": true,
+ "builtIn": true,
+ "authenticationExecutions": [
+ {
+ "authenticator": "http-basic-authenticator",
+ "authenticatorFlow": false,
+ "requirement": "REQUIRED",
+ "priority": 10,
+ "autheticatorFlow": false,
+ "userSetupAllowed": false
+ }
+ ]
+ }
+ ],
+ "authenticatorConfig": [
+ {
+ "alias": "create unique user config",
+ "config": {
+ "require.password.update.after.registration": "false"
+ }
+ },
+ {
+ "alias": "review profile config",
+ "config": {
+ "update.profile.on.first.login": "missing"
+ }
+ }
+ ],
+ "requiredActions": [
+ {
+ "alias": "CONFIGURE_TOTP",
+ "name": "Configure OTP",
+ "providerId": "CONFIGURE_TOTP",
+ "enabled": true,
+ "defaultAction": false,
+ "priority": 10,
+ "config": {}
+ },
+ {
+ "alias": "TERMS_AND_CONDITIONS",
+ "name": "Terms and Conditions",
+ "providerId": "TERMS_AND_CONDITIONS",
+ "enabled": false,
+ "defaultAction": false,
+ "priority": 20,
+ "config": {}
+ },
+ {
+ "alias": "UPDATE_PASSWORD",
+ "name": "Update Password",
+ "providerId": "UPDATE_PASSWORD",
+ "enabled": true,
+ "defaultAction": false,
+ "priority": 30,
+ "config": {}
+ },
+ {
+ "alias": "UPDATE_PROFILE",
+ "name": "Update Profile",
+ "providerId": "UPDATE_PROFILE",
+ "enabled": true,
+ "defaultAction": false,
+ "priority": 40,
+ "config": {}
+ },
+ {
+ "alias": "VERIFY_EMAIL",
+ "name": "Verify Email",
+ "providerId": "VERIFY_EMAIL",
+ "enabled": true,
+ "defaultAction": false,
+ "priority": 50,
+ "config": {}
+ },
+ {
+ "alias": "delete_account",
+ "name": "Delete Account",
+ "providerId": "delete_account",
+ "enabled": false,
+ "defaultAction": false,
+ "priority": 60,
+ "config": {}
+ },
+ {
+ "alias": "webauthn-register",
+ "name": "Webauthn Register",
+ "providerId": "webauthn-register",
+ "enabled": true,
+ "defaultAction": false,
+ "priority": 70,
+ "config": {}
+ },
+ {
+ "alias": "webauthn-register-passwordless",
+ "name": "Webauthn Register Passwordless",
+ "providerId": "webauthn-register-passwordless",
+ "enabled": true,
+ "defaultAction": false,
+ "priority": 80,
+ "config": {}
+ },
+ {
+ "alias": "VERIFY_PROFILE",
+ "name": "Verify Profile",
+ "providerId": "VERIFY_PROFILE",
+ "enabled": true,
+ "defaultAction": false,
+ "priority": 90,
+ "config": {}
+ },
+ {
+ "alias": "delete_credential",
+ "name": "Delete Credential",
+ "providerId": "delete_credential",
+ "enabled": true,
+ "defaultAction": false,
+ "priority": 100,
+ "config": {}
+ },
+ {
+ "alias": "update_user_locale",
+ "name": "Update User Locale",
+ "providerId": "update_user_locale",
+ "enabled": true,
+ "defaultAction": false,
+ "priority": 1000,
+ "config": {}
+ }
+ ],
+ "browserFlow": "browser",
+ "registrationFlow": "registration",
+ "directGrantFlow": "direct grant",
+ "resetCredentialsFlow": "reset credentials",
+ "clientAuthenticationFlow": "clients",
+ "dockerAuthenticationFlow": "docker auth",
+ "firstBrokerLoginFlow": "first broker login",
+ "attributes": {
+ "cibaBackchannelTokenDeliveryMode": "poll",
+ "cibaExpiresIn": "120",
+ "cibaAuthRequestedUserHint": "login_hint",
+ "oauth2DeviceCodeLifespan": "600",
+ "oauth2DevicePollingInterval": "5",
+ "parRequestUriLifespan": "60",
+ "cibaInterval": "5",
+ "realmReusableOtpCode": "false"
+ },
+ "keycloakVersion": "26.0.8",
+ "userManagedAccessAllowed": false,
+ "organizationsEnabled": false,
+ "clientProfiles": {
+ "profiles": []
+ },
+ "clientPolicies": {
+ "policies": []
+ },
+ "id": "jim",
+ "users": [
+ {
+ "id": "00000000-0000-0000-0000-000000000001",
+ "username": "admin",
+ "enabled": true,
+ "emailVerified": true,
+ "firstName": "Admin",
+ "lastName": "User",
+ "email": "admin@jim.local",
+ "credentials": [
+ {
+ "type": "password",
+ "value": "admin",
+ "temporary": false
+ }
+ ],
+ "realmRoles": [
+ "default-roles-jim"
+ ]
+ },
+ {
+ "id": "00000000-0000-0000-0000-000000000002",
+ "username": "user",
+ "enabled": true,
+ "emailVerified": true,
+ "firstName": "Test",
+ "lastName": "User",
+ "email": "user@jim.local",
+ "credentials": [
+ {
+ "type": "password",
+ "value": "user",
+ "temporary": false
+ }
+ ],
+ "realmRoles": [
+ "default-roles-jim"
+ ]
+ }
+ ]
+}
diff --git a/.devcontainer/setup.sh b/.devcontainer/setup.sh
index 2ab442fb1..4d552963e 100755
--- a/.devcontainer/setup.sh
+++ b/.devcontainer/setup.sh
@@ -73,7 +73,7 @@ else
done
print_success ".env created from .env.example"
- print_warning "Remember to update SSO settings for real authentication!"
+ print_success "SSO pre-configured for bundled Keycloak (admin/admin, user/user)"
fi
# 4. Auto-tune PostgreSQL for devcontainer specs
@@ -88,7 +88,7 @@ else
print_warning "postgres-tune.sh not found - skipping auto-tuning"
fi
-# 5. Install PowerShell Pester module for testing
+# 5. Install PowerShell Pester module for testing (socat is in the Dockerfile)
print_step "Installing PowerShell Pester module..."
if pwsh -NoProfile -Command 'Set-PSRepository PSGallery -InstallationPolicy Trusted; Install-Module -Name Pester -MinimumVersion 5.0 -Force -Scope CurrentUser' 2>/dev/null; then
print_success "Pester module installed"
@@ -213,6 +213,7 @@ echo " jim-db - Start PostgreSQL"
echo ""
echo "Available Services:"
echo " PostgreSQL: localhost:5432"
+echo " Keycloak IdP: http://localhost:8181 (admin / admin)"
echo ""
echo " When running locally (F5):"
echo " JIM Web: https://localhost:7000"
@@ -227,13 +228,13 @@ echo ""
echo "🚀 To start developing (choose one):"
echo ""
echo " Option 1 - Local Debug (Recommended):"
-echo " 1. Press F5 in VS Code"
-echo " 2. Select 'JIM Full Stack' or 'JIM Web Stack'"
-echo " 3. Set breakpoints and debug"
+echo " 1. Run: jim-db && jim-keycloak"
+echo " 2. Press F5 in VS Code"
+echo " 3. Sign in with: admin / admin"
echo ""
echo " Option 2 - Docker Stack:"
-echo " 1. Review .env file and update SSO settings"
-echo " 2. Run: jim-stack"
-echo " 3. Open: http://localhost:5200"
+echo " 1. Run: jim-stack"
+echo " 2. Open: http://localhost:5200"
+echo " 3. Sign in with: admin / admin"
echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
diff --git a/.env.example b/.env.example
index fe48dba27..4567bf762 100644
--- a/.env.example
+++ b/.env.example
@@ -72,8 +72,20 @@ JIM_LOG_REQUESTS=false
# JIM_INFRASTRUCTURE_API_KEY=jim_ak_your_generated_key_here
# =============================================================================
-# SSO/OIDC Configuration
+# SSO/OIDC Configuration — Development defaults (bundled Keycloak)
# =============================================================================
+# The devcontainer ships a pre-configured Keycloak instance that starts
+# automatically with jim-stack. These defaults work out of the box —
+# override them below to use an external IdP instead.
+#
+# Bundled Keycloak:
+# Admin console: http://localhost:8181 (admin / admin)
+# Test users: admin / admin | user / user
+# Realm: jim
+#
+# To use an external IdP, replace these values with your provider's settings.
+# See docs/SSO_SETUP_GUIDE.md for provider-specific configuration.
+#
# The OIDC authority URL - examples for common identity providers:
#
# Cloud-based:
@@ -91,7 +103,7 @@ JIM_LOG_REQUESTS=false
# Authentik: https://{authentik-server}/application/o/{app-slug}
# Zitadel: https://{your-instance}.zitadel.cloud
#
-JIM_SSO_AUTHORITY=https://login.microsoftonline.com/your-tenant-id/v2.0
+JIM_SSO_AUTHORITY=http://localhost:8181/realms/jim
# The OAuth client/application ID
# Format varies by provider:
@@ -102,10 +114,10 @@ JIM_SSO_AUTHORITY=https://login.microsoftonline.com/your-tenant-id/v2.0
# AD FS: client identifier string
# Google: numeric string ending in .apps.googleusercontent.com
#
-JIM_SSO_CLIENT_ID=your-client-id
+JIM_SSO_CLIENT_ID=jim-web
# The OAuth client secret
-JIM_SSO_SECRET=your-client-secret
+JIM_SSO_SECRET=jim-dev-secret
# The API scope for JWT bearer authentication (API endpoints)
# Examples:
@@ -115,17 +127,18 @@ JIM_SSO_SECRET=your-client-secret
# Keycloak: {client-id}
# AD FS: api://{client-id}
#
-JIM_SSO_API_SCOPE=api://your-client-id/access_as_user
+JIM_SSO_API_SCOPE=jim-api
# Trusted token issuers for API JWT validation (comma-separated)
# Usually not needed - JIM auto-detects the issuer from the authority URL.
# Only set this if your provider's issuer URL differs from its authority URL,
# or you need to trust multiple issuers (e.g., during a provider migration).
#
-# Example:
-# JIM_SSO_VALID_ISSUERS=https://idp.example.com/,https://new-idp.example.com/
+# The bundled Keycloak defaults below cover both access paths:
+# - localhost:8181 (browser, local F5 debugging)
+# - jim.keycloak:8080 (Docker DNS, back-channel from jim.web container)
#
-JIM_SSO_VALID_ISSUERS=
+JIM_SSO_VALID_ISSUERS=http://localhost:8181/realms/jim,http://jim.keycloak:8080/realms/jim
# =============================================================================
# User Identity Mapping
@@ -165,4 +178,5 @@ JIM_SSO_MV_ATTRIBUTE="Subject Identifier"
# sub: Varies by provider (Entra ID uses a cryptographic hash, others use GUIDs)
# email: admin@example.com
#
-JIM_SSO_INITIAL_ADMIN=your-admin-sub-claim-value
+# The default below matches the bundled Keycloak admin user (fixed UUID).
+JIM_SSO_INITIAL_ADMIN=00000000-0000-0000-0000-000000000001
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ed468bbda..9fb07e4b3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
+#### Bundled Keycloak IdP for Development (#197)
+
+- ✨ Zero-config SSO — `jim-stack` starts a pre-configured Keycloak instance alongside JIM; developers sign in immediately with `admin` / `admin`
+- ✨ Pre-configured realm with `jim-web` (confidential + PKCE) and `jim-powershell` (public + PKCE) clients, `jim-api` scope, and two test users
+- ✨ `.env.example` defaults point to the bundled Keycloak — no manual IdP configuration needed for local development
+- ✨ `jim-keycloak` / `jim-keycloak-stop` / `jim-keycloak-logs` aliases for standalone Keycloak (F5 debugging workflow)
+- ✨ Keycloak admin console accessible at `http://localhost:8181`
+- 🔒 HTTP OIDC authority support for development (RequireHttpsMetadata conditionally disabled)
+
+### Fixed
+
+- 🔒 Attribute change history is no longer cascade-deleted when a metaverse or connected system attribute definition is removed — the FK is set to null and snapshot `AttributeName`/`AttributeType` properties preserve the audit trail indefinitely (#58)
+
+### Added
+
#### Worker Redesign Option A (#394)
- ✨ Pure domain engine (`ISyncEngine`) — 7 stateless methods with zero I/O dependencies, making core sync logic independently testable with plain objects
diff --git a/README.md b/README.md
index 3e5878e92..c31ca4e92 100644
--- a/README.md
+++ b/README.md
@@ -175,7 +175,7 @@ For production hardening (TLS, reverse proxy, upgrades, monitoring), see the [De
### For Developers (Contribute)
-**Prerequisites:** Configure SSO using the [SSO Setup Guide](docs/SSO_SETUP_GUIDE.md) — JIM requires authentication even during development.
+**Prerequisites:** None — the devcontainer ships a bundled Keycloak for SSO. Sign in with `admin` / `admin`. To use an external IdP, see the [SSO Setup Guide](docs/SSO_SETUP_GUIDE.md).
**Option 1 — GitHub Codespaces (one click):**
diff --git a/docker-compose.integration-tests.yml b/docker-compose.integration-tests.yml
index 9fb353eb6..fb70e5290 100644
--- a/docker-compose.integration-tests.yml
+++ b/docker-compose.integration-tests.yml
@@ -37,14 +37,14 @@ services:
context: ./test/integration/docker/samba-ad-prebuilt
dockerfile: Dockerfile
args:
- SAMBA_DOMAIN: SUBATOMIC.LOCAL
+ SAMBA_DOMAIN: PANOPLY.LOCAL
SAMBA_PASSWORD: Test@123!
container_name: samba-ad-primary
privileged: true
environment:
# diegogslomp/samba-ad-dc environment variables
- - REALM=SUBATOMIC.LOCAL
- - DOMAIN=SUBATOMIC
+ - REALM=PANOPLY.LOCAL
+ - DOMAIN=PANOPLY
- ADMIN_PASS=Test@123!
- DNS_FORWARDER=8.8.8.8
# Legacy environment variables for compatibility with existing scripts
@@ -86,14 +86,14 @@ services:
context: ./test/integration/docker/samba-ad-prebuilt
dockerfile: Dockerfile
args:
- SAMBA_DOMAIN: SOURCEDOMAIN.LOCAL
+ SAMBA_DOMAIN: RESURGAM.LOCAL
SAMBA_PASSWORD: Test@123!
container_name: samba-ad-source
privileged: true
environment:
# diegogslomp/samba-ad-dc environment variables
- - REALM=SOURCEDOMAIN.LOCAL
- - DOMAIN=SOURCEDOMAIN
+ - REALM=RESURGAM.LOCAL
+ - DOMAIN=RESURGAM
- ADMIN_PASS=Test@123!
- DNS_FORWARDER=8.8.8.8
# Legacy environment variables for compatibility with existing scripts
@@ -134,14 +134,14 @@ services:
context: ./test/integration/docker/samba-ad-prebuilt
dockerfile: Dockerfile
args:
- SAMBA_DOMAIN: TARGETDOMAIN.LOCAL
+ SAMBA_DOMAIN: GENTIAN.LOCAL
SAMBA_PASSWORD: Test@123!
container_name: samba-ad-target
privileged: true
environment:
# diegogslomp/samba-ad-dc environment variables
- - REALM=TARGETDOMAIN.LOCAL
- - DOMAIN=TARGETDOMAIN
+ - REALM=GENTIAN.LOCAL
+ - DOMAIN=GENTIAN
- ADMIN_PASS=Test@123!
- DNS_FORWARDER=8.8.8.8
# Legacy environment variables for compatibility with existing scripts
@@ -239,30 +239,49 @@ services:
timeout: 5s
retries: 5
- # Phase 2 - Additional LDAP validation
- openldap:
- image: osixia/openldap:latest
- container_name: openldap-test
+ # OpenLDAP with two suffixes for multi-partition testing (Issue #72, Phase 1b)
+ # Suffixes: dc=yellowstone,dc=local and dc=glitterband,dc=local
+ openldap-primary:
+ image: ${OPENLDAP_IMAGE_PRIMARY:-ghcr.io/tetronio/jim-openldap:primary}
+ build:
+ context: ./test/integration/docker/openldap
+ dockerfile: Dockerfile
+ container_name: openldap-primary
environment:
- - LDAP_ORGANISATION=TestOrg
- - LDAP_DOMAIN=test.local
- - LDAP_ADMIN_PASSWORD=admin
- - LDAP_CONFIG_PASSWORD=config
- - LDAP_READONLY_USER=false
- - LDAP_TLS=false
+ - LDAP_ROOT=dc=yellowstone,dc=local
+ - LDAP_ADMIN_USERNAME=admin
+ - LDAP_ADMIN_PASSWORD=Test@123!
+ - LDAP_CONFIG_ADMIN_ENABLED=yes
+ - LDAP_CONFIG_ADMIN_USERNAME=admin
+ - LDAP_CONFIG_ADMIN_PASSWORD=Test@123!
+ - LDAP_ENABLE_TLS=no
+ - LDAP_SKIP_DEFAULT_TREE=no
+ - LDAP_ENABLE_ACCESSLOG=yes
# No ports exposed - internal to Docker network only for testing
volumes:
- - openldap-data:/var/lib/ldap
- - openldap-config:/etc/ldap/slapd.d
+ - openldap-primary-data:/bitnami/openldap
+ - test-csv-data:/connector-files
networks:
- jim-network
profiles:
- - phase2
+ - openldap
healthcheck:
- test: ["CMD", "ldapsearch", "-x", "-H", "ldap://localhost", "-b", "dc=test,dc=local", "-D", "cn=admin,dc=test,dc=local", "-w", "admin"]
- interval: 30s
- timeout: 10s
+ test: ["CMD", "ldapsearch", "-x", "-H", "ldap://localhost:1389",
+ "-b", "dc=yellowstone,dc=local",
+ "-D", "cn=admin,dc=yellowstone,dc=local", "-w", "Test@123!",
+ "-LLL", "(objectClass=organizationalUnit)"]
+ interval: 10s
+ timeout: 5s
retries: 5
+ start_period: 15s
+ deploy:
+ resources:
+ limits:
+ cpus: ${OPENLDAP_PRIMARY_CPUS:-2.0}
+ memory: ${OPENLDAP_PRIMARY_MEMORY:-2G}
+ reservations:
+ cpus: ${OPENLDAP_PRIMARY_CPUS_RESERVED:-0.5}
+ memory: ${OPENLDAP_PRIMARY_MEMORY_RESERVED:-512M}
# Phase 2 - MySQL testing
mysql-test:
@@ -311,10 +330,8 @@ volumes:
# Other volumes
test-csv-data:
name: jim-integration-csv-data
- openldap-data:
- name: jim-integration-openldap-data
- openldap-config:
- name: jim-integration-openldap-config
+ openldap-primary-data:
+ name: jim-integration-openldap-primary-data
sqlserver-data:
name: jim-integration-sqlserver-data
postgres-test-data:
diff --git a/docker-compose.override.yml b/docker-compose.override.yml
index 33c477bd9..5ff224ae8 100644
--- a/docker-compose.override.yml
+++ b/docker-compose.override.yml
@@ -6,6 +6,9 @@ services:
# Override JIM_DB_HOSTNAME from .env to use bundled database container
- JIM_DB_HOSTNAME=jim.database
- JIM_INFRASTRUCTURE_API_KEY=${JIM_INFRASTRUCTURE_API_KEY:-}
+ # Override SSO authority to use Docker DNS for back-channel OIDC discovery.
+ # The browser-facing redirect is rewritten to localhost:8181 by JIM.Web.
+ - JIM_SSO_AUTHORITY=http://jim.keycloak:8080/realms/jim
# OpenLDAP TLS configuration for LDAPS connections
- LDAPTLS_REQCERT=never
ports:
@@ -13,6 +16,9 @@ services:
volumes:
# Map repo test data to connector-files/test-data for development
- ./test/test-data:/var/connector-files/test-data
+ depends_on:
+ jim.keycloak:
+ condition: service_healthy
jim.worker:
environment:
# Override JIM_DB_HOSTNAME from .env to use bundled database container
@@ -44,3 +50,28 @@ services:
# Expose PostgreSQL port for local database tools (VS Code extensions, etc.)
ports:
- "5432:5432"
+
+ # Bundled Keycloak IdP for zero-config SSO in development.
+ # Pre-configured with a 'jim' realm, test users (admin/admin, user/user),
+ # and clients for the web UI (jim-web) and PowerShell module (jim-powershell).
+ # Admin console: http://localhost:8181 (admin/admin)
+ jim.keycloak:
+ image: quay.io/keycloak/keycloak:26.0
+ container_name: jim.keycloak
+ restart: unless-stopped
+ command: start-dev --import-realm --health-enabled=true
+ environment:
+ KC_BOOTSTRAP_ADMIN_USERNAME: admin
+ KC_BOOTSTRAP_ADMIN_PASSWORD: admin
+ ports:
+ - "8180:8080"
+ volumes:
+ - ./.devcontainer/keycloak/jim-realm.json:/opt/keycloak/data/import/jim-realm.json:ro
+ healthcheck:
+ test: ["CMD-SHELL", "exec 3<>/dev/tcp/127.0.0.1/9000; echo -e 'GET /health/ready HTTP/1.1\\r\\nhost: localhost\\r\\nConnection: close\\r\\n\\r\\n' >&3; timeout 1 cat <&3 | grep -q '200 OK'"]
+ interval: 10s
+ timeout: 5s
+ retries: 15
+ start_period: 30s
+ networks:
+ - jim-network
diff --git a/docs/DEVELOPER_GUIDE.md b/docs/DEVELOPER_GUIDE.md
index e8fa075d4..6f3558979 100644
--- a/docs/DEVELOPER_GUIDE.md
+++ b/docs/DEVELOPER_GUIDE.md
@@ -644,10 +644,9 @@ JIM uses GitHub Codespaces to provide a fully configured development environment
1. Open repository in GitHub
2. Click **Code** > **Codespaces** > **Create codespace on main**
3. Wait for provisioning (automatic setup via `.devcontainer/setup.sh`)
-4. Update the auto-generated `.env` file with your SSO configuration
-5. Use shell aliases: `jim-db`, `jim-web`, `jim-stack`, etc.
+4. Use shell aliases: `jim-db`, `jim-web`, `jim-stack`, etc.
-> **Note**: The setup script automatically creates a `.env` file with development defaults. You can also set a `DOTENV_BASE64` GitHub Codespaces secret to restore your own `.env` file automatically.
+> **Note**: The setup script automatically creates a `.env` file with development defaults. SSO is pre-configured for the bundled Keycloak — sign in with `admin` / `admin`. You can also set a `DOTENV_BASE64` GitHub Codespaces secret to restore your own `.env` file automatically.
**Available Shell Aliases**:
- `jim` - List all available jim aliases
@@ -727,7 +726,9 @@ Configuration via environment variables (defined in `.env`). See `.env.example`
### SSO/Authentication (IDP-Agnostic)
JIM works with any OIDC-compliant Identity Provider (Entra ID, Okta, Auth0, Keycloak, AD FS, etc.).
-For detailed setup instructions, see the [SSO Setup Guide](SSO_SETUP_GUIDE.md).
+**Development**: The devcontainer ships a bundled Keycloak — SSO works out of the box with `admin` / `admin`. No configuration needed.
+
+**Production**: Override the `JIM_SSO_*` variables with your provider's settings. See the [SSO Setup Guide](SSO_SETUP_GUIDE.md).
- `JIM_SSO_AUTHORITY`: OIDC authority URL (e.g., `https://login.microsoftonline.com/{tenant-id}/v2.0`)
- `JIM_SSO_CLIENT_ID`: OIDC client/application ID
diff --git a/docs/INTEGRATION_TESTING.md b/docs/INTEGRATION_TESTING.md
index 2d24cbec5..1b6714a21 100644
--- a/docs/INTEGRATION_TESTING.md
+++ b/docs/INTEGRATION_TESTING.md
@@ -81,15 +81,16 @@ This single script handles everything:
**Available Scenarios (`-Scenario` parameter):**
-| Scenario | Description | Containers Used |
-|----------|-------------|-----------------|
-| `Scenario1-HRToIdentityDirectory` | HR + Training CSV -> Subatomic AD + Cross-Domain provisioning (Joiner/Mover/Leaver) | samba-ad-primary |
-| `Scenario2-CrossDomainSync` | Quantum Dynamics APAC -> EMEA directory sync | samba-ad-source, samba-ad-target |
-| `Scenario4-DeletionRules` | Deletion rules and grace period testing | samba-ad-primary |
-| `Scenario5-MatchingRules` | Object matching rules testing | samba-ad-primary |
-| `Scenario6-SchedulerService` | Scheduler service end-to-end testing (parallel steps require 4 systems) | samba-ad-primary (requires Scenario1 setup) |
-| `Scenario7-ClearConnectedSystemObjects` | Clear connector space testing (deleteChangeHistory true/false, edge cases) | samba-ad-primary (requires Scenario1 setup) |
-| `Scenario8-CrossDomainEntitlementSync` | Group synchronisation between APAC and EMEA domains | samba-ad-source, samba-ad-target |
+| Scenario | Description | Containers Used | OpenLDAP |
+|----------|-------------|-----------------|----------|
+| `Scenario1-HRToIdentityDirectory` | HR + Training CSV -> AD provisioning (Joiner/Mover/Leaver) | samba-ad-primary / openldap-primary | ✅ |
+| `Scenario2-CrossDomainSync` | APAC -> EMEA directory sync | samba-ad-source, samba-ad-target / openldap-primary | ✅ |
+| `Scenario4-DeletionRules` | Deletion rules and grace period testing | samba-ad-primary / openldap-primary | ✅ |
+| `Scenario5-MatchingRules` | Object matching rules testing | samba-ad-primary / openldap-primary | ✅ |
+| `Scenario6-SchedulerService` | Scheduler service end-to-end testing | samba-ad-primary / openldap-primary | ✅ |
+| `Scenario7-ClearConnectedSystemObjects` | Clear connector space testing | samba-ad-primary / openldap-primary | ✅ |
+| `Scenario8-CrossDomainEntitlementSync` | Group sync between APAC and EMEA domains | samba-ad-source, samba-ad-target / openldap-primary | ✅ |
+| `Scenario9-PartitionScopedImports` | Partition-scoped import run profiles | samba-ad-primary / openldap-primary | ✅ |
**Available Templates (`-Template` parameter):**
@@ -108,6 +109,27 @@ Choose a template based on your testing goals:
See [Data Scale Templates](#data-scale-templates) for detailed template specifications.
+**Available Directory Types (`-DirectoryType` parameter):**
+
+| Directory Type | Description | Backend |
+|----------------|-------------|---------|
+| `SambaAD` (default) | Samba Active Directory | LDAPS on port 636, `objectGUID`, AD schema discovery |
+| `OpenLDAP` | OpenLDAP with multi-suffix partitions | LDAP on port 1389, `entryUUID`, RFC 4512 schema, accesslog delta import |
+| `All` | Both directory types (full regression) | Runs all scenarios against SambaAD first, then OpenLDAP |
+
+```powershell
+# Run against OpenLDAP
+./test/integration/Run-IntegrationTests.ps1 -Scenario All -Template Small -DirectoryType OpenLDAP
+
+# Run against both directory types (full cross-directory regression)
+./test/integration/Run-IntegrationTests.ps1 -Scenario All -Template Small -DirectoryType All
+
+# Run a specific scenario against both directory types
+./test/integration/Run-IntegrationTests.ps1 -Scenario Scenario1-HRToIdentityDirectory -DirectoryType All
+```
+
+> **Note:** OpenLDAP uses a single container (`openldap-primary`) with two naming contexts (suffixes) for multi-partition scenarios, while Samba AD uses separate containers (`samba-ad-source`, `samba-ad-target`). The test framework abstracts these differences via `Get-DirectoryConfig`.
+
**Alternative: Manual step-by-step (for debugging or more control)**
```powershell
@@ -338,7 +360,7 @@ All external systems run as Docker containers defined in `docker-compose.integra
│ │
│ Phase 1 (MVP): │
│ ┌──────────────────┐ ┌──────────────────┐ ┌──────────────┐ │
-│ │ Subatomic AD │ │ Quantum Dynamics │ │ Quantum │ │
+│ │ Panoply AD │ │ Panoply │ │ Quantum │ │
│ │ (Scenarios 1&3) │ │ APAC (Scen. 2) │ │ Dynamics │ │
│ │ Port: 389/636 │ │ Port: 10389/636 │ │ EMEA: 11389 │ │
│ └──────────────────┘ └──────────────────┘ └──────────────┘ │
@@ -437,11 +459,11 @@ All templates generate realistic enterprise data following normal distribution p
**Systems**:
- Source: CSV (HR system)
-- Target: Subatomic AD
+- Target: Panoply AD
**Test Data**:
-- HR CSV includes Company attribute: "Subatomic" for employees, partner companies for contractors
-- Partner companies: Nexus Dynamics, Orbital Systems, Quantum Bridge, Stellar Logistics, Vertex Solutions
+- HR CSV includes Company attribute: "Panoply" for employees, partner companies for contractors
+- Partner companies: Nexus Dynamics, Akinya, Rockhopper, Stellar Logistics, Vertex Solutions
**Test Steps** (executed sequentially):
@@ -521,8 +543,8 @@ Repository-level tests for the dual-path stats derivation logic (outcome-based v
**Purpose**: Validate unidirectional synchronisation of person entities between two directory services.
**Systems**:
-- Source: Quantum Dynamics APAC (authoritative)
-- Target: Quantum Dynamics EMEA
+- Source: Panoply APAC (authoritative)
+- Target: Panoply EMEA
**Test Steps** (executed sequentially):
@@ -555,7 +577,7 @@ Repository-level tests for the dual-path stats derivation logic (outcome-based v
**Purpose**: Validate exporting directory users to CSV for distribution/reporting.
**Systems**:
-- Source: Subatomic AD
+- Source: Panoply AD
- Target: CSV (GAL export)
**Test Steps** (executed sequentially):
@@ -588,7 +610,7 @@ Repository-level tests for the dual-path stats derivation logic (outcome-based v
**Systems**:
- Source: CSV (HR system)
-- Target: Subatomic AD
+- Target: Panoply AD
**Test Steps** (executed sequentially):
@@ -637,7 +659,7 @@ Repository-level tests for the dual-path stats derivation logic (outcome-based v
**Systems**:
- Source: CSV (HR system) with `hrId` (GUID) as external ID
-- Target: Subatomic AD
+- Target: Panoply AD
**Test Steps** (executed sequentially):
@@ -765,11 +787,11 @@ These scenarios test group management capabilities - a core ILM function where t
**Systems**:
- Source: JIM Metaverse (groups created via JIM API, not imported from a Connected System)
-- Target: Subatomic AD (OU=Entitlements,OU=Groups,OU=Corp,DC=subatomic,DC=local)
+- Target: Panoply AD (OU=Entitlements,OU=Groups,OU=Corp,DC=panoply,DC=local)
**Group Types Created**:
- **Department Groups**: `Dept-{Department}` (e.g., `Dept-Finance`, `Dept-Information Technology`)
-- **Company Groups**: `Company-{Company}` (e.g., `Company-Subatomic`, `Company-Nexus Dynamics`)
+- **Company Groups**: `Company-{Company}` (e.g., `Company-Panoply`, `Company-Nexus Dynamics`)
- **Job Title Groups**: `Role-{Title}` (e.g., `Role-Manager`, `Role-Analyst`)
**Test Steps** (executed sequentially):
@@ -794,9 +816,9 @@ These scenarios test group management capabilities - a core ILM function where t
**Concept**: Organisations often have existing groups in AD that were created manually or by other tools. This scenario tests bringing those groups under JIM management, making JIM authoritative for their membership.
**Systems**:
-- Source: Subatomic AD (existing groups in OU=Legacy Groups,OU=Groups,OU=Corp)
+- Source: Panoply AD (existing groups in OU=Legacy Groups,OU=Groups,OU=Corp)
- Target: JIM Metaverse (becomes authoritative after import)
-- Export Target: Subatomic AD (same groups, now JIM-managed)
+- Export Target: Panoply AD (same groups, now JIM-managed)
**Test Steps** (executed sequentially):
@@ -817,8 +839,8 @@ These scenarios test group management capabilities - a core ILM function where t
**Concept**: In multi-domain environments, groups may need to be replicated across domains. This scenario tests importing groups from AD1 (authoritative) and exporting them to AD2, ensuring AD2 groups mirror AD1.
**Systems**:
-- Source: Quantum Dynamics APAC (OU=Entitlements,OU=SourceCorp - authoritative for groups)
-- Target: Quantum Dynamics EMEA (OU=Entitlements,OU=TargetCorp - replica of source groups)
+- Source: Panoply APAC (OU=Entitlements,OU=SourceCorp - authoritative for groups)
+- Target: Panoply EMEA (OU=Entitlements,OU=TargetCorp - replica of source groups)
**Important**: Each domain uses dedicated OUs to avoid conflicts with other scenarios.
@@ -861,7 +883,7 @@ These scenarios test group management capabilities - a core ILM function where t
**Systems**:
- Source 1: SQL Server (HRIS System A - Business Unit A)
- Source 2: Oracle Database (HRIS System B - Business Unit B)
-- Target 1: Subatomic AD
+- Target 1: Panoply AD
- Target 2: CSV (Reporting)
**Test Steps** (executed sequentially):
@@ -1016,15 +1038,15 @@ $hrSystem = New-JIMConnectedSystem -Name "HR CSV" `
}
# Create Samba AD Connected System (Target)
-$adSystem = New-JIMConnectedSystem -Name "Subatomic AD" `
+$adSystem = New-JIMConnectedSystem -Name "Panoply AD" `
-ConnectorType "LDAP" `
-Configuration @{
Server = "samba-ad-primary"
Port = 389
- BaseDN = "DC=subatomic,DC=local"
- BindDN = "CN=Administrator,CN=Users,DC=subatomic,DC=local"
+ BaseDN = "DC=panoply,DC=local"
+ BindDN = "CN=Administrator,CN=Users,DC=panoply,DC=local"
BindPassword = "Test@123!"
- UserContainer = "OU=Users,OU=Corp,DC=subatomic,DC=local"
+ UserContainer = "OU=Users,OU=Corp,DC=panoply,DC=local"
}
# Create Inbound Sync Rule (HR -> Metaverse)
@@ -1762,12 +1784,12 @@ JIM/
| Service | Container Port | Host Port | Protocol |
|----------------------|----------------|-----------|----------|
-| Subatomic AD | 389 | 389 | LDAP |
-| Subatomic AD | 636 | 636 | LDAPS |
-| Quantum Dynamics APAC | 389 | 10389 | LDAP |
-| Quantum Dynamics APAC | 636 | 10636 | LDAPS |
-| Quantum Dynamics EMEA | 389 | 11389 | LDAP |
-| Quantum Dynamics EMEA | 636 | 11636 | LDAPS |
+| Panoply AD | 389 | 389 | LDAP |
+| Panoply AD | 636 | 636 | LDAPS |
+| Panoply APAC | 389 | 10389 | LDAP |
+| Panoply APAC | 636 | 10636 | LDAPS |
+| Panoply EMEA | 389 | 11389 | LDAP |
+| Panoply EMEA | 636 | 11636 | LDAPS |
| SQL Server | 1433 | 1433 | TCP |
| Oracle XE | 1521 | 1521 | TCP |
| PostgreSQL (Test) | 5432 | 5433 | TCP |
diff --git a/docs/SSO_SETUP_GUIDE.md b/docs/SSO_SETUP_GUIDE.md
index f5ac2cf0c..6f65ebf76 100644
--- a/docs/SSO_SETUP_GUIDE.md
+++ b/docs/SSO_SETUP_GUIDE.md
@@ -2,7 +2,19 @@
> Step-by-step instructions for configuring Single Sign-On with JIM
-This guide covers setting up JIM with three identity providers:
+## Development (Bundled Keycloak)
+
+The devcontainer ships a pre-configured Keycloak instance that starts automatically with `jim-stack`. **No SSO configuration is needed for local development.**
+
+- **Test users:** `admin` / `admin` and `user` / `user`
+- **Keycloak admin console:** `http://localhost:8181` (admin / admin)
+- **Realm:** `jim`
+
+SSO works out of the box — just run `jim-stack` and sign in. To use an external IdP instead, override the `JIM_SSO_*` variables in your `.env` file with your provider's settings (see below).
+
+---
+
+This guide covers setting up JIM with an external identity provider for production:
- [Microsoft Entra ID (Azure AD)](#microsoft-entra-id-azure-ad)
- [AD FS (Active Directory Federation Services)](#ad-fs-active-directory-federation-services)
- [Keycloak](#keycloak)
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-AppLayerComponents-key.svg b/docs/diagrams/images/dark/jim-structurizr-1-AppLayerComponents-key.svg
index 97793feba..1e7df2925 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-AppLayerComponents-key.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-AppLayerComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-AppLayerComponents.svg b/docs/diagrams/images/dark/jim-structurizr-1-AppLayerComponents.svg
index 2ad848569..fec73ae47 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-AppLayerComponents.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-AppLayerComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-ConnectorComponents-key.svg b/docs/diagrams/images/dark/jim-structurizr-1-ConnectorComponents-key.svg
index f49aa0613..c8b77dc0e 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-ConnectorComponents-key.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-ConnectorComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-ConnectorComponents.svg b/docs/diagrams/images/dark/jim-structurizr-1-ConnectorComponents.svg
index f5225d70f..ee5a98d40 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-ConnectorComponents.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-ConnectorComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-Containers-key.svg b/docs/diagrams/images/dark/jim-structurizr-1-Containers-key.svg
index c94ab6d4b..9d04413c5 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-Containers-key.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-Containers-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-Containers.svg b/docs/diagrams/images/dark/jim-structurizr-1-Containers.svg
index 652a52394..f6cac4347 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-Containers.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-Containers.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-SchedulerComponents-key.svg b/docs/diagrams/images/dark/jim-structurizr-1-SchedulerComponents-key.svg
index fffb9926a..552183655 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-SchedulerComponents-key.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-SchedulerComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-SchedulerComponents.svg b/docs/diagrams/images/dark/jim-structurizr-1-SchedulerComponents.svg
index 899a9bf93..6264572b5 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-SchedulerComponents.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-SchedulerComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-SystemContext-key.svg b/docs/diagrams/images/dark/jim-structurizr-1-SystemContext-key.svg
index 1876350e9..6921194ed 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-SystemContext-key.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-SystemContext-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-SystemContext.svg b/docs/diagrams/images/dark/jim-structurizr-1-SystemContext.svg
index f3c8fa974..893aaf688 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-SystemContext.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-SystemContext.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-WebAppComponents-key.svg b/docs/diagrams/images/dark/jim-structurizr-1-WebAppComponents-key.svg
index 9595f6106..be7eb5a20 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-WebAppComponents-key.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-WebAppComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-WebAppComponents.svg b/docs/diagrams/images/dark/jim-structurizr-1-WebAppComponents.svg
index 653e2a0a8..cd8c3d6af 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-WebAppComponents.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-WebAppComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-WorkerComponents-key.svg b/docs/diagrams/images/dark/jim-structurizr-1-WorkerComponents-key.svg
index fffb9926a..552183655 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-WorkerComponents-key.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-WorkerComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/dark/jim-structurizr-1-WorkerComponents.svg b/docs/diagrams/images/dark/jim-structurizr-1-WorkerComponents.svg
index 510beaa35..3634ec66e 100644
--- a/docs/diagrams/images/dark/jim-structurizr-1-WorkerComponents.svg
+++ b/docs/diagrams/images/dark/jim-structurizr-1-WorkerComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-AppLayerComponents-key.svg b/docs/diagrams/images/light/jim-structurizr-1-AppLayerComponents-key.svg
index 74f88266b..3f4c07385 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-AppLayerComponents-key.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-AppLayerComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-AppLayerComponents.svg b/docs/diagrams/images/light/jim-structurizr-1-AppLayerComponents.svg
index 2707c0e70..4a30904ac 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-AppLayerComponents.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-AppLayerComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-ConnectorComponents-key.svg b/docs/diagrams/images/light/jim-structurizr-1-ConnectorComponents-key.svg
index 594258d42..03ddc3cdf 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-ConnectorComponents-key.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-ConnectorComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-ConnectorComponents.svg b/docs/diagrams/images/light/jim-structurizr-1-ConnectorComponents.svg
index c9b1c7c9e..21128393b 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-ConnectorComponents.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-ConnectorComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-Containers-key.svg b/docs/diagrams/images/light/jim-structurizr-1-Containers-key.svg
index 7cfd1b53e..999715eca 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-Containers-key.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-Containers-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-Containers.svg b/docs/diagrams/images/light/jim-structurizr-1-Containers.svg
index 8ceadd1e8..be216b22b 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-Containers.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-Containers.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-SchedulerComponents-key.svg b/docs/diagrams/images/light/jim-structurizr-1-SchedulerComponents-key.svg
index d8f7652ee..3385935bc 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-SchedulerComponents-key.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-SchedulerComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-SchedulerComponents.svg b/docs/diagrams/images/light/jim-structurizr-1-SchedulerComponents.svg
index 70471f22c..baac400ff 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-SchedulerComponents.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-SchedulerComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-SystemContext-key.svg b/docs/diagrams/images/light/jim-structurizr-1-SystemContext-key.svg
index b3d91e676..4676b0c24 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-SystemContext-key.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-SystemContext-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-SystemContext.svg b/docs/diagrams/images/light/jim-structurizr-1-SystemContext.svg
index 7abd06704..e6795d7ce 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-SystemContext.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-SystemContext.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-WebAppComponents-key.svg b/docs/diagrams/images/light/jim-structurizr-1-WebAppComponents-key.svg
index 7ff184db5..346c9c0cd 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-WebAppComponents-key.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-WebAppComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-WebAppComponents.svg b/docs/diagrams/images/light/jim-structurizr-1-WebAppComponents.svg
index 1505952f1..0966b309c 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-WebAppComponents.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-WebAppComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-WorkerComponents-key.svg b/docs/diagrams/images/light/jim-structurizr-1-WorkerComponents-key.svg
index d8f7652ee..3385935bc 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-WorkerComponents-key.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-WorkerComponents-key.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/images/light/jim-structurizr-1-WorkerComponents.svg b/docs/diagrams/images/light/jim-structurizr-1-WorkerComponents.svg
index 718d0e659..747099e75 100644
--- a/docs/diagrams/images/light/jim-structurizr-1-WorkerComponents.svg
+++ b/docs/diagrams/images/light/jim-structurizr-1-WorkerComponents.svg
@@ -1 +1 @@
-
\ No newline at end of file
+
\ No newline at end of file
diff --git a/docs/diagrams/mermaid/ACTIVITY_AND_RPEI_FLOW.md b/docs/diagrams/mermaid/ACTIVITY_AND_RPEI_FLOW.md
index 00e6ab0df..7320a5e73 100644
--- a/docs/diagrams/mermaid/ACTIVITY_AND_RPEI_FLOW.md
+++ b/docs/diagrams/mermaid/ACTIVITY_AND_RPEI_FLOW.md
@@ -1,6 +1,6 @@
# Activity and RPEI Flow
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
This diagram shows how Activities are created, how Run Profile Execution Items (RPEIs) are accumulated during operations, and how the final activity status is determined. Activities are the immutable audit record for every operation in JIM.
diff --git a/docs/diagrams/mermaid/CONNECTOR_LIFECYCLE.md b/docs/diagrams/mermaid/CONNECTOR_LIFECYCLE.md
index 63ef0445e..3a46bc392 100644
--- a/docs/diagrams/mermaid/CONNECTOR_LIFECYCLE.md
+++ b/docs/diagrams/mermaid/CONNECTOR_LIFECYCLE.md
@@ -1,6 +1,6 @@
# Connector Lifecycle
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
This diagram shows how connectors are resolved, configured, opened, used, and closed across import and export operations. Connectors implement capability interfaces that determine their lifecycle shape.
@@ -39,10 +39,10 @@ flowchart TD
CreateFile --> GetRunProfile
GetRunProfile --> RouteByType{RunProfile
RunType?}
- RouteByType -->|FullImport
DeltaImport| ImportProcessor[SyncImportTaskProcessor]
- RouteByType -->|FullSynchronisation| FullSyncProcessor[SyncFullSyncTaskProcessor]
- RouteByType -->|DeltaSynchronisation| DeltaSyncProcessor[SyncDeltaSyncTaskProcessor]
- RouteByType -->|Export| ExportProcessor[SyncExportTaskProcessor]
+ RouteByType -->|FullImport
DeltaImport| ImportProcessor[SyncImportTaskProcessor
ISyncServer + ISyncRepository]
+ RouteByType -->|FullSynchronisation| FullSyncProcessor[SyncFullSyncTaskProcessor
ISyncEngine + ISyncServer + ISyncRepository]
+ RouteByType -->|DeltaSynchronisation| DeltaSyncProcessor[SyncDeltaSyncTaskProcessor
ISyncEngine + ISyncServer + ISyncRepository]
+ RouteByType -->|Export| ExportProcessor[SyncExportTaskProcessor
ISyncServer + ISyncRepository]
```
## Import Lifecycle
diff --git a/docs/diagrams/mermaid/DELTA_SYNC_FLOW.md b/docs/diagrams/mermaid/DELTA_SYNC_FLOW.md
index 9ebbd24b7..2ed97c6ca 100644
--- a/docs/diagrams/mermaid/DELTA_SYNC_FLOW.md
+++ b/docs/diagrams/mermaid/DELTA_SYNC_FLOW.md
@@ -1,6 +1,6 @@
# Delta Sync Flow
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
This diagram shows how Delta Synchronisation differs from Full Synchronisation. Both use identical per-CSO processing logic; the only difference is CSO selection and a few lifecycle steps.
@@ -81,7 +81,7 @@ flowchart TD
## Key Design Decisions
-- **Identical per-CSO logic**: Both full and delta sync share the exact same `ProcessConnectedSystemObjectAsync()` from `SyncTaskProcessorBase`. The only difference is which CSOs are selected for processing.
+- **Identical per-CSO logic**: Both full and delta sync share the exact same `ProcessConnectedSystemObjectAsync()` from `SyncTaskProcessorBase`, using `ISyncEngine` for pure domain decisions and `ISyncServer`/`ISyncRepository` for orchestration and data access. The only difference is which CSOs are selected for processing.
- **Early exit optimisation**: Delta sync checks if any CSOs have been modified before loading caches and entering the page loop. If nothing has changed, it updates the watermark and returns immediately.
diff --git a/docs/diagrams/mermaid/EXPORT_EXECUTION_FLOW.md b/docs/diagrams/mermaid/EXPORT_EXECUTION_FLOW.md
index e390d5a09..e2b5505c7 100644
--- a/docs/diagrams/mermaid/EXPORT_EXECUTION_FLOW.md
+++ b/docs/diagrams/mermaid/EXPORT_EXECUTION_FLOW.md
@@ -1,8 +1,8 @@
# Export Execution Flow
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
-This diagram shows how pending exports are executed against connected systems via connectors. The export processor (`SyncExportTaskProcessor`) delegates to the `ExportExecutionServer` for the core execution logic, which supports batching, parallelism, deferred reference resolution, and retry with backoff.
+This diagram shows how pending exports are executed against connected systems via connectors. The export processor (`SyncExportTaskProcessor`) uses `ISyncServer` to delegate to `ExportExecutionServer` for the core execution logic, and `ISyncRepository` for bulk data access. Supports batching, parallelism, deferred reference resolution, and retry with backoff.
## Export Task Processing
diff --git a/docs/diagrams/mermaid/FULL_IMPORT_FLOW.md b/docs/diagrams/mermaid/FULL_IMPORT_FLOW.md
index c1e6f811d..42ce33df4 100644
--- a/docs/diagrams/mermaid/FULL_IMPORT_FLOW.md
+++ b/docs/diagrams/mermaid/FULL_IMPORT_FLOW.md
@@ -1,9 +1,11 @@
# Full Import Flow
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
This diagram shows how objects are imported from a connected system into JIM's connector space. Both Full Import and Delta Import use the same processor (`SyncImportTaskProcessor`); the connector handles delta filtering internally via watermark/persisted data.
+Since v0.7.1, the import processor uses `ISyncServer` for orchestration (settings, caching, reconciliation) and `ISyncRepository` for dedicated bulk data access (CSO writes, RPEIs).
+
## Overall Import Flow
```mermaid
@@ -46,9 +48,9 @@ flowchart TD
DeletionDetection --> RefResolution[Reference Resolution
Resolve unresolved reference strings
into CSO links by external ID]
- %% --- Persist ---
- RefResolution --> PersistCreate[Batch create new CSOs
with change objects]
- PersistCreate --> PersistUpdate[Batch update existing CSOs
with change objects]
+ %% --- Persist via ISyncRepository ---
+ RefResolution --> PersistCreate[Batch create new CSOs
with change objects via ISyncRepository]
+ PersistCreate --> PersistUpdate[Batch update existing CSOs
with change objects via ISyncRepository]
%% --- Reconciliation ---
PersistUpdate --> Reconcile[Reconcile Pending Exports
See Confirming Import below]
diff --git a/docs/diagrams/mermaid/FULL_SYNC_CSO_PROCESSING.md b/docs/diagrams/mermaid/FULL_SYNC_CSO_PROCESSING.md
index 0ea20f600..f8f84022a 100644
--- a/docs/diagrams/mermaid/FULL_SYNC_CSO_PROCESSING.md
+++ b/docs/diagrams/mermaid/FULL_SYNC_CSO_PROCESSING.md
@@ -1,6 +1,6 @@
# Full Synchronisation - CSO Processing Flow
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
This diagram shows the core decision tree for processing a single Connected System Object (CSO) during Full or Delta Synchronisation. This is the central flow of JIM's identity management engine.
@@ -8,11 +8,16 @@ Both Full Sync and Delta Sync use identical processing logic per-CSO. The only d
- **Full Sync**: processes ALL CSOs in the Connected System
- **Delta Sync**: processes only CSOs modified since `LastDeltaSyncCompletedAt`
+Since v0.7.1, sync decisions are split across three layers:
+- **ISyncEngine** — Pure domain logic (projection, attribute flow, deletion rules, export confirmation). Stateless, I/O-free.
+- **ISyncServer** — Orchestration facade (matching, scoping, drift detection, export evaluation). Delegates to application-layer servers.
+- **ISyncRepository** — Dedicated data access (bulk CSO/MVO writes, pending exports, RPEIs).
+
## Overall Page Processing
```mermaid
flowchart TD
- Start([Start Sync]) --> Prepare[Prepare: count CSOs + pending exports
Load sync rules, object types
Build drift detection cache
Build export evaluation cache
Pre-load pending exports into dictionary]
+ Start([Start Sync]) --> Prepare[Prepare: count CSOs + pending exports
Load sync rules, object types via ISyncRepository
Build drift detection cache
Build export evaluation cache
Pre-load pending exports into dictionary]
Prepare --> PageLoop{More CSO
pages?}
PageLoop -->|Yes| LoadPage[Load page of CSOs
without attributes for performance]
@@ -45,7 +50,7 @@ This is the decision tree within `ProcessConnectedSystemObjectAsync` for a singl
```mermaid
flowchart TD
- Entry([ProcessConnectedSystemObjectAsync]) --> ConfirmPE[Confirm pending exports
Check if previously exported values
now match CSO attributes]
+ Entry([ProcessConnectedSystemObjectAsync]) --> ConfirmPE[Confirm pending exports
ISyncEngine.EvaluatePendingExportConfirmation
checks if exported values match CSO attributes]
ConfirmPE --> CheckObsolete{CSO status
= Obsolete?}
%% --- Obsolete CSO path ---
@@ -55,13 +60,13 @@ flowchart TD
CheckJoined -->|Yes| CheckOosAction{InboundOutOfScope
Action?}
CheckOosAction -->|RemainJoined| KeepJoin[Delete CSO but preserve
MVO join state
Once managed always managed]
- CheckOosAction -->|Disconnect| RemoveAttrs{RemoveContributed
AttributesOnObsoletion
enabled on object type?}
+ CheckOosAction -->|Disconnect| RemoveAttrs{ISyncEngine.DetermineOutOfScopeAction
RemoveContributed
AttributesOnObsoletion
enabled on object type?}
RemoveAttrs -->|Yes| RecallAttrs[Attribute Recall:
Find MVO attributes where
ContributedBySystemId = this system
Add to PendingAttributeValueRemovals
Track removedAttributes set]
RemoveAttrs -->|No| BreakJoin
RecallAttrs --> QueueRecall[Queue MVO for export evaluation
with removedAttributes set
Pure recalls skip export evaluation]
QueueRecall --> BreakJoin[Break CSO-MVO join
Set JoinType = NotJoined]
- BreakJoin --> EvalDeletion[Evaluate MVO deletion rule]
+ BreakJoin --> EvalDeletion[ISyncEngine.EvaluateMvoDeletionRule
Pure decision on MVO fate]
EvalDeletion --> DeletionRule{MVO deletion
rule?}
DeletionRule -->|Manual| NoDelete[No automatic deletion
MVO remains]
@@ -96,7 +101,7 @@ flowchart TD
CheckMvo -->|No| AttemptJoin[Attempt Join
For each import sync rule:
Find matching MVO by join criteria]
AttemptJoin --> JoinResult{Match
found?}
- JoinResult -->|No match| AttemptProject{Sync rule has
ProjectToMetaverse
= true?}
+ JoinResult -->|No match| AttemptProject{ISyncEngine.EvaluateProjection
Sync rule has
ProjectToMetaverse = true?}
AttemptProject -->|Yes| Project[Create new MVO
Set type from sync rule
Link CSO to new MVO]
AttemptProject -->|No| Done
@@ -107,10 +112,10 @@ flowchart TD
%% --- Attribute Flow path ---
EstablishJoin --> AttrFlow
Project --> AttrFlow
- CheckMvo -->|Yes| AttrFlow[Inbound Attribute Flow
Pass 1: scalar attributes only
For each sync rule mapping:
- Direct: CSO attr --> MVO attr
- Expression: evaluate --> MVO attr
- ContributedBySystemId set on all new values
Skip reference attributes]
+ CheckMvo -->|Yes| AttrFlow[ISyncEngine.FlowInboundAttributes
Pass 1: scalar attributes only
For each sync rule mapping:
- Direct: CSO attr --> MVO attr
- Expression: evaluate --> MVO attr
- ContributedBySystemId set on all new values
Skip reference attributes]
AttrFlow --> QueueRef[Queue CSO for deferred
reference attribute processing
Pass 2 at end of page]
- QueueRef --> ApplyChanges[Apply pending attribute
additions and removals to MVO]
+ QueueRef --> ApplyChanges[ISyncEngine.ApplyPendingAttributeChanges
Apply pending attribute
additions and removals to MVO]
ApplyChanges --> QueueMvo[Queue MVO for batch
persist and export evaluation]
QueueMvo --> DriftDetect[Drift Detection
Compare CSO values against
expected MVO state
Create corrective pending exports
for EnforceState export rules]
DriftDetect --> Result([Return change result:
Projected / Joined / AttributeFlow / NoChanges])
@@ -122,13 +127,15 @@ flowchart TD
## Key Design Decisions
-- **Two-pass attribute flow**: Scalar attributes are processed first (pass 1), then reference attributes are deferred to a second pass after all CSOs in the page have MVOs. This ensures group member references can resolve to MVOs that were created later in the same page.
+- **Three-layer sync architecture (v0.7.1)**: Sync decisions are split across `ISyncEngine` (pure domain logic — projection, attribute flow, deletion rules, export confirmation), `ISyncServer` (orchestration — matching, scoping, drift detection, export evaluation), and `ISyncRepository` (dedicated data access — bulk CSO/MVO writes, pending exports, RPEIs). This separation enables deterministic unit testing of business logic without I/O.
+
+- **Two-pass attribute flow**: Scalar attributes are processed first (pass 1 via `ISyncEngine.FlowInboundAttributes`), then reference attributes are deferred to a second pass after all CSOs in the page have MVOs. This ensures group member references can resolve to MVOs that were created later in the same page.
-- **Batch persistence**: MVO creates/updates, pending exports, and CSO deletions are all batched per-page to reduce database round trips. This is critical for performance at scale.
+- **Batch persistence**: MVO creates/updates, pending exports, and CSO deletions are all batched per-page via `ISyncRepository` bulk operations to reduce database round trips. This is critical for performance at scale.
- **No-net-change detection**: Before creating pending exports, the system checks if the target CSO already has the expected values (using pre-cached data). This avoids unnecessary export operations.
-- **Drift detection**: After inbound attribute flow, the system checks whether CSO values match expected MVO state. If an `EnforceState` export rule exists and the CSO has drifted, a corrective pending export is created.
+- **Drift detection**: After inbound attribute flow, `DriftDetectionService` checks whether CSO values match expected MVO state. If an `EnforceState` export rule exists and the CSO has drifted, a corrective pending export is created.
- **Attribute recall via ContributedBySystemId**: Every MVO attribute value tracks which connected system contributed it. When a CSO is obsoleted and `RemoveContributedAttributesOnObsoletion` is enabled on the object type, all attributes contributed by that system are recalled (removed from the MVO). The `removedAttributes` set is passed to export evaluation, where pure recall operations (all changes are removals) skip export evaluation entirely to avoid expression mapping errors against incomplete data.
diff --git a/docs/diagrams/mermaid/MVO_DELETION_AND_GRACE_PERIOD.md b/docs/diagrams/mermaid/MVO_DELETION_AND_GRACE_PERIOD.md
index 9d783d4d8..ee11ed1a5 100644
--- a/docs/diagrams/mermaid/MVO_DELETION_AND_GRACE_PERIOD.md
+++ b/docs/diagrams/mermaid/MVO_DELETION_AND_GRACE_PERIOD.md
@@ -1,6 +1,6 @@
# MVO Deletion and Grace Period
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
This diagram shows the full lifecycle of Metaverse Object (MVO) deletion, from the trigger event (CSO disconnection) through deletion rule evaluation, grace period handling, and deferred housekeeping cleanup.
@@ -32,14 +32,14 @@ flowchart TD
QueueRecall --> BreakJoin[Break CSO-MVO join
Set JoinType = NotJoined]
BreakJoin --> CountRemaining[Count remaining CSOs
before break, subtract 1]
- CountRemaining --> EvalDeletion[ProcessMvoDeletionRuleAsync]
+ CountRemaining --> EvalDeletion[ISyncEngine.EvaluateMvoDeletionRule
Pure decision on MVO fate]
```
## Deletion Rule Evaluation
```mermaid
flowchart TD
- Start([ProcessMvoDeletionRuleAsync]) --> CheckOrigin{MVO Origin?}
+ Start([ISyncEngine.EvaluateMvoDeletionRule]) --> CheckOrigin{MVO Origin?}
CheckOrigin -->|Internal| Protected([Skip - internal MVOs
protected from automatic deletion])
CheckOrigin -->|Projected| GetRule{Deletion
rule?}
diff --git a/docs/diagrams/mermaid/PENDING_EXPORT_LIFECYCLE.md b/docs/diagrams/mermaid/PENDING_EXPORT_LIFECYCLE.md
index bb4f7cce2..7763b3e06 100644
--- a/docs/diagrams/mermaid/PENDING_EXPORT_LIFECYCLE.md
+++ b/docs/diagrams/mermaid/PENDING_EXPORT_LIFECYCLE.md
@@ -1,6 +1,6 @@
# Pending Export Lifecycle
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
This diagram shows the full lifecycle of a Pending Export from creation during synchronisation, through export execution, to confirmation during a confirming import. Pending Exports are the mechanism by which JIM propagates changes from the metaverse to target connected systems.
@@ -101,7 +101,7 @@ flowchart LR
## Pending Export Confirmation During Sync
-During Full/Delta Sync, pending exports are also checked for confirmation (separate from the confirming import path above). This happens in `ProcessPendingExport` within `SyncTaskProcessorBase`:
+During Full/Delta Sync, pending exports are also checked for confirmation (separate from the confirming import path above). This uses `ISyncEngine.EvaluatePendingExportConfirmation` for the pure comparison logic, invoked from `SyncTaskProcessorBase`:
```mermaid
flowchart TD
diff --git a/docs/diagrams/mermaid/SCHEDULE_EXECUTION_LIFECYCLE.md b/docs/diagrams/mermaid/SCHEDULE_EXECUTION_LIFECYCLE.md
index 0e6fe7a30..15c0705e7 100644
--- a/docs/diagrams/mermaid/SCHEDULE_EXECUTION_LIFECYCLE.md
+++ b/docs/diagrams/mermaid/SCHEDULE_EXECUTION_LIFECYCLE.md
@@ -1,6 +1,6 @@
# Schedule Execution Lifecycle
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
This diagram shows how schedules are triggered, how step groups are queued and advanced, and how the scheduler and worker collaborate to drive multi-step execution to completion.
diff --git a/docs/diagrams/mermaid/WORKER_TASK_LIFECYCLE.md b/docs/diagrams/mermaid/WORKER_TASK_LIFECYCLE.md
index c7648ad34..032e5ad7d 100644
--- a/docs/diagrams/mermaid/WORKER_TASK_LIFECYCLE.md
+++ b/docs/diagrams/mermaid/WORKER_TASK_LIFECYCLE.md
@@ -1,6 +1,6 @@
# Worker Task Lifecycle
-> Generated against JIM v0.3.0 (`0d1c88e9`). If the codebase has changed significantly since then, these diagrams may be out of date.
+> Last updated: 2026-03-26 — JIM v0.7.1 (`00907431`)
This diagram shows how the JIM Worker service picks up, executes, and completes tasks. It covers the main polling loop, task dispatch, heartbeat management, cancellation handling, and housekeeping.
@@ -41,11 +41,11 @@ flowchart TD
## Task Execution (per spawned task)
-Each task runs in its own `Task.Run` with an isolated `JimApplication` and `JimDbContext` to avoid EF Core connection sharing issues.
+Each task runs in its own `Task.Run` with an isolated `JimApplication`, `JimDbContext`, `ISyncRepository`, and `ISyncServer` to avoid EF Core connection sharing issues. Sync/delta sync processors also receive a stateless `ISyncEngine` for pure domain decisions.
```mermaid
flowchart TD
- Spawned([Task.Run starts]) --> CreateJim[Create dedicated JimApplication
with fresh JimDbContext]
+ Spawned([Task.Run starts]) --> CreateJim[Create dedicated JimApplication
with fresh JimDbContext
Create ISyncRepository + ISyncServer
for this task's DbContext]
CreateJim --> ReRetrieve[Re-retrieve WorkerTask
using task-specific JimApplication
Avoid cross-instance issues]
ReRetrieve --> SetExecuted[Set Activity.Executed = UtcNow]
@@ -56,11 +56,11 @@ flowchart TD
ResolveConnector --> ResolveRP[Get RunProfile from
ConnectedSystem.RunProfiles]
ResolveRP --> RunType{RunProfile
RunType?}
- RunType -->|FullImport| FI[SyncImportTaskProcessor
PerformFullImportAsync]
+ RunType -->|FullImport| FI[SyncImportTaskProcessor
PerformFullImportAsync
Uses ISyncServer + ISyncRepository]
RunType -->|DeltaImport| DI[SyncImportTaskProcessor
PerformFullImportAsync
Connector handles delta filtering]
- RunType -->|FullSynchronisation| FS[SyncFullSyncTaskProcessor
PerformFullSyncAsync]
- RunType -->|DeltaSynchronisation| DS[SyncDeltaSyncTaskProcessor
PerformDeltaSyncAsync]
- RunType -->|Export| EX[SyncExportTaskProcessor
PerformExportAsync]
+ RunType -->|FullSynchronisation| FS[SyncFullSyncTaskProcessor
PerformFullSyncAsync
Uses ISyncEngine + ISyncServer + ISyncRepository]
+ RunType -->|DeltaSynchronisation| DS[SyncDeltaSyncTaskProcessor
PerformDeltaSyncAsync
Uses ISyncEngine + ISyncServer + ISyncRepository]
+ RunType -->|Export| EX[SyncExportTaskProcessor
PerformExportAsync
Uses ISyncServer + ISyncRepository]
FI --> CompleteActivity
DI --> CompleteActivity
@@ -136,7 +136,12 @@ flowchart TD
## Key Design Decisions
-- **Isolated DbContext per task**: Each task gets its own `JimApplication` and `JimDbContext` to avoid EF Core connection sharing issues. The main loop has its own instance for polling and heartbeats.
+- **Three-layer sync architecture**: Worker processors use three collaborating interfaces injected at task spawn time:
+ - **ISyncEngine** — Pure domain logic (projection decisions, attribute flow, deletion rules, export confirmation). Stateless, I/O-free, fully unit-testable. Used by full sync and delta sync processors.
+ - **ISyncServer** — Orchestration facade that delegates to existing application-layer servers (ExportEvaluationServer, ExportExecutionServer, ScopingEvaluationServer, DriftDetectionService) and ISyncRepository. All processors use this.
+ - **ISyncRepository** — Dedicated data access boundary for sync operations (bulk CSO/MVO writes, pending exports, RPEIs). Replaces scattered access through multiple server properties.
+
+- **Isolated DbContext per task**: Each task gets its own `JimApplication`, `JimDbContext`, `ISyncRepository`, and `ISyncServer` to avoid EF Core connection sharing issues. The main loop has its own instance for polling and heartbeats.
- **Heartbeat-based liveness**: Active tasks have their heartbeats updated every polling cycle (2 seconds). The scheduler uses heartbeat timestamps to detect crashed workers and recover stale tasks.
diff --git a/docs/diagrams/structurizr/README.md b/docs/diagrams/structurizr/README.md
index ec8d9cd30..ca20dce7f 100644
--- a/docs/diagrams/structurizr/README.md
+++ b/docs/diagrams/structurizr/README.md
@@ -1,5 +1,7 @@
# JIM C4 Architecture Diagrams
+> **Last Updated:** 2026-03-26 (JIM v0.7.1)
+
This folder contains the C4 model architecture diagrams for JIM, defined using [Structurizr DSL](https://docs.structurizr.com/dsl).
## Diagrams
@@ -111,18 +113,20 @@ From Structurizr Lite, you can export diagrams as:
## Viewing Interactive Diagrams
-### Using Structurizr Lite (Local Docker)
+### Using Structurizr Local (Docker)
+
+> **Note:** `structurizr/lite` is deprecated and no longer starts. Use `structurizr/structurizr` with the `local` command instead.
-1. Start Structurizr Lite from the repository root:
+1. Start Structurizr Local from the repository root:
```powershell
# PowerShell (Windows/Linux/macOS)
- docker run -it --rm -p 8085:8080 -v ${PWD}/docs/diagrams/structurizr:/usr/local/structurizr structurizr/lite
+ docker run -it --rm -p 8085:8080 -v ${PWD}/docs/diagrams/structurizr:/usr/local/structurizr structurizr/structurizr local
```
```bash
# Bash
- docker run -it --rm -p 8085:8080 -v $(pwd)/docs/diagrams/structurizr:/usr/local/structurizr structurizr/lite
+ docker run -it --rm -p 8085:8080 -v $(pwd)/docs/diagrams/structurizr:/usr/local/structurizr structurizr/structurizr local
```
2. Open http://localhost:8085 in your browser
@@ -147,8 +151,8 @@ The workspace contains C4 diagrams at three levels:
### Level 3: Component
- **WebAppComponents** - Blazor Pages, API Controllers, Authentication Middleware
-- **AppLayerComponents** - JimApplication Facade and domain services
-- **WorkerComponents** - Worker Host and task processors
+- **AppLayerComponents** - JimApplication Facade, SyncEngine (pure domain logic), SyncServer (worker orchestration), domain services, IJimRepository, ISyncRepository
+- **WorkerComponents** - Worker Host and task processors (using ISyncEngine, ISyncServer, ISyncRepository)
- **ConnectorComponents** - LDAP and File connector implementations
- **SchedulerComponents** - Scheduler Host
diff --git a/docs/diagrams/structurizr/workspace.dsl b/docs/diagrams/structurizr/workspace.dsl
index 66bc6b0b4..b77cc32d2 100644
--- a/docs/diagrams/structurizr/workspace.dsl
+++ b/docs/diagrams/structurizr/workspace.dsl
@@ -31,13 +31,16 @@ workspace "JIM Identity Management System" "C4 model for JIM - a central identit
}
appLayer = container "Application Layer" "Business logic and domain services" "JIM.Application" {
- jimApplication = component "JimApplication Facade" "Single entry point to all domain services" "C# Facade Class"
+ jimApplication = component "JimApplication Facade" "Single entry point to all domain services for Web and Scheduler" "C# Facade Class"
+ syncEngine = component "SyncEngine" "Pure domain logic for sync decisions - projection, attribute flow, deletion rules, export confirmation. Stateless, I/O-free" "C# Service"
+ syncServer = component "SyncServer" "Orchestration facade for Worker processors - delegates to domain servers and ISyncRepository" "C# Service"
metaverseServer = component "MetaverseServer" "Metaverse object CRUD, querying, attribute management" "C# Service"
connectedSystemServer = component "ConnectedSystemServer" "Connected system lifecycle, configuration, run profiles, sync rules, and attribute mappings" "C# Service"
objectMatchingServer = component "ObjectMatchingServer" "Join logic between ConnectedSystemObjects and MetaverseObjects" "C# Service"
exportEvaluationServer = component "ExportEvaluationServer" "Determines pending exports based on attribute changes" "C# Service"
scopingEvaluationServer = component "ScopingEvaluationServer" "Evaluates sync rule scoping filters" "C# Service"
- exportExecutionServer = component "ExportExecutionServer" "Executes pending exports with retry logic" "C# Service"
+ exportExecutionServer = component "ExportExecutionServer" "Executes pending exports with retry logic and parallel batching" "C# Service"
+ driftDetectionService = component "DriftDetectionService" "Detects target system drift from authoritative MVO state, creates corrective pending exports" "C# Service"
schedulerServer = component "SchedulerServer" "Schedule management, due time evaluation, execution advancement, crash recovery" "C# Service"
searchServer = component "SearchServer" "Metaverse search and query functionality" "C# Service"
securityServer = component "SecurityServer" "Role-based access control and user-to-role assignments" "C# Service"
@@ -46,7 +49,10 @@ workspace "JIM Identity Management System" "C4 model for JIM - a central identit
serviceSettingsServer = component "ServiceSettingsServer" "Global service configuration management" "C# Service"
activityServer = component "ActivityServer" "Activity logging, audit trail, execution statistics" "C# Service"
taskingServer = component "TaskingServer" "Worker task queue management" "C# Service"
+ fileSystemServer = component "FileSystemServer" "File system browsing for connector file path configuration" "C# Service"
+ exampleDataServer = component "ExampleDataServer" "Example and test data generation management" "C# Service"
repository = component "IJimRepository" "Data access abstraction - interfaces defined in JIM.Data, implemented by JIM.PostgresData (EF Core)" "Repository Interface"
+ syncRepository = component "ISyncRepository" "Dedicated data access for sync operations - bulk CSO/MVO writes, pending exports, RPEIs. Implemented by PostgresData.SyncRepository" "Repository Interface"
}
worker = container "Worker Service" "Processes queued synchronisation tasks - import, sync, export operations" ".NET 9.0 Background Service" {
@@ -70,7 +76,7 @@ workspace "JIM Identity Management System" "C4 model for JIM - a central identit
database = container "PostgreSQL Database" "Stores configuration, metaverse objects, staging area, sync rules, activity history, task queue" "PostgreSQL 18" "Database"
- pwsh = container "PowerShell Module" "Cross-platform module with 64 cmdlets for automation and scripting" "PowerShell 7" "Client Library"
+ pwsh = container "PowerShell Module" "Cross-platform module with 79 cmdlets for automation and scripting" "PowerShell 7" "Client Library"
!docs docs
!adrs adrs
@@ -120,12 +126,15 @@ workspace "JIM Identity Management System" "C4 model for JIM - a central identit
jim.webApp.apiControllers -> jim.appLayer.jimApplication "Uses" "Method calls"
# ===== Application Layer Component Relationships =====
+
+ # JimApplication facade delegates to domain servers (used by Web and Scheduler)
jim.appLayer.jimApplication -> jim.appLayer.metaverseServer "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.connectedSystemServer "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.objectMatchingServer "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.exportEvaluationServer "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.scopingEvaluationServer "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.exportExecutionServer "Delegates to" "Method calls"
+ jim.appLayer.jimApplication -> jim.appLayer.driftDetectionService "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.schedulerServer "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.searchServer "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.securityServer "Delegates to" "Method calls"
@@ -134,12 +143,25 @@ workspace "JIM Identity Management System" "C4 model for JIM - a central identit
jim.appLayer.jimApplication -> jim.appLayer.serviceSettingsServer "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.activityServer "Delegates to" "Method calls"
jim.appLayer.jimApplication -> jim.appLayer.taskingServer "Delegates to" "Method calls"
+ jim.appLayer.jimApplication -> jim.appLayer.fileSystemServer "Delegates to" "Method calls"
+ jim.appLayer.jimApplication -> jim.appLayer.exampleDataServer "Delegates to" "Method calls"
+
+ # SyncServer orchestration (used by Worker processors)
+ jim.appLayer.syncServer -> jim.appLayer.exportEvaluationServer "Delegates to" "Method calls"
+ jim.appLayer.syncServer -> jim.appLayer.exportExecutionServer "Delegates to" "Method calls"
+ jim.appLayer.syncServer -> jim.appLayer.scopingEvaluationServer "Delegates to" "Method calls"
+ jim.appLayer.syncServer -> jim.appLayer.driftDetectionService "Delegates to" "Method calls"
+ jim.appLayer.syncServer -> jim.appLayer.syncRepository "Uses" "ISyncRepository"
+ # Cross-server dependencies
jim.appLayer.objectMatchingServer -> jim.appLayer.metaverseServer "Uses" "Method calls"
jim.appLayer.exportEvaluationServer -> jim.appLayer.connectedSystemServer "Uses" "Method calls"
jim.appLayer.exportExecutionServer -> jim.appLayer.metaverseServer "Uses" "Method calls"
jim.appLayer.exportExecutionServer -> jim.appLayer.connectedSystemServer "Uses" "Method calls"
+ jim.appLayer.exportEvaluationServer -> jim.appLayer.syncRepository "Uses" "ISyncRepository"
+ jim.appLayer.exportExecutionServer -> jim.appLayer.syncRepository "Uses" "ISyncRepository"
+ # IJimRepository consumers (general data access)
jim.appLayer.metaverseServer -> jim.appLayer.repository "Uses" "IJimRepository"
jim.appLayer.connectedSystemServer -> jim.appLayer.repository "Uses" "IJimRepository"
jim.appLayer.schedulerServer -> jim.appLayer.repository "Uses" "IJimRepository"
@@ -151,7 +173,9 @@ workspace "JIM Identity Management System" "C4 model for JIM - a central identit
jim.appLayer.activityServer -> jim.appLayer.repository "Uses" "IJimRepository"
jim.appLayer.taskingServer -> jim.appLayer.repository "Uses" "IJimRepository"
+ # Data access to database
jim.appLayer.repository -> jim.database "Reads/Writes" "EF Core (JIM.PostgresData)"
+ jim.appLayer.syncRepository -> jim.database "Reads/Writes" "EF Core (JIM.PostgresData)"
# ===== Worker Component Relationships =====
jim.worker.workerHost -> jim.appLayer.jimApplication "Polls for tasks" "Method calls"
@@ -160,10 +184,16 @@ workspace "JIM Identity Management System" "C4 model for JIM - a central identit
jim.worker.workerHost -> jim.worker.deltaSyncProcessor "Dispatches" "Method calls"
jim.worker.workerHost -> jim.worker.exportProcessor "Dispatches" "Method calls"
- jim.worker.importProcessor -> jim.appLayer.jimApplication "Uses" "Method calls"
- jim.worker.fullSyncProcessor -> jim.appLayer.jimApplication "Uses" "Method calls"
- jim.worker.deltaSyncProcessor -> jim.appLayer.jimApplication "Uses" "Method calls"
- jim.worker.exportProcessor -> jim.appLayer.jimApplication "Uses" "Method calls"
+ jim.worker.importProcessor -> jim.appLayer.syncServer "Uses" "ISyncServer"
+ jim.worker.importProcessor -> jim.appLayer.syncRepository "Uses" "ISyncRepository"
+ jim.worker.fullSyncProcessor -> jim.appLayer.syncEngine "Uses" "ISyncEngine"
+ jim.worker.fullSyncProcessor -> jim.appLayer.syncServer "Uses" "ISyncServer"
+ jim.worker.fullSyncProcessor -> jim.appLayer.syncRepository "Uses" "ISyncRepository"
+ jim.worker.deltaSyncProcessor -> jim.appLayer.syncEngine "Uses" "ISyncEngine"
+ jim.worker.deltaSyncProcessor -> jim.appLayer.syncServer "Uses" "ISyncServer"
+ jim.worker.deltaSyncProcessor -> jim.appLayer.syncRepository "Uses" "ISyncRepository"
+ jim.worker.exportProcessor -> jim.appLayer.syncServer "Uses" "ISyncServer"
+ jim.worker.exportProcessor -> jim.appLayer.syncRepository "Uses" "ISyncRepository"
# ===== Scheduler Component Relationships =====
jim.scheduler.schedulerHost -> jim.appLayer.jimApplication "Evaluates schedules and creates tasks" "Method calls"
diff --git a/docs/notes/RFC4533_LDAP_CONTENT_SYNC.md b/docs/notes/RFC4533_LDAP_CONTENT_SYNC.md
new file mode 100644
index 000000000..9cb52e4a9
--- /dev/null
+++ b/docs/notes/RFC4533_LDAP_CONTENT_SYNC.md
@@ -0,0 +1,85 @@
+# RFC 4533 — LDAP Content Synchronisation (syncrepl)
+
+## Status: Research Note (2026-04-01)
+
+Institutional knowledge captured during OpenLDAP import concurrency work. This documents a potential future evolution of JIM's OpenLDAP import strategy from the current Pattern B (connection-per-combo parallelism) to native LDAP Content Sync. No backlog issue — this is a reference note, not a commitment.
+
+## Context
+
+OpenLDAP's RFC 2696 Simple Paged Results implementation has a connection-scoped paging cookie limitation: any new search on the same connection invalidates all outstanding paging cursors. JIM currently works around this by giving each container+objectType combo its own dedicated LdapConnection and running them in parallel (Pattern B, implemented in `LdapConnectorImport.GetFullImportObjectsParallel`).
+
+Pattern B is the industry-standard approach and performs well for typical deployments (2-12 combos). However, RFC 4533 would be the ideal long-term solution because it eliminates paging entirely and provides native change tracking.
+
+## What is RFC 4533?
+
+RFC 4533 defines the LDAP Content Synchronisation Operation (syncrepl). It is OpenLDAP's native replication protocol, designed specifically for keeping a consumer in sync with a provider.
+
+**Two modes:**
+- **refreshOnly** — polling mode. Consumer sends a Sync Request with an optional cookie. Provider returns all content (if no cookie) or only changes since the cookie (if cookie provided). Functionally equivalent to full import + delta import without paging.
+- **refreshAndPersist** — streaming mode. After the initial refresh phase, the provider keeps the connection open and pushes changes in real-time. This would map to a "live sync" capability.
+
+**Key advantages over current approach:**
+- Initial `refreshOnly` with no cookie performs a full content transfer without paging cookies — completely sidesteps the RFC 2696 limitation
+- Subsequent calls with a sync cookie return only incremental changes (adds, modifies, deletes) — a natural delta import that is more reliable than accesslog parsing
+- The server manages the change tracking state, not the client — eliminates the accesslog watermark complexity and the `olcSizeLimit` edge cases we handle today
+- No connection pooling needed — a single connection handles the full sync session
+
+## LDAP Protocol Details
+
+**Request control OID:** `1.3.6.1.4.1.4203.1.9.1.1` (Sync Request)
+**Response control OIDs:**
+- `1.3.6.1.4.1.4203.1.9.1.2` (Sync State) — attached to each entry, indicates add/modify/delete/present
+- `1.3.6.1.4.1.4203.1.9.1.3` (Sync Done) — attached to the final SearchResultDone message, contains the updated sync cookie
+
+**Message flow (refreshOnly):**
+```
+Client Server
+ | |
+ |--- SearchRequest ------------>| (with Sync Request control, mode=refreshOnly)
+ | |
+ |<-- SearchResultEntry ---------| (with Sync State control: add/modify)
+ |<-- SearchResultEntry ---------| (with Sync State control: add/modify)
+ | ... |
+ |<-- SearchResultDone ----------| (with Sync Done control: new cookie)
+ | |
+```
+
+**Sync cookie:** Opaque blob managed by the server. Contains enough state for the server to determine what has changed since the cookie was issued. Must be stored by JIM between imports (maps naturally to `PersistedConnectorData`).
+
+## Implementation Effort
+
+**The main barrier is the lack of .NET library support.** `System.DirectoryServices.Protocols` does not implement RFC 4533 natively. Implementations exist in:
+- **Java**: Ldaptive, Apache Directory LDAP API, LSC Project
+- **Python**: python-ldap syncrepl module
+- **Go**: Various in-progress implementations
+
+**What JIM would need to build:**
+1. Custom `DirectoryControl` subclasses for Sync Request, Sync State, and Sync Done controls
+2. BER encoding/decoding for the control values (ASN.1 structures defined in RFC 4533 Section 2)
+3. Cookie storage and lifecycle management (straightforward — maps to existing `PersistedConnectorData`)
+4. Entry processing that handles the four Sync State modes: add, modify, delete, present
+
+**Estimated complexity:** Medium-high. The BER encoding/decoding is the hardest part — the rest maps cleanly to existing JIM import abstractions. Could reference the Ldaptive Java implementation as a guide.
+
+## Who Uses syncrepl in Production?
+
+- **OpenLDAP's own multi-master replication** — syncrepl is the foundation of OpenLDAP's replication architecture
+- **LSC Project** (LDAP Synchronization Connector) — uses syncrepl as its preferred sync mechanism for OpenLDAP sources
+- **FreeIPA** — uses syncrepl internally for 389 DS replication
+- **Keycloak** — does NOT use syncrepl; uses Simple Paged Results with dedicated connections (same pattern as JIM's Pattern B)
+
+## When Would This Be Worth Implementing?
+
+Consider if any of these become true:
+- Customers report that accesslog-based delta imports are unreliable at scale (olcSizeLimit edge cases, accesslog database corruption)
+- Customers need real-time sync capabilities (refreshAndPersist mode)
+- A .NET library adds RFC 4533 support, eliminating the custom BER encoding work
+- OpenLDAP becomes a primary target directory (rather than secondary to AD)
+
+## References
+
+- [RFC 4533 — LDAP Content Synchronization Operation](https://datatracker.ietf.org/doc/html/rfc4533)
+- [RFC 2696 — LDAP Simple Paged Results](https://www.rfc-editor.org/rfc/rfc2696.html)
+- [Ldaptive SyncRepl implementation (Java)](https://www.ldaptive.org/docs/guide/operations/search.html#sync-repl)
+- [OpenLDAP Replication documentation](https://www.openldap.org/doc/admin26/replication.html)
+- [LSC Project syncrepl source connector](https://www.lsc-project.org/documentation/latest/about.html)
diff --git a/docs/notes/done/DRIFT_DETECTION_SPURIOUS_MEMBER_REMOVAL.md b/docs/notes/done/DRIFT_DETECTION_SPURIOUS_MEMBER_REMOVAL.md
index 85c282a85..d71b44be2 100644
--- a/docs/notes/done/DRIFT_DETECTION_SPURIOUS_MEMBER_REMOVAL.md
+++ b/docs/notes/done/DRIFT_DETECTION_SPURIOUS_MEMBER_REMOVAL.md
@@ -28,8 +28,8 @@ have significantly more members.
Two LDAP (Active Directory) connected systems in a cross-domain entitlement sync:
-- **Source CS** ("Quantum Dynamics APAC") - Authoritative source of users and groups
-- **Target CS** ("Quantum Dynamics EMEA") - Receives users and groups from Source
+- **Source CS** ("Panoply APAC") - Authoritative source of users and groups
+- **Target CS** ("Panoply EMEA") - Receives users and groups from Source
### Sync Rules (6 total)
diff --git a/docs/notes/done/LDAP_EXPORT_EMPTY_DN_ON_ATTRIBUTE_RECALL.md b/docs/notes/done/LDAP_EXPORT_EMPTY_DN_ON_ATTRIBUTE_RECALL.md
index 6cd5b8dac..89140154f 100644
--- a/docs/notes/done/LDAP_EXPORT_EMPTY_DN_ON_ATTRIBUTE_RECALL.md
+++ b/docs/notes/done/LDAP_EXPORT_EMPTY_DN_ON_ATTRIBUTE_RECALL.md
@@ -13,7 +13,7 @@ LDAP export fails during the leaver scenario with:
```
System.DirectoryServices.Protocols.DirectoryOperationException:
The distinguished name contains invalid syntax.
-Empty RDN value on OU=,OU=Users,OU=Corp,DC=subatomic,DC=local not permitted!
+Empty RDN value on OU=,OU=Users,OU=Corp,DC=panoply,DC=local not permitted!
```
The export is an **Update** operation (not a Delete), and it attempts a ModifyDN (rename/move) with an invalid target DN containing an empty `OU=` component.
@@ -25,7 +25,7 @@ The export is an **Update** operation (not a Delete), and it attempts a ModifyDN
547a7773-... (Update)
[WRN] MarkExportFailed: Export 547a7773-... failed (attempt 1/5).
Error: The distinguished name contains invalid syntax.
- Empty RDN value on OU=,OU=Users,OU=Corp,DC=subatomic,DC=local not permitted!
+ Empty RDN value on OU=,OU=Users,OU=Corp,DC=panoply,DC=local not permitted!
```
## Root Cause
@@ -46,7 +46,7 @@ CSV Delta Sync
--> MVO is still "in scope" (no scoping filter, type still matches)
--> Creates UPDATE PendingExport with attribute changes
--> DN expression evaluates with null department:
- "CN=User,OU=,OU=Users,OU=Corp,DC=subatomic,DC=local"
+ "CN=User,OU=,OU=Users,OU=Corp,DC=panoply,DC=local"
^^ empty!
--> Step 2: EvaluateOutOfScopeExportsAsync()
--> MVO is still in scope (no scoping filter) --> no-op
diff --git a/docs/plans/doing/GUID_UUID_HANDLING.md b/docs/plans/doing/GUID_UUID_HANDLING.md
index eb361dea9..ed6915f08 100644
--- a/docs/plans/doing/GUID_UUID_HANDLING.md
+++ b/docs/plans/doing/GUID_UUID_HANDLING.md
@@ -1,7 +1,7 @@
# GUID/UUID Handling Strategy
-- **Status:** Doing (Phases 1–3 complete)
-- **Last Updated**: 2026-01-28
+- **Status:** Doing (Phases 1–4 complete)
+- **Last Updated**: 2026-03-30
- **Milestone**: Pre-connector expansion (before SCIM, database, or web service connectors)
## Overview
@@ -225,41 +225,86 @@ Consider replacing `CsvReader.GetField()` with `IdentifierParser.TryFromSt
---
-### Phase 4: OpenLDAP/entryUUID Support (When OpenLDAP Export Needed)
+### Phase 4: OpenLDAP/entryUUID Support ✅
-**4.1 Add `entryUUID` attribute handling to LDAP import**
+Implemented as part of OpenLDAP integration support.
-Detect non-AD directories (already done via RootDSE) and read `entryUUID` as a string attribute using `Guid.TryParse()` or `IdentifierParser.FromString()` instead of binary conversion.
+**4.1 Add `entryUUID` attribute handling to LDAP import** ✅
-**4.2 Add byte order awareness to LDAP export**
+- RootDSE detection distinguishes AD/Samba from OpenLDAP/Generic directories
+- `entryUUID` declared as the external ID attribute for OpenLDAP and Generic directory types in `LdapConnectorRootDse.cs`
+- Schema parser (`Rfc4512SchemaParser.cs`) recognises `entryUUID` and maps it to `AttributeDataType.Text`
+- Import reads `entryUUID` as a string attribute, parsed via `IdentifierParser.FromString()`
+- Delta import via accesslog extracts `entryUUID` from both `reqEntryUUID` and `reqOld` attributes
-When exporting GUID-type attributes to non-AD directories, use `IdentifierParser.ToRfc4122Bytes()` instead of `Guid.ToByteArray()`. Determine which method to use based on the directory type detected during connection.
+**4.2 Add byte order awareness to LDAP export** ✅
-**4.3 Add `GuidByteOrder` metadata to connector configuration**
+- Post-create fetch correctly handles `entryUUID` as a string for non-AD directories
+- AD/Samba export continues to use `IdentifierParser.FromMicrosoftBytes()` for binary `objectGUID`
+- **Note:** Binary RFC 4122 UUID export for custom attributes deferred to Phase 5 — OpenLDAP's standard identifier (`entryUUID`) is string-based, so no binary byte order conversion is needed in practice
+
+**4.3 `GuidByteOrder` metadata** — deferred to Phase 5
+
+Not needed for OpenLDAP (string-based identifiers). Will be implemented when SQL/database connectors introduce binary UUID columns that require explicit byte order configuration.
+
+---
+
+### Phase 5: SQL/Database Connector Binary UUID Support (When Database Connectors Built)
+
+Database connectors that store UUIDs in binary columns require byte order awareness. The `IdentifierParser` utility already provides `FromRfc4122Bytes()` and `ToRfc4122Bytes()`, but the connector layer needs metadata to determine which byte order a given target uses.
+
+**5.1 Add `GuidByteOrder` metadata to connector configuration**
```
public enum GuidByteOrder
{
String, // No binary handling needed (CSV, SCIM, most APIs)
- MicrosoftNative, // AD, SQL Server
- Rfc4122 // OpenLDAP, PostgreSQL binary, Oracle
+ MicrosoftNative, // AD, SQL Server uniqueidentifier
+ Rfc4122 // PostgreSQL uuid binary, Oracle RAW(16)
}
```
-Store as connector-level metadata so the export path knows which byte order to use without per-attribute decisions.
+Store as connector-level metadata so the import/export paths know which byte order to use without per-attribute decisions.
+
+**5.2 PostgreSQL direct connector UUID handling**
+
+When importing/exporting `uuid` columns via a PostgreSQL database connector:
+- Import: Use `IdentifierParser.FromRfc4122Bytes()` for binary UUID values
+- Export: Use `IdentifierParser.ToRfc4122Bytes()` for writing binary UUIDs
+- String representation: No conversion needed (standard hyphenated format)
+
+**5.3 Oracle connector RAW(16) handling**
+
+When importing/exporting `RAW(16)` columns via an Oracle database connector:
+- Import: Use `IdentifierParser.FromRfc4122Bytes()` — Oracle uses big-endian (RFC 4122-like) byte order
+- Export: Use `IdentifierParser.ToRfc4122Bytes()` for writing RAW(16) values
+
+**5.4 SQL Server connector uniqueidentifier handling**
+
+When importing/exporting `uniqueidentifier` columns via a SQL Server database connector:
+- Import: Use `IdentifierParser.FromMicrosoftBytes()` — SQL Server uses Microsoft byte order (same as .NET `Guid`)
+- Export: Use `Guid.ToByteArray()` / `IdentifierParser.ToMicrosoftBytes()` — no conversion needed
+
+**5.5 MySQL connector UUID handling**
+
+MySQL stores UUIDs in varying formats:
+- `CHAR(36)`: String representation — no binary handling needed
+- `BINARY(16)`: Byte order varies by application convention — use `GuidByteOrder` metadata to determine the correct conversion
---
-### Phase 5: Cross-Connector Round-Trip Tests (Before New Connectors)
+### Phase 6: Cross-Connector Round-Trip Tests (After Database Connectors)
-**5.1 Add round-trip integration tests**
+**6.1 Add round-trip integration tests**
Verify that a GUID imported from one connector type survives storage in JIM and export to another connector type:
-- LDAP import (binary) -> store in PostgreSQL -> CSV export (string) -> CSV import (string) -> LDAP export (binary): original bytes preserved
+- LDAP import (binary, Microsoft byte order) -> store in PostgreSQL -> CSV export (string) -> CSV import (string) -> LDAP export (binary): original bytes preserved
+- PostgreSQL direct import (RFC 4122 binary) -> store in JIM -> LDAP export (Microsoft binary): byte order correctly swapped
+- Oracle RAW(16) import -> store in JIM -> SQL Server export: byte order correctly converted
- Known GUID value: verify specific byte sequences at each stage
-**5.2 Add SCIM identifier tests (when SCIM connector built)**
+**6.2 Add SCIM identifier tests (when SCIM connector built)**
- SCIM `id` stored as opaque string (not forced to Guid)
- SCIM `externalId` populated with JIM's MVO ID
@@ -270,12 +315,14 @@ Verify that a GUID imported from one connector type survives storage in JIM and
## Success Criteria
-1. All `new Guid(byte[])` and `Guid.ToByteArray()` calls have documented byte order assumptions
-2. `IdentifierParser` utility exists with comprehensive unit tests
-3. All connector code uses `IdentifierParser` instead of inline GUID operations
-4. GUID round-trip tests pass across all connector combinations
-5. OpenLDAP `entryUUID` is supported when that connector path is implemented
-6. No GUID-related data corruption when mixing connector types
+1. ✅ All `new Guid(byte[])` and `Guid.ToByteArray()` calls have documented byte order assumptions
+2. ✅ `IdentifierParser` utility exists with comprehensive unit tests
+3. ✅ All connector code uses `IdentifierParser` instead of inline GUID operations
+4. ✅ OpenLDAP `entryUUID` is supported for import and export
+5. `GuidByteOrder` metadata allows connectors to declare their binary UUID format
+6. Database connectors (PostgreSQL, Oracle, SQL Server, MySQL) use correct byte order conversions
+7. GUID round-trip tests pass across all connector combinations
+8. No GUID-related data corruption when mixing connector types
---
diff --git a/docs/plans/doing/MVO_COPY_BINARY_PERSISTENCE.md b/docs/plans/doing/MVO_COPY_BINARY_PERSISTENCE.md
new file mode 100644
index 000000000..970c2aa09
--- /dev/null
+++ b/docs/plans/doing/MVO_COPY_BINARY_PERSISTENCE.md
@@ -0,0 +1,150 @@
+# MVO COPY Binary Persistence
+
+- **Status:** Doing (creates complete, updates deferred)
+- **Milestone**: v0.9-STABILISATION
+- **GitHub Issue**: [#436](https://github.com/TetronIO/JIM/issues/436)
+- **Parent**: [#338](https://github.com/TetronIO/JIM/issues/338) (closed — Phases 1–6 creates complete)
+- **Related**: [`docs/plans/done/WORKER_DATABASE_PERFORMANCE_OPTIMISATION.md`](../done/WORKER_DATABASE_PERFORMANCE_OPTIMISATION.md)
+- **Created**: 2026-03-27
+
+## Overview
+
+Convert MVO (Metaverse Object) persistence from EF Core `AddRange`/`SaveChangesAsync` to COPY binary and raw SQL, mirroring the proven pattern used for CSO persistence in #338 Phase 3.
+
+MVO persistence was the **last major hot path still using EF Core's per-row SQL generation** in the sync flush pipeline. CSO creates, RPEI inserts, and sync outcome inserts already use COPY binary via `ParallelBatchWriter`.
+
+## Business Value
+
+- **Eliminates the last single-connection bottleneck** in the sync flush — MVO creates now use COPY binary, matching CSO/RPEI throughput
+- **Consistent architecture** — all bulk write hot paths use the same COPY binary pattern
+- **Expected 5–20x improvement** in MVO create throughput (based on CSO COPY binary results)
+
+---
+
+## MVO Creates ✅
+
+Convert `CreateMetaverseObjectsAsync` to use COPY binary, following the exact pattern from `SyncRepository.CsOperations.cs`.
+
+**Implementation** (`SyncRepository.MvoOperations.cs`):
+
+- `CreateMetaverseObjectsBulkAsync` — entry point with ID pre-generation, intra-batch reference fixup, route selection
+- `CreateMvosOnSingleConnectionAsync` — single-connection fallback for small batches
+- `BulkInsertMvosOnConnectionAsync` — COPY binary for `MetaverseObjects` (10 columns, `xmin` excluded)
+- `BulkInsertMvoAttributeValuesOnConnectionAsync` — COPY binary for `MetaverseObjectAttributeValues` (13 columns)
+- `BulkInsertMvosViaEfAsync` — parameterised multi-row INSERT fallback (single-connection path)
+- `BulkInsertMvoAttributeValuesViaEfAsync` — parameterised multi-row INSERT fallback (single-connection path)
+
+**Delegation** (`SyncRepository.cs`): `CreateMetaverseObjectsAsync` now routes to the owned `CreateMetaverseObjectsBulkAsync` instead of delegating to `MetaverseRepository`.
+
+### EF Change Tracker Bridge
+
+Integration testing revealed that bypassing EF for MVO creates caused a `DbUpdateConcurrencyException` (`xmin` mismatch) during the subsequent page flush. The root cause:
+
+1. Downstream sync code (`CreatePendingMvoChangeObjectsAsync`) adds `MetaverseObjectChange` entities to `mvo.Changes` navigation collections
+2. When `SaveChangesAsync` runs later (via `UpdateActivityAsync`), EF needs to track the parent MVO to discover and persist these child entities
+3. With COPY binary, MVOs were untracked — EF either missed the child entities or discovered the MVOs through navigation traversal with stale `xmin = 0`
+
+**Fix**: After COPY binary persistence, MVOs and their attribute values are attached to the EF change tracker as `Unchanged` with shadow FKs (`TypeId`, `MetaverseObjectId`) set explicitly. This is a **temporary bridge** — it will be removed when MVO change tracking and export evaluation are also converted to raw SQL.
+
+### Column Reference
+
+**MetaverseObjects** (10 columns):
+
+| Column | Type | Nullable | Notes |
+|--------|------|----------|-------|
+| Id | uuid | No | Pre-generated `Guid.NewGuid()` |
+| Created | timestamp with time zone | No | |
+| LastUpdated | timestamp with time zone | Yes | |
+| TypeId | integer | No | Shadow FK to MetaverseObjectType — read from `mvo.Type.Id` |
+| Status | integer | No | Enum `MetaverseObjectStatus` |
+| Origin | integer | No | Enum `MetaverseObjectOrigin` |
+| LastConnectorDisconnectedDate | timestamp with time zone | Yes | |
+| DeletionInitiatedByType | integer | No | Enum `ActivityInitiatorType` |
+| DeletionInitiatedById | uuid | Yes | |
+| DeletionInitiatedByName | text | Yes | |
+
+Note: `xmin` is a PostgreSQL system column (concurrency token) — assigned automatically by PostgreSQL on INSERT, must NOT be included in COPY statements.
+
+**MetaverseObjectAttributeValues** (13 columns):
+
+| Column | Type | Nullable | Notes |
+|--------|------|----------|-------|
+| Id | uuid | No | Pre-generated `Guid.NewGuid()` |
+| MetaverseObjectId | uuid | No | Shadow FK to MetaverseObject (parent) |
+| AttributeId | integer | No | FK to MetaverseAttribute |
+| StringValue | text | Yes | |
+| DateTimeValue | timestamp with time zone | Yes | |
+| IntValue | integer | Yes | |
+| LongValue | bigint | Yes | |
+| ByteValue | bytea | Yes | |
+| GuidValue | uuid | Yes | |
+| BoolValue | boolean | Yes | |
+| ReferenceValueId | uuid | Yes | FK to MetaverseObject (reference) |
+| UnresolvedReferenceValueId | uuid | Yes | FK to ConnectedSystemObject |
+| ContributedBySystemId | integer | Yes | FK to ConnectedSystem |
+
+### FK Fixup
+
+The caller at `SyncTaskProcessorBase` relies on `cso.MetaverseObject.Id` being populated after `CreateMetaverseObjectsAsync`. With pre-generated IDs, `mvo.Id` is set before persistence — no caller changes needed.
+
+### Success Criteria — Met
+
+- ✅ `CreateMetaverseObjectsAsync` uses COPY binary for large batches, parameterised INSERT for small batches
+- ✅ Pre-generated IDs ensure CSO FK fixup works without EF relationship fixup
+- ✅ All 2,353 unit tests pass without modification
+- ✅ Integration tests pass (Scenario 1 Small confirmed)
+- ✅ No regressions in sync integrity
+
+---
+
+## MVO Updates (deferred)
+
+`UpdateMetaverseObjectsAsync` remains on EF Core. Tracked under [#436](https://github.com/TetronIO/JIM/issues/436) — tackle only when profiling shows MVO updates are a significant bottleneck in delta sync.
+
+### Why updates are fundamentally harder than creates
+
+Creates are a single operation (INSERT rows). Updates involve **four distinct operations in one flush**:
+
+| Operation | What | Raw SQL approach |
+|-----------|------|-----------------|
+| UPDATE parent MVO rows | Scalar property changes (Status, LastUpdated, deletion fields) | `UPDATE ... FROM (VALUES ...)` batch — straightforward |
+| INSERT new attribute values | AVs added during inbound attribute flow | COPY binary — proven pattern from creates |
+| UPDATE existing attribute values | AVs modified during inbound attribute flow | `UPDATE ... FROM (VALUES ...)` — 13 nullable columns |
+| DELETE removed attribute values | AVs recalled during disconnect/out-of-scope | `DELETE ... WHERE "Id" IN (...)` — need to identify which |
+
+The crux is **knowing which attribute values are new vs modified vs deleted** without EF's change tracker. Currently:
+
+1. `ProcessInboundAttributeFlow` accumulates changes in `PendingAttributeValueAdditions` / `PendingAttributeValueRemovals` (both `[NotMapped]`)
+2. `ApplyPendingMetaverseObjectAttributeChanges` moves pending items into/out of `mvo.AttributeValues`, then clears the pending collections
+3. `UpdateMetaverseObjectsAsync` receives MVOs with the final `AttributeValues` list and uses EF's `IsKeySet` to classify: `Id == Guid.Empty` → Added, otherwise → Modified
+
+A raw SQL conversion would need the caller to **preserve the classification** (e.g., keep `PendingAttributeValueAdditions`/`Removals` intact for the repository to consume) or the repository to diff against the database. Both approaches are invasive.
+
+### Entity state complexity
+
+MVOs arrive at `UpdateMetaverseObjectsAsync` in mixed states depending on the call context:
+
+| Context | MVO state | AV state | `AutoDetectChanges` |
+|---------|-----------|----------|---------------------|
+| Per-page flush | Tracked (in-memory from join/project/flow) | Mixed: tracked originals + new additions | Enabled |
+| Cross-page reference resolution | Detached (post-`ClearChangeTracker` reload) | Mixed: reloaded from DB + new additions | **Disabled** |
+| Singular update (deletion marking) | Tracked | No AV changes | Enabled |
+| Singular update (out-of-scope disconnect) | Tracked | Removals applied | Enabled |
+
+The cross-page path is the dangerous one — `AutoDetectChangesEnabled = false` prevents `SaveChangesAsync` from walking navigation properties into shared `MetaverseAttribute`/`MetaverseObjectType` instances (which would cause identity conflicts). The current `UpdateDetachedSafe` + per-entity `Entry().State =` pattern was the result of 11 debugging attempts documented in `docs/notes/done/CROSS_PAGE_REFERENCE_IDENTITY_CONFLICT.md`.
+
+### Recommendation
+
+**Wait for profiling data.** Creates are the high-volume operation during initial sync (hundreds/thousands of MVOs per page). Updates are smaller per-page during delta sync, and the EF overhead is proportionally less significant. The complexity and regression risk of converting updates is not justified without evidence that it's a bottleneck.
+
+---
+
+## Risks and Mitigations
+
+| Risk | Mitigation | Outcome |
+|------|------------|---------|
+| Shadow FK columns have wrong names | Verified against migration snapshot | ✅ Names confirmed correct |
+| `xmin` in COPY causes error | Excluded from COPY column list | ✅ PostgreSQL assigns automatically |
+| Caller expects EF relationship fixup for MVO IDs | Pre-generated IDs | ✅ Caller code compatible |
+| Downstream code relies on EF tracking MVOs | Temporary tracker bridge (attach as Unchanged) | ✅ Integration tests pass |
+| Unit tests use in-memory provider | Delegation only affects Postgres SyncRepository; InMemory unchanged | ✅ All tests pass |
diff --git a/docs/plans/done/CSO_ATTRIBUTE_TABLE_REDESIGN.md b/docs/plans/done/CSO_ATTRIBUTE_TABLE_REDESIGN.md
index 3d018ed8c..2bba27b6e 100644
--- a/docs/plans/done/CSO_ATTRIBUTE_TABLE_REDESIGN.md
+++ b/docs/plans/done/CSO_ATTRIBUTE_TABLE_REDESIGN.md
@@ -44,7 +44,7 @@ A clean two-column table with:
| displayName | Dept-Engineering |
| distinguishedName | CN=Dept-Engineering,OU=Entitlements,... |
| groupType | -2147483640 |
-| mail | dept-engineering@sourcedomain.local |
+| mail | dept-engineering@resurgam.local |
| managedBy | [User] Oliver Smith (distinguishedName:..) |
| member | [User] Jack Pearson (+1 more) [v] |
| objectClass | group, top |
diff --git a/docs/plans/done/DEVCONTAINER_IDP.md b/docs/plans/done/DEVCONTAINER_IDP.md
new file mode 100644
index 000000000..0895d1656
--- /dev/null
+++ b/docs/plans/done/DEVCONTAINER_IDP.md
@@ -0,0 +1,227 @@
+# Devcontainer Identity Provider (Keycloak)
+
+- **Status:** Done
+- **Issue:** [#197](https://github.com/TetronIO/JIM/issues/197)
+- **Last Updated:** 2026-03-30
+- **Milestone:** v0.9-STABILISATION
+
+## Overview
+
+Integrate a pre-configured Keycloak instance into the devcontainer Docker stack so that developers can sign in to JIM immediately without configuring an external Identity Provider. The Keycloak instance ships with a realm export containing clients, scopes, and test users — zero manual setup required.
+
+## Problem
+
+Developers must currently:
+
+1. Register an application with an external IdP (Entra ID, Keycloak, Okta, etc.)
+2. Generate client credentials
+3. Configure 6+ environment variables in `.env`
+4. Understand OIDC concepts to get values right
+
+This creates a barrier for quick evaluation, onboarding new contributors, and local development. The devcontainer should be self-contained.
+
+## Goals
+
+- Developers can `jim-stack` and sign in to JIM without any IdP configuration
+- External IdP configuration remains supported by overriding `.env` values
+- PowerShell module (`Connect-JIM`) works against the bundled Keycloak
+- No changes to JIM's OIDC code (it is already IdP-agnostic)
+- No impact on production deployments
+
+## Non-Goals
+
+- Production Keycloak deployment (customer's responsibility)
+- Persistent Keycloak data (H2 ephemeral database is fine — realm re-imports on restart)
+- Keycloak customisation UI/themes
+- Keycloak as a connected system / connector target
+
+---
+
+## Technical Design
+
+### Keycloak Service
+
+Add Keycloak to `docker-compose.override.yml` (dev-only, not in the production `docker-compose.yml`).
+
+```yaml
+jim.keycloak:
+ image: quay.io/keycloak/keycloak:26.0
+ container_name: JIM.Keycloak
+ restart: unless-stopped
+ command: start-dev --import-realm --health-enabled=true
+ environment:
+ KEYCLOAK_ADMIN: admin
+ KEYCLOAK_ADMIN_PASSWORD: admin
+ ports:
+ - "8080:8080"
+ volumes:
+ - ./.devcontainer/keycloak/jim-realm.json:/opt/keycloak/data/import/jim-realm.json:ro
+ healthcheck:
+ test: ["CMD-SHELL", "exec 3<>/dev/tcp/127.0.0.1/8080; echo -e 'GET /health/ready HTTP/1.1\\r\\nhost: localhost\\r\\nConnection: close\\r\\n\\r\\n' >&3; timeout 1 cat <&3 | grep -q '200 OK'"]
+ interval: 10s
+ timeout: 5s
+ retries: 15
+ start_period: 30s
+ networks:
+ - jim-network
+```
+
+Key decisions:
+
+| Decision | Choice | Rationale |
+|----------|--------|-----------|
+| Compose file | `docker-compose.override.yml` | Dev-only; production compose is unaffected |
+| Mode | `start-dev` | No TLS or hostname config required |
+| Database | H2 (default) | Ephemeral is fine — realm re-imports on restart |
+| Realm import | `--import-realm` flag | Scans `/opt/keycloak/data/import/` on startup; skips if realm exists |
+| Health check | `/health/ready` via bash TCP | Keycloak image has no `curl`; bash `/dev/tcp` works natively |
+| Image pinning | Version tag (e.g. `26.0`) | Consistent with JIM's dependency pinning policy |
+
+### Realm Export
+
+File: `.devcontainer/keycloak/jim-realm.json`
+
+Pre-configured `jim` realm containing:
+
+**Clients:**
+
+| Client ID | Type | Purpose |
+|-----------|------|---------|
+| `jim-web` | Confidential + PKCE (S256) | JIM web application (auth code flow) |
+| `jim-powershell` | Public + PKCE (S256) | PowerShell module (loopback redirect) |
+
+**`jim-web` client configuration:**
+- `publicClient: false` (confidential)
+- `secret: jim-dev-secret` (known, dev-only)
+- `standardFlowEnabled: true`
+- `pkce.code.challenge.method: S256`
+- Redirect URIs: `http://localhost:5200/*`, `https://localhost:7000/*`
+- Web origins: `http://localhost:5200`, `https://localhost:7000`
+
+**`jim-powershell` client configuration:**
+- `publicClient: true`
+- `standardFlowEnabled: true`
+- `pkce.code.challenge.method: S256`
+- Redirect URIs: `http://localhost:8400/*` through `http://localhost:8409/*`
+- Web origins: `+`
+
+**Client Scopes:**
+- `jim-api` scope with audience mapper (`jim-web` audience, added to access token)
+
+**Test Users:**
+
+| Username | Password | Email | Role |
+|----------|----------|-------|------|
+| `admin` | `admin` | `admin@jim.local` | Initial admin (matches `JIM_SSO_INITIAL_ADMIN`) |
+| `user` | `user` | `user@jim.local` | Regular user |
+
+Both users: `enabled: true`, `emailVerified: true`, `temporary: false` (no forced password change).
+
+### Environment Configuration
+
+**`.env.example` defaults** point to the bundled Keycloak — works out of the box:
+
+```bash
+JIM_SSO_AUTHORITY=http://localhost:8181/realms/jim
+JIM_SSO_CLIENT_ID=jim-web
+JIM_SSO_SECRET=jim-dev-secret
+JIM_SSO_API_SCOPE=jim-api
+JIM_SSO_CLAIM_TYPE=sub
+JIM_SSO_MV_ATTRIBUTE=Subject Identifier
+JIM_SSO_INITIAL_ADMIN=00000000-0000-0000-0000-000000000001
+JIM_SSO_VALID_ISSUERS=http://localhost:8181/realms/jim,http://jim.keycloak:8080/realms/jim
+```
+
+The admin user has a fixed UUID (`00000000-0000-0000-0000-000000000001`) set in the realm export, so the `sub` claim is predictable.
+
+### Port Architecture ✅
+
+Docker-in-Docker proxy ports are not auto-forwarded by VS Code Dev Containers unless present at devcontainer build time. The solution uses a three-hop path:
+
+```
+Browser (Mac) VS Code port forward socat bridge Docker port mapping Keycloak container
+localhost:8181 --> devcontainer:8181 --> devcontainer:8180 --> jim.keycloak:8080
+```
+
+- **Port 8181**: socat userspace listener (VS Code detects and forwards to host)
+- **Port 8180**: Docker host port mapping (`8180:8080` in compose)
+- **Port 8080**: Keycloak's internal container port
+
+The `socat` bridge is started automatically by `jim-stack`, `jim-build`, `jim-restart`, and `jim-keycloak` aliases. `socat` is installed by `setup.sh`.
+
+### Issuer / Dual-Network Resolution ✅
+
+The `jim.web` container reaches Keycloak via Docker DNS (`jim.keycloak:8080`) for back-channel OIDC discovery, but browser redirects must go to `localhost:8181`. Solved with:
+
+1. **`JIM_SSO_AUTHORITY=http://jim.keycloak:8080/realms/jim`** overridden in `docker-compose.override.yml` — back-channel uses Docker DNS.
+2. **`OnRedirectToIdentityProvider` rewrite** in `Program.cs` — rewrites browser redirects from `jim.keycloak:8080` to the `localhost` issuer found in `JIM_SSO_VALID_ISSUERS`.
+3. **`JIM_SSO_VALID_ISSUERS`** accepts both `http://localhost:8181/realms/jim` and `http://jim.keycloak:8080/realms/jim` — tokens may be issued by either hostname.
+4. **`ValidIssuers`** set on both OIDC and JWT Bearer `TokenValidationParameters`.
+
+**For local debugging (Workflow 1):**
+
+When running JIM via F5, it reads `.env` directly where `JIM_SSO_AUTHORITY=http://localhost:8181/realms/jim` — no Docker DNS involved. Start Keycloak standalone with `jim-keycloak`.
+
+---
+
+## Implementation Plan
+
+### Phase 1: Realm Export and Keycloak Service ✅
+
+1. Created `.devcontainer/keycloak/jim-realm.json` — `jim` realm with `jim-web` (confidential + PKCE), `jim-powershell` (public + PKCE), `jim-api` scope with audience mapper, built-in OIDC client scopes, and two test users with fixed UUIDs
+2. Added `jim.keycloak` service to `docker-compose.override.yml` — Keycloak 26.0, `start-dev` mode, `--import-realm`, health check on port 9000
+3. Added port `8181` to `devcontainer.json` `forwardPorts`
+4. Added socat bridge (`8181->8180`) to work around VS Code Dev Containers not forwarding Docker-in-Docker proxy ports
+5. Resolved issuer/hostname dual-network issue via `OnRedirectToIdentityProvider` rewrite + dual `ValidIssuers`
+6. Added `JIM_SSO_AUTHORITY` override in `docker-compose.override.yml` for `jim.web` to use Docker DNS
+7. Added `jim.web` `depends_on` Keycloak health check
+8. Set `RequireHttpsMetadata=false` conditionally for HTTP authorities
+
+### Phase 2: Environment Integration ✅
+
+1. Updated `.env.example` with working Keycloak defaults (authority, client, secret, scope, initial admin UUID, dual valid issuers)
+2. Updated `setup.sh` — replaced SSO warning with confirmation message, updated startup instructions, added socat installation
+3. Test: fresh devcontainer creation → `jim-stack` → sign in with `admin` / `admin`
+
+### Phase 3: Local Debugging Support ✅
+
+1. Added `jim-keycloak`, `jim-keycloak-stop`, `jim-keycloak-logs` aliases to `jim-aliases.sh`
+2. `.env` defaults point to `localhost:8181` — works for both Docker stack and F5 workflows
+3. socat bridge auto-starts from `jim-stack`, `jim-build`, `jim-restart`, `jim-keycloak`
+4. Test: `jim-db && jim-keycloak` → F5 → sign in with `admin` / `admin`
+
+### Phase 4: Documentation ✅
+
+1. Updated `docs/SSO_SETUP_GUIDE.md` with "Development (Bundled Keycloak)" section at the top
+2. Updated `docs/DEVELOPER_GUIDE.md` — removed SSO prerequisite, added "works out of the box" notes
+3. Updated `README.md` — developer prerequisites no longer require external SSO setup
+4. Added changelog entry under [Unreleased]
+
+---
+
+## Success Criteria
+
+1. Fresh devcontainer → `jim-stack` → sign in with `admin`/`admin` — zero IdP configuration
+2. `Connect-JIM` PowerShell module works against bundled Keycloak
+3. Overriding `.env` SSO variables switches to an external IdP (bundled Keycloak ignored)
+4. Production `docker-compose.yml` is unaffected (no Keycloak service)
+5. Keycloak admin console accessible at `http://localhost:8181` for debugging
+6. Both development workflows work (Docker stack and F5 local debugging)
+
+---
+
+## Dependencies
+
+- **Keycloak Docker image**: `quay.io/keycloak/keycloak:26.0` (or latest stable at implementation time)
+- No new NuGet packages required
+- `socat` package (installed by `setup.sh`)
+- Minor OIDC changes in `Program.cs` (HTTP authority support, browser redirect rewrite, dual issuer validation)
+
+## Risks and Mitigations
+
+| Risk | Impact | Mitigation |
+|------|--------|------------|
+| Issuer mismatch (Docker DNS vs localhost) | Token validation fails | `OnRedirectToIdentityProvider` rewrite + dual `ValidIssuers` on both OIDC and JWT Bearer |
+| Keycloak startup time (30-60s) | `jim.web` starts before Keycloak is ready | Health check + `depends_on` with `condition: service_healthy` |
+| Image size (~400MB) | Slower first pull | One-time cost; cached after first pull |
+| Keycloak version drift | Breaking changes in realm format | Pin to specific version tag; update as part of Dependabot cycle |
diff --git a/docs/plans/done/INTEGRATION_TEST_FULL_REGRESSION.md b/docs/plans/done/INTEGRATION_TEST_FULL_REGRESSION.md
index b8ab66140..abb020685 100644
--- a/docs/plans/done/INTEGRATION_TEST_FULL_REGRESSION.md
+++ b/docs/plans/done/INTEGRATION_TEST_FULL_REGRESSION.md
@@ -145,7 +145,7 @@ function Reset-JIMForNextScenario {
docker volume rm jim-db-volume 2>&1 | Out-Null
# 3. Clean Samba AD test data
- docker exec samba-ad-primary samba-tool ou delete "OU=Corp,DC=subatomic,DC=local" --force-subtree-delete 2>&1 | Out-Null
+ docker exec samba-ad-primary samba-tool ou delete "OU=Corp,DC=panoply,DC=local" --force-subtree-delete 2>&1 | Out-Null
# ... clean other OUs as needed
# 4. Generate new API key, update .env
diff --git a/docs/plans/OPENLDAP_INTEGRATION_TESTING.md b/docs/plans/done/OPENLDAP_INTEGRATION_TESTING.md
similarity index 57%
rename from docs/plans/OPENLDAP_INTEGRATION_TESTING.md
rename to docs/plans/done/OPENLDAP_INTEGRATION_TESTING.md
index 44887f26c..81f284caf 100644
--- a/docs/plans/OPENLDAP_INTEGRATION_TESTING.md
+++ b/docs/plans/done/OPENLDAP_INTEGRATION_TESTING.md
@@ -1,7 +1,8 @@
# OpenLDAP Integration Testing
-- **Status:** Planned
+- **Status:** Done (Phases 1-6 complete — all scenarios pass on both SambaAD and OpenLDAP at Medium; S3 deferred)
- **Created:** 2026-03-09
+- **Issue:** [#72](https://github.com/TetronIO/JIM/issues/72)
## Overview
@@ -39,7 +40,7 @@ Rationale:
| **User naming** | `sAMAccountName`, `userPrincipalName`, `CN=DisplayName` | `uid`, `cn`, no UPN equivalent |
| **Group membership** | `member` (DN-valued) / `memberOf` (back-link) | `member` on `groupOfNames`, or `uniqueMember` on `groupOfUniqueNames` |
| **Account disable** | `userAccountControl` bitmask (0x2) | No native concept — typically `pwdAccountLockedTime` (ppolicy overlay) or custom attribute |
-| **DN format** | `CN=Name,OU=Users,DC=domain,DC=local` | `uid=username,ou=People,dc=openldap,dc=local` |
+| **DN format** | `CN=Name,OU=Users,DC=domain,DC=local` | `uid=username,ou=People,dc=yellowstone,dc=local` |
| **Delta import** | USN-based (`uSNChanged`, tombstones) | Changelog (`cn=changelog`, `changeNumber`) — already implemented in connector |
| **Paging** | Supported (disabled for Samba due to duplicate results) | Supported via Simple Paged Results control |
| **Protected attributes** | `nTSecurityDescriptor`, `userAccountControl`, etc. (SAM layer) | None |
@@ -50,9 +51,9 @@ Rationale:
## Docker Image Strategy
-### Base Image: `bitnami/openldap`
+### Base Image: `bitnamilegacy/openldap` ✅
-The existing `docker-compose.integration-tests.yml` references `osixia/openldap:latest` but this image is **unmaintained** (last meaningful update 2022, Debian Stretch EOL, open CVEs). Replace with `bitnami/openldap`:
+Bitnami migrated their images off Docker Hub in August 2025. The `bitnamilegacy/openldap:latest` image (OpenLDAP 2.6.10) is the successor, replacing the unmaintained `osixia/openldap:latest`:
| Criteria | `osixia/openldap` | `bitnami/openldap` |
|----------|--------------------|--------------------|
@@ -83,84 +84,40 @@ The Dockerfile copies bootstrap LDIFs to the auto-load directory. On first start
**Build script** (`Build-OpenLdapImage.ps1`) is simpler than the Samba equivalent — just wraps `docker build` with tagging and optional push to `ghcr.io/tetronio/jim-openldap:primary`.
-### Docker Compose Configuration
-
-Replace the existing `osixia/openldap` service in `docker-compose.integration-tests.yml`:
-
-```yaml
-# OpenLDAP primary instance
-openldap-primary:
- image: ${OPENLDAP_IMAGE_PRIMARY:-ghcr.io/tetronio/jim-openldap:primary}
- build:
- context: ./test/integration/docker/openldap
- dockerfile: Dockerfile
- container_name: openldap-primary
- environment:
- - LDAP_ROOT=dc=openldap,dc=local
- - LDAP_ADMIN_DN=cn=admin,dc=openldap,dc=local
- - LDAP_ADMIN_PASSWORD=Test@123!
- - LDAP_ENABLE_TLS=no
- - LDAP_CUSTOM_LDIF_DIR=/ldifs
- volumes:
- - openldap-primary-data:/bitnami/openldap
- - test-csv-data:/connector-files
- networks:
- - jim-network
- profiles:
- - openldap
- healthcheck:
- test: ["CMD", "ldapsearch", "-x", "-H", "ldap://localhost:1389",
- "-b", "dc=openldap,dc=local",
- "-D", "cn=admin,dc=openldap,dc=local", "-w", "Test@123!"]
- interval: 10s
- timeout: 5s
- retries: 5
- start_period: 10s
- deploy:
- resources:
- limits:
- cpus: ${OPENLDAP_PRIMARY_CPUS:-2.0}
- memory: ${OPENLDAP_PRIMARY_MEMORY:-2G}
- reservations:
- cpus: ${OPENLDAP_PRIMARY_CPUS_RESERVED:-0.5}
- memory: ${OPENLDAP_PRIMARY_MEMORY_RESERVED:-512M}
-```
+### Docker Compose Configuration ✅
+
+The `osixia/openldap` service has been replaced in `docker-compose.integration-tests.yml`. See the actual compose file for the current configuration. Key points:
-Key points:
- Profile `openldap` — only started when OpenLDAP tests are requested
- Port 1389 (bitnami non-root default) — not exposed to host, internal to `jim-network`
- No privileged mode required
-- Remove the existing `osixia/openldap` service definition and its volumes (`openldap-data`, `openldap-config`)
+- Two suffixes: `dc=yellowstone,dc=local` (primary via `LDAP_ROOT`) and `dc=glitterband,dc=local` (added by init script)
+- Accesslog overlay enabled for future delta import testing
+- Config admin enabled for `cn=config` modifications by the init script
-### Bootstrap OU Structure
+### Bootstrap OU Structure ✅
-The bootstrap LDIF (`01-base-ous.ldif`) creates the equivalent of Samba AD's baseline OUs:
+The bootstrap LDIF (`01-base-ous-yellowstone.ldif`) creates the equivalent of Samba AD's baseline OUs for the primary suffix. The second suffix (`dc=glitterband,dc=local`) is created by the init script `01-add-second-suffix.sh`:
```ldif
# OU=People — equivalent to OU=Users,OU=Corp in Samba AD
-dn: ou=People,dc=openldap,dc=local
+dn: ou=People,dc=yellowstone,dc=local
objectClass: organizationalUnit
ou: People
# OU=Groups — equivalent to OU=Groups,OU=Corp in Samba AD
-dn: ou=Groups,dc=openldap,dc=local
+dn: ou=Groups,dc=yellowstone,dc=local
objectClass: organizationalUnit
ou: Groups
# Department OUs under People (created dynamically by Populate-OpenLDAP.ps1)
```
-### OpenLDAP Changelog Overlay (for Delta Import)
+### OpenLDAP Changelog Overlay (for Delta Import) ✅
The connector's changelog-based delta import queries `cn=changelog` for entries with `changeNumber > lastProcessed`. This requires the **accesslog overlay** (`slapo-accesslog`) to be enabled.
-The Dockerfile or bootstrap config must enable this overlay. For `bitnami/openldap`, this can be done via a custom schema/config LDIF or by setting:
-
-```
-LDAP_EXTRA_SCHEMAS=accesslog
-```
-
-If the accesslog overlay is not straightforward to configure via environment variables, it can be configured via a bootstrap LDIF that modifies `cn=config`. This needs investigation during implementation — if too complex, delta import testing can be deferred (full import is the priority).
+The `bitnamilegacy/openldap` image supports this natively via the `LDAP_ENABLE_ACCESSLOG=yes` environment variable, which is set in the Docker Compose configuration. The accesslog database is visible as `cn=accesslog` in the RootDSE naming contexts.
## Test Data Population: `Populate-OpenLDAP.ps1`
@@ -192,7 +149,7 @@ New script parallel to `Populate-SambaAD.ps1`. Generates LDIF and loads via `lda
The script generates standard LDIF entries. Example user:
```ldif
-dn: uid=john.smith,ou=People,dc=openldap,dc=local
+dn: uid=john.smith,ou=People,dc=yellowstone,dc=local
objectClass: inetOrgPerson
objectClass: organizationalPerson
objectClass: person
@@ -202,7 +159,7 @@ cn: John Smith
sn: Smith
givenName: John
displayName: John Smith
-mail: john.smith@openldap.local
+mail: john.smith@yellowstone.local
title: Software Engineer
departmentNumber: Engineering
userPassword: {SSHA}base64encodedpassword
@@ -211,16 +168,16 @@ userPassword: {SSHA}base64encodedpassword
Example group:
```ldif
-dn: cn=Engineering,ou=Groups,dc=openldap,dc=local
+dn: cn=Engineering,ou=Groups,dc=yellowstone,dc=local
objectClass: groupOfNames
cn: Engineering
description: Engineering department group
-member: uid=john.smith,ou=People,dc=openldap,dc=local
+member: uid=john.smith,ou=People,dc=yellowstone,dc=local
```
### Bulk Loading
-- Use `docker exec openldap-primary ldapadd -x -D "cn=admin,dc=openldap,dc=local" -w "Test@123!" -c -f /path/to/users.ldif`
+- Use `docker exec openldap-primary ldapadd -x -D "cn=admin,dc=yellowstone,dc=local" -w "Test@123!" -c -f /path/to/users.ldif`
- The `-c` flag continues on errors (useful for idempotent re-runs)
- Batch into chunks of ~10,000 entries per LDIF file for large templates
- LDIF files can be written to the shared `test-csv-data` volume (mounted at `/connector-files`)
@@ -286,12 +243,12 @@ function Get-DirectoryConfig {
Port = 636
UseSSL = $true
CertValidation = "Skip Validation (Not Recommended)"
- BindDN = "CN=Administrator,CN=Users,DC=subatomic,DC=local"
+ BindDN = "CN=Administrator,CN=Users,DC=panoply,DC=local"
BindPassword = "Test@123!"
AuthType = "Simple"
- BaseDN = "DC=subatomic,DC=local"
- UserContainer = "OU=Users,OU=Corp,DC=subatomic,DC=local"
- GroupContainer = "OU=Groups,OU=Corp,DC=subatomic,DC=local"
+ BaseDN = "DC=panoply,DC=local"
+ UserContainer = "OU=Users,OU=Corp,DC=panoply,DC=local"
+ GroupContainer = "OU=Groups,OU=Corp,DC=panoply,DC=local"
UserObjectClass = "user"
GroupObjectClass = "group"
UserRdnAttr = "CN"
@@ -299,7 +256,7 @@ function Get-DirectoryConfig {
DepartmentAttr = "department"
DeleteBehaviour = "Disable"
DisableAttribute = "userAccountControl"
- DnTemplate = 'CN={displayName},OU=Users,OU=Corp,DC=subatomic,DC=local'
+ DnTemplate = 'CN={displayName},OU=Users,OU=Corp,DC=panoply,DC=local'
}
}
"OpenLDAP" {
@@ -309,12 +266,12 @@ function Get-DirectoryConfig {
Port = 1389
UseSSL = $false
CertValidation = $null
- BindDN = "cn=admin,dc=openldap,dc=local"
+ BindDN = "cn=admin,dc=yellowstone,dc=local"
BindPassword = "Test@123!"
AuthType = "Simple"
- BaseDN = "dc=openldap,dc=local"
- UserContainer = "ou=People,dc=openldap,dc=local"
- GroupContainer = "ou=Groups,dc=openldap,dc=local"
+ BaseDN = "dc=yellowstone,dc=local"
+ UserContainer = "ou=People,dc=yellowstone,dc=local"
+ GroupContainer = "ou=Groups,dc=yellowstone,dc=local"
UserObjectClass = "inetOrgPerson"
GroupObjectClass = "groupOfNames"
UserRdnAttr = "uid"
@@ -322,7 +279,7 @@ function Get-DirectoryConfig {
DepartmentAttr = "departmentNumber"
DeleteBehaviour = "Delete"
DisableAttribute = $null
- DnTemplate = 'uid={uid},ou=People,dc=openldap,dc=local'
+ DnTemplate = 'uid={uid},ou=People,dc=yellowstone,dc=local'
}
}
}
@@ -453,72 +410,128 @@ This will throw `InvalidOperationException` for OpenLDAP (which has `entryUUID`,
## Implementation Phases
-### Phase 1: Docker Infrastructure
+### Phase 1: Docker Infrastructure ✅
**Deliverables:**
-- `test/integration/docker/openldap/Dockerfile`
-- `test/integration/docker/openldap/bootstrap/01-base-ous.ldif`
-- `test/integration/docker/openldap/Build-OpenLdapImage.ps1`
-- Updated `docker-compose.integration-tests.yml` (replace `osixia/openldap` with `bitnami/openldap`, `openldap` profile)
-- Manual verification: container starts, health check passes, admin can bind and search
+- `test/integration/docker/openldap/Dockerfile` — based on `bitnamilegacy/openldap` (OpenLDAP 2.6.10)
+- `test/integration/docker/openldap/bootstrap/01-base-ous-yellowstone.ldif` — root entry and OUs for primary suffix
+- `test/integration/docker/openldap/scripts/01-add-second-suffix.sh` — creates second MDB database (`dc=glitterband,dc=local`) via `cn=config` at startup
+- `test/integration/docker/openldap/Build-OpenLdapImage.ps1` — build script with content-hash labelling
+- Updated `docker-compose.integration-tests.yml` (replaced `osixia/openldap` with `bitnamilegacy/openldap`, `openldap` profile)
+- Two naming contexts verified: `dc=yellowstone,dc=local` and `dc=glitterband,dc=local`
+- Accesslog overlay enabled (`LDAP_ENABLE_ACCESSLOG=yes`) for future delta import testing
+- Health check passes, both suffixes queryable with admin bind
-### Phase 2: Test Data Population
+### Phase 2: Test Data Population ✅
**Deliverables:**
-- `test/integration/Populate-OpenLDAP.ps1` — generates LDIF for `inetOrgPerson` users and `groupOfNames` groups, loads via `ldapadd`
-- Manual verification: population works at Nano/Micro/Small scales, users and groups queryable
+- `test/integration/Populate-OpenLDAP.ps1` — generates `inetOrgPerson` users and `groupOfNames` groups across both suffixes, loads via `ldapadd` piped through stdin
+- Users split between suffixes: odd indices to Yellowstone, even to Glitterband — distinct users per partition for Scenario 9 assertions
+- `groupOfNames` MUST constraint handled: initial member assigned during group creation
+- Additional memberships added via `ldapmodify`
+- Verified at Nano (3 users) and Micro (10 users) scales
-### Phase 3: Test Framework Parameterisation
+### Phase 3: Test Framework Parameterisation ✅
**Deliverables:**
-- `Get-DirectoryConfig` function in `Test-Helpers.ps1`
-- `-DirectoryType` parameter on `Run-IntegrationTests.ps1`
-- Refactored `LDAP-Helpers.ps1` to accept directory config
-- Parameterised `Setup-Scenario1.ps1` (start with Scenario 1 as the pilot)
-- Parameterised `Invoke-Scenario1.ps1` validation assertions
+- `Get-DirectoryConfig` function in `Test-Helpers.ps1` — returns directory-specific config (container, host, port, bind DN, object classes, etc.) for SambaAD or OpenLDAP
+- `-DirectoryType` parameter on `Run-IntegrationTests.ps1` — controls Docker profile, health checks, population, and OU preparation
+- Refactored `LDAP-Helpers.ps1` — all functions accept `$DirectoryConfig` hashtable alongside individual params for backward compatibility
+- Parameterised `Setup-Scenario1.ps1` — LDAP connected system name, host, port, bind DN, SSL, and auth type all driven by `$DirectoryConfig`
+- Parameterised `Invoke-Scenario1.ps1` — container name for docker exec calls driven by `$DirectoryConfig`
+- SambaAD remains the default — all existing behaviour preserved when `-DirectoryType` is not specified
-### Phase 4: Connector Fixes
+### Phase 4: Connector Fixes ✅
**Deliverables:**
-- RFC 4512 schema discovery path in `LdapConnectorSchema.cs`
-- `entryUUID` external ID recommendation for non-AD directories
-- Partition discovery fix for non-AD directories
-- Export handling for directory-specific differences
-- Unit tests for all new/modified connector code
-
-### Phase 5: End-to-End Validation
+- `LdapDirectoryType` enum with `ActiveDirectory`, `SambaAD`, `OpenLDAP`, `Generic` — all directory-specific behaviour centralised in computed properties on `LdapConnectorRootDse`
+- RFC 4512 schema discovery via `cn=Subschema` (`Rfc4512SchemaParser.cs`) with 37 unit tests — tokeniser, objectClass/attributeType parsing, SYNTAX OID mapping, writability from USAGE field
+- Partition discovery via `namingContexts` rootDSE attribute for non-AD directories
+- `entryUUID` and `distinguishedName` synthesised as attributes on all RFC schema object types (operational attributes not in any class's MUST/MAY)
+- `distinguishedName` synthesised during import from `entry.DistinguishedName` (OpenLDAP doesn't return it as an attribute)
+- Accesslog-based delta import (`GetDeltaResultsUsingAccesslog`) — queries `cn=accesslog` using `reqStart` timestamp watermarks
+- OpenLDAP detection via `structuralObjectClass: OpenLDAProotDSE` (fallback when `vendorName` not set)
+- DN-aware RDN attribute detection in export (`IsRdnAttribute` parses RDN from DN, not hardcoded `cn`)
+- Changelog query gated behind delta import only (not full import)
+- "Include Auxiliary Classes" connected system setting (both AD and RFC paths)
+- Related issues created: #433 (AD schema batch optimisation), #434 (filter internal object classes from UI)
+
+### Phase 5: End-to-End Validation ✅
**Deliverables:**
-- Scenario 1 (HR → OpenLDAP) passing at Nano scale
-- Fix issues found during E2E testing
-- Scale up through Micro → Small → Medium
-- Changelog-based delta import testing (if accesslog overlay configured)
-- Document any remaining limitations
+- Scenario 1 (HR → OpenLDAP) passing at Nano scale — all 8 test steps
+- Accesslog-based delta import confirmed working for export confirmation
+- Integration test parameterisation: all `samba-tool`/`ldbsearch` verifications replaced with `Get-LDAPUser`/`Test-LDAPUserExists`; all hardcoded container names, partitions, attributes, and mappings driven by `$DirectoryConfig`
+- AD-specific tests (Disable/Enable via `userAccountControl`) gracefully skipped for OpenLDAP
+- Mover Rename verifies `cn` update (not DN change) for OpenLDAP (uid-based RDN)
+- Mover Move verifies `departmentNumber` update (not OU move) for OpenLDAP (flat OU structure)
-### Phase 6: Remaining Scenarios (Future)
+**Scenario 1 test results (Nano scale):**
-**Deliverables:**
-- Parameterise Scenarios 2, 4, 5, 8 for OpenLDAP
-- OpenLDAP-to-OpenLDAP cross-directory sync (Scenario 2 equivalent)
-- XLarge-scale performance benchmarking against OpenLDAP
+| Step | Status | Notes |
+|------|--------|-------|
+| Joiner | ✅ Pass | Full lifecycle incl. accesslog delta import confirmation |
+| Mover (Attribute Change) | ✅ Pass | Title update exported and confirmed |
+| Mover Rename | ✅ Pass | cn/displayName updated (DN unchanged) |
+| Mover Move | ✅ Pass | departmentNumber updated (no OU move) |
+| Disable | ⏭ Skipped | No userAccountControl on OpenLDAP |
+| Enable | ⏭ Skipped | No userAccountControl on OpenLDAP |
+| Leaver | ✅ Pass | Grace period deprovisioning |
+| Reconnection | ✅ Pass | Delete → restore within grace period |
+
+### Phase 6: Remaining Scenarios (In Progress)
+
+**Goal:** Parameterise all remaining integration test scenarios for OpenLDAP.
+
+**Recommended order** (value vs effort):
+
+| Priority | Scenario | AD-specific refs | Effort | Status | Notes |
+|----------|----------|-----------------|--------|--------|-------|
+| 1 | **S9: Partition-Scoped Imports** | 5 | Low | ✅ Done | True multi-partition filtering with Yellowstone + Glitterband suffixes |
+| 2 | **S7: Clear Connected System Objects** | 0 | Low | ✅ Done | DirectoryConfig threading only — scenario is entirely CSV-based |
+| 3 | **S6: Scheduler Service** | 2 | Low | ✅ Done | DirectoryConfig, system name parameterised, docker cp replaced with bind mount |
+| 4 | **S2: Cross-Domain Sync** | 11 | Medium | ✅ Done | Two LDAP connected systems (Yellowstone→Glitterband), all 4 tests passing. Unblocked by #435. |
+| 5 | **S5: Matching Rules** | 17 | Medium | ✅ Done | DirectoryConfig threading, docker cp removed, user cleanup parameterised |
+| 6 | **S3: GAL Sync** | 0 | N/A | ⏭ Deferred | Not yet implemented — placeholder script only. Out of scope for this phase. |
+| 7 | **S4: Deletion Rules** | 26 | High | ✅ Done | All 7 tests passing — LDAP-Helpers replace samba-tool, .ContainsKey() for missing attrs |
+| 8 | **S8: Cross-Domain Entitlement Sync** | 50 | High | ✅ Done | All 6 tests passing at MediumLarge. Accesslog mapsize/sizelimit configured. Delta import fallback prevented (null watermark fix). Connector warnings moved to Activity.WarningMessage. |
+
+**Implementation advice for each scenario:**
+
+**S9 (Partition-Scoped Imports):** ✅ Complete. Both `Setup-Scenario9.ps1` and `Invoke-Scenario9-PartitionScopedImports.ps1` parameterised with `$DirectoryConfig`. For OpenLDAP: selects both partitions (Yellowstone + Glitterband), creates four run profiles (scoped primary, scoped second, unscoped, full sync), and asserts true partition isolation — scoped imports to each partition return only that partition's users, and combined counts match total. `Run-IntegrationTests.ps1` updated with Step 4c to call `Populate-OpenLDAP.ps1` before scenarios when using OpenLDAP.
+
+**S2 (Cross-Domain Sync):** Requires two LDAP connected systems pointing at the same OpenLDAP instance but different suffixes. The `Get-DirectoryConfig` function already has `SecondSuffix` and `SecondBindDN` for this purpose. Setup script needs a second connected system creation block.
+
+**S5 (Matching Rules):** Primarily attribute name substitution (`sAMAccountName`→`uid`, `employeeID`→`employeeNumber`, etc.) and object type substitution (`user`→`inetOrgPerson`). Follow the same parameterisation pattern used in S1's `Setup-Scenario1.ps1`.
+
+**S8 (Cross-Domain Entitlement Sync):** ✅ Complete. All 6 tests passing at MediumLarge scale (InitialSync, ForwardSync, DetectDrift, ReassertState, NewGroup, DeleteGroup). Key fixes applied: (1) OpenLDAP accesslog database configured with 1GB mapsize and unlimited sizelimit to prevent `MDB_MAP_FULL` at scale. (2) Delta import null watermark prevention — when accesslog is empty (e.g., after snapshot restore), full import generates a fallback timestamp so the next delta import doesn't fall back unnecessarily. (3) Connector-level warnings (e.g., `DeltaImportFallbackToFullImport`) moved from phantom RPEIs to `Activity.WarningMessage` — eliminates misleading RPEI rows with no CSO association. (4) `SplitOnCapitalLetters` fixed for camelCase LDAP object types (`groupOfNames` → `Group Of Names`). (5) External Object Type displayed as-is on Activity detail page (no word splitting).
+
+**S4 (Deletion Rules):** OpenLDAP has no account disable mechanism. The deletion rule tests that use `Disable` behaviour and verify `userAccountControl` will need to be skipped or adapted. The `Delete` behaviour (actual LDAP delete) should work unchanged.
+
+**Scale testing:** After all scenarios pass at Nano, scale up through Micro→Small→Medium. The connector code is scale-independent so this should be straightforward — any failures will be performance/timeout related, not logic bugs.
## Risks and Mitigations
-| Risk | Impact | Likelihood | Mitigation |
-|------|--------|------------|------------|
-| Schema discovery rewrite is complex | High | High | Phase 4 is the largest work item; consider starting with a minimal RFC schema parser that handles `inetOrgPerson` and `groupOfNames` only, expanding later |
-| OpenLDAP changelog overlay hard to configure | Medium | Medium | If accesslog overlay is too complex to enable in Docker, defer delta import testing — full import covers the critical path |
-| `bitnami/openldap` doesn't support `slapadd` for bulk loading | Medium | Medium | Fall back to `ldapadd` (slower but functional); consider pre-built populated images for XLarge |
-| `groupOfNames` empty group constraint breaks export | Medium | High | Needs connector code change; temporary mitigation is to always ensure groups have at least one member in test data |
-| Performance regression at XLarge if OpenLDAP population is slow | Low | Medium | Build pre-populated snapshot images (like Samba approach) for Large/XLarge templates |
+| Risk | Impact | Likelihood | Status | Mitigation |
+|------|--------|------------|--------|------------|
+| Schema discovery rewrite is complex | High | High | ✅ Resolved | RFC 4512 parser implemented with 37 unit tests. Handles all standard object classes and attribute types. |
+| OpenLDAP changelog overlay hard to configure | Medium | Medium | ✅ Resolved | Implemented accesslog-based delta import using `cn=accesslog` with `reqStart` timestamps. Works with Bitnami's `LDAP_ENABLE_ACCESSLOG=yes`. |
+| `bitnami/openldap` doesn't support `slapadd` for bulk loading | Medium | Medium | ✅ Resolved | Using `ldapadd` via stdin piping. Works at Nano/Micro/Small scales. Pre-built images needed for XLarge. |
+| `groupOfNames` empty group constraint breaks export | Medium | High | ✅ Resolved | Connector handles placeholder member transparently (configurable DN, default `cn=placeholder`). 21 unit tests. Refint error handling for directories with referential integrity overlay. |
+| Paged results cookie invalid on multi-type imports | Medium | High | ✅ Resolved | OpenLDAP's RFC 2696 cursor is connection-scoped — unrelated searches between paged calls invalidate it. Fix: skip completed container+objectType combos on subsequent pages. |
+| Performance regression at XLarge if OpenLDAP population is slow | Low | Medium | ✅ Resolved | Pre-populated snapshot images implemented (`Build-OpenLDAPSnapshots.ps1`) with content-hash staleness detection, matching Samba AD pattern |
+| Samba AD regression from connector changes | Medium | Low | ✅ Verified | Full regression (8/8 scenarios, Small template) passed on Samba AD. All connector changes gated behind `LdapDirectoryType` checks. |
## Success Criteria
-- [ ] OpenLDAP container starts and is healthy in integration test environment
-- [ ] `Populate-OpenLDAP.ps1` successfully creates users and groups at Small template scale
-- [ ] `Run-IntegrationTests.ps1 -DirectoryType OpenLDAP` parameter works and selects correct infrastructure
-- [ ] JIM LDAP connector can connect to OpenLDAP, discover schema, and refresh partitions
-- [ ] Scenario 1 Joiner step passes against OpenLDAP (CSV → JIM → OpenLDAP provisioning)
-- [ ] Scenario 1 Mover and Leaver steps pass against OpenLDAP
-- [ ] Delta import works against OpenLDAP (changelog-based or full import fallback documented)
-- [ ] All existing Samba AD tests continue to pass unchanged (no regressions)
+- [x] OpenLDAP container starts and is healthy in integration test environment
+- [x] `Populate-OpenLDAP.ps1` successfully creates users and groups at Small template scale
+- [x] `Run-IntegrationTests.ps1 -DirectoryType OpenLDAP` parameter works and selects correct infrastructure
+- [x] JIM LDAP connector can connect to OpenLDAP, discover schema, and refresh partitions
+- [x] Scenario 1 Joiner step passes against OpenLDAP (CSV → JIM → OpenLDAP provisioning)
+- [x] Scenario 1 Mover and Leaver steps pass against OpenLDAP
+- [x] Delta import works against OpenLDAP (accesslog-based with reqStart timestamps)
+- [x] All existing Samba AD tests continue to pass unchanged (8/8 scenarios, Small template — 2026-04-01)
+- [x] All scenarios (S1-S9, excluding S3 deferred) parameterised for OpenLDAP
+- [x] All OpenLDAP scenarios pass (8/8 scenarios, Small template — 2026-04-01)
+- [x] Scale testing through Micro → Small → Medium (full regression at Medium on both SambaAD and OpenLDAP — 2026-04-01)
diff --git a/docs/plans/done/SCENARIO_8_CROSS_DOMAIN_ENTITLEMENT_SYNC.md b/docs/plans/done/SCENARIO_8_CROSS_DOMAIN_ENTITLEMENT_SYNC.md
index cd405648d..a48215777 100644
--- a/docs/plans/done/SCENARIO_8_CROSS_DOMAIN_ENTITLEMENT_SYNC.md
+++ b/docs/plans/done/SCENARIO_8_CROSS_DOMAIN_ENTITLEMENT_SYNC.md
@@ -22,8 +22,8 @@
- Support for Nano through XXLarge templates
- **Phase 2: JIM Configuration** - Implemented
- - Source LDAP Connected System (Quantum Dynamics APAC)
- - Target LDAP Connected System (Quantum Dynamics EMEA)
+ - Source LDAP Connected System (Panoply APAC)
+ - Target LDAP Connected System (Panoply EMEA)
- User and Group object type selection with required attributes
- Import/Export sync rules for both users and groups
- Attribute flow mappings including DN expressions
@@ -129,12 +129,12 @@ Scenario 8 validates synchronising entitlement groups (security groups, distribu
│ Scenario 8: Group Sync Flow │
├────────────────────────────────────────────────────────────────────────────────┤
│ │
-│ Quantum Dynamics APAC (Source) Quantum Dynamics EMEA (Target) │
+│ Panoply APAC (Source) Panoply EMEA (Target) │
│ ┌──────────────────────────────┐ ┌────────────────────────────────┐ │
│ │ OU=Entitlements,OU=Corp │ │ OU=Entitlements,OU=CorpManaged │ │
│ │ │ │ │ │
│ │ ┌─────────────────────┐ │ │ ┌─────────────────────┐ │ │
-│ │ │ Company-Subatomic │ │ │ │ Company-Subatomic │ │ │
+│ │ │ Company-Panoply │ │ │ │ Company-Panoply │ │ │
│ │ │ Members: │ │ │ │ Members: │ │ │
│ │ │ - CN=John,OU=... │─────┼──────────┼─>│ - CN=John,OU=... │ │ │
│ │ │ - CN=Jane,OU=... │ │ │ │ - CN=Jane,OU=... │ │ │
@@ -152,8 +152,8 @@ Scenario 8 validates synchronising entitlement groups (security groups, distribu
│ ┌────────────────────────────────────────────────────────────────────┐ │
│ │ JIM Metaverse │ │
│ │ ┌─────────────────────────────────────────────────────────────┐ │ │
-│ │ │ Group MVO: Company-Subatomic │ │ │
-│ │ │ Account Name: Company-Subatomic │ │ │
+│ │ │ Group MVO: Company-Panoply │ │ │
+│ │ │ Account Name: Company-Panoply │ │ │
│ │ │ Group Type: Universal Security │ │ │
│ │ │ Members: [ref:User-John-MVO, ref:User-Jane-MVO] │ │ │
│ │ └─────────────────────────────────────────────────────────────┘ │ │
@@ -167,7 +167,7 @@ Scenario 8 validates synchronising entitlement groups (security groups, distribu
The member attribute contains DN references that must be transformed between domains:
```
-Source AD member: CN=John Smith,OU=Users,OU=Corp,DC=sourcedomain,DC=local
+Source AD member: CN=John Smith,OU=Users,OU=Corp,DC=resurgam,DC=local
↓
Import (CSO reference)
↓
@@ -175,7 +175,7 @@ Metaverse member: Reference to User MVO (John Smith)
↓
Export (DN calculation)
↓
-Target AD member: CN=John Smith,OU=Users,OU=CorpManaged,DC=targetdomain,DC=local
+Target AD member: CN=John Smith,OU=Users,OU=CorpManaged,DC=gentian,DC=local
```
**Prerequisites**: Users must be synced between domains first so that:
@@ -191,14 +191,14 @@ Target AD member: CN=John Smith,OU=Users,OU=CorpManaged,DC=targetdomain,DC=loca
| System | Role | Container | Domain |
|--------|------|-----------|--------|
-| Quantum Dynamics APAC | Source (Authoritative) | `samba-ad-source` | `DC=sourcedomain,DC=local` |
-| Quantum Dynamics EMEA | Target (Replica) | `samba-ad-target` | `DC=targetdomain,DC=local` |
+| Panoply APAC | Source (Authoritative) | `samba-ad-source` | `DC=resurgam,DC=local` |
+| Panoply EMEA | Target (Replica) | `samba-ad-target` | `DC=gentian,DC=local` |
### Container Structure
**Source AD:**
```
-DC=sourcedomain,DC=local
+DC=resurgam,DC=local
└── OU=Corp
├── OU=Users
└── OU=Entitlements
@@ -206,7 +206,7 @@ DC=sourcedomain,DC=local
**Target AD:**
```
-DC=targetdomain,DC=local
+DC=gentian,DC=local
└── OU=CorpManaged
├── OU=Users
└── OU=Entitlements
@@ -272,7 +272,7 @@ Groups follow a hierarchical naming model for realistic enterprise representatio
| Category | Pattern | Example | Purpose |
|----------|---------|---------|---------|
-| Company | `Company-{Name}` | `Company-Subatomic` | Top-level organisational groups |
+| Company | `Company-{Name}` | `Company-Panoply` | Top-level organisational groups |
| Department | `Dept-{Name}` | `Dept-Engineering` | Functional team groups |
| Location | `Location-{City}` | `Location-Sydney` | Geographic groups |
| Project | `Project-{Name}` | `Project-Phoenix` | Dynamic project teams (scalable) |
@@ -336,9 +336,9 @@ When a group is mail-enabled, these additional attributes are populated:
| Attribute | Required | Example Value |
|-----------|----------|---------------|
-| `mail` | Yes | `dept-engineering@sourcedomain.local` |
+| `mail` | Yes | `dept-engineering@resurgam.local` |
| `mailNickname` | Yes | `Dept-Engineering` |
-| `proxyAddresses` | Optional | `SMTP:dept-engineering@sourcedomain.local` |
+| `proxyAddresses` | Optional | `SMTP:dept-engineering@resurgam.local` |
#### Naming Convention for Mail-Enabled Groups
@@ -346,7 +346,7 @@ Mail-enabled groups follow a pattern that indicates their email address:
| Category | Group Name | Mail Address |
|----------|------------|--------------|
-| Company | `Company-Subatomic` | `company-subatomic@domain.local` |
+| Company | `Company-Panoply` | `company-panoply@domain.local` |
| Department | `Dept-Engineering` | `dept-engineering@domain.local` |
| Location | `Location-Sydney` | `location-sydney@domain.local` |
| Project | `Project-Phoenix` | `project-phoenix@domain.local` |
@@ -377,7 +377,7 @@ This tests the single-valued DN reference attribute sync, which uses the same re
### Reference Data
**Company Names:**
-- Subatomic, NexusDynamics, OrbitalSystems, QuantumBridge, StellarLogistics
+- Panoply, NexusDynamics, OrbitalSystems, QuantumBridge, StellarLogistics
- VortexTech, CatalystCorp, HorizonIndustries, PulsarEnterprises, NovaNetworks
**Department Names:**
@@ -606,7 +606,7 @@ function Get-Scenario8GroupScale {
**DN Expression:**
```
-"CN=" + EscapeDN(mv["Display Name"]) + ",OU=Entitlements,OU=CorpManaged,DC=targetdomain,DC=local"
+"CN=" + EscapeDN(mv["Display Name"]) + ",OU=Entitlements,OU=CorpManaged,DC=gentian,DC=local"
```
### Phase 3: Test Script
diff --git a/src/JIM.Application/Interfaces/ISyncEngine.cs b/src/JIM.Application/Interfaces/ISyncEngine.cs
index b2ef7bb25..0b3ac5c2a 100644
--- a/src/JIM.Application/Interfaces/ISyncEngine.cs
+++ b/src/JIM.Application/Interfaces/ISyncEngine.cs
@@ -37,6 +37,7 @@ ProjectionDecision EvaluateProjection(
///
/// Flows inbound attribute values from a CSO to its joined MVO using a sync rule's attribute flow mappings.
/// Mutates the MVO's PendingAttributeValueAdditions and PendingAttributeValueRemovals collections.
+ /// Returns any warnings generated during attribute flow (e.g. multi-valued to single-valued truncation).
///
/// The source CSO (must have MetaverseObject set).
/// The sync rule defining attribute flow mappings.
@@ -45,7 +46,8 @@ ProjectionDecision EvaluateProjection(
/// If true, skip reference attributes (deferred to second pass).
/// If true, process only reference attributes.
/// If true, this is the final cross-page resolution pass.
- void FlowInboundAttributes(
+ /// A list of warnings generated during attribute flow, empty if none.
+ List FlowInboundAttributes(
ConnectedSystemObject cso,
SyncRule syncRule,
IReadOnlyList objectTypes,
@@ -98,13 +100,27 @@ InboundOutOfScopeAction DetermineOutOfScopeAction(
IReadOnlyList activeSyncRules);
///
- /// Checks if a CSO attribute value matches a pending export attribute change value.
- /// Used during export confirmation to verify whether exported changes were persisted.
+ /// Reconciles a Connected System Object against a pre-loaded pending export.
+ /// Compares imported CSO attribute values against pending export assertions to confirm,
+ /// mark for retry, or mark as failed. This method does NOT perform any database operations —
+ /// the caller is responsible for persistence.
///
- /// The CSO's current attribute value.
- /// The pending export attribute change to compare against.
- /// True if the values match.
- bool AttributeValuesMatch(
- ConnectedSystemObjectAttributeValue csoValue,
- PendingExportAttributeValueChange pendingChange);
+ /// The CSO that was just imported/updated.
+ /// The pre-loaded pending export for this CSO (or null if none).
+ /// The result object to populate with reconciliation outcomes.
+ void ReconcileCsoAgainstPendingExport(
+ ConnectedSystemObject connectedSystemObject,
+ PendingExport? pendingExport,
+ PendingExportReconciliationResult result);
+
+ ///
+ /// Determines if an attribute change has been confirmed by comparing the exported value
+ /// against the imported CSO attribute value. Handles all attribute data types comprehensively.
+ ///
+ /// The CSO whose current attributes to check.
+ /// The pending export attribute change to verify.
+ /// True if the attribute change has been confirmed by the CSO's current state.
+ bool IsAttributeChangeConfirmed(
+ ConnectedSystemObject cso,
+ PendingExportAttributeValueChange attrChange);
}
diff --git a/src/JIM.Application/Servers/ConnectedSystemServer.cs b/src/JIM.Application/Servers/ConnectedSystemServer.cs
index 15532669d..4a680d753 100644
--- a/src/JIM.Application/Servers/ConnectedSystemServer.cs
+++ b/src/JIM.Application/Servers/ConnectedSystemServer.cs
@@ -3370,6 +3370,8 @@ private static void AddChangeAttributeValueObject(ConnectedSystemObjectChange co
attributeChange = new ConnectedSystemObjectChangeAttribute
{
Attribute = connectedSystemObjectAttributeValue.Attribute,
+ AttributeName = connectedSystemObjectAttributeValue.Attribute.Name,
+ AttributeType = connectedSystemObjectAttributeValue.Attribute.Type,
ConnectedSystemChange = connectedSystemObjectChange
};
connectedSystemObjectChange.AttributeChanges.Add(attributeChange);
@@ -3616,10 +3618,8 @@ private static void ValidateMappingTypeCompatibility(SyncRuleMapping mapping)
throw new ArgumentException(
$"Type mismatch: source attribute '{sourceAttrName}' ({sourceType}) is not compatible with target attribute '{targetAttrName}' ({targetType}). Source and target attributes must have the same type.");
- // Validate plurality compatibility - cannot flow multi-valued to single-valued
- if (sourcePlurality == AttributePlurality.MultiValued && targetPlurality == AttributePlurality.SingleValued)
- throw new ArgumentException(
- $"Plurality mismatch: cannot flow multi-valued source attribute '{sourceAttrName}' to single-valued target attribute '{targetAttrName}'. A single-valued attribute cannot hold multiple values.");
+ // Multi-valued to single-valued is permitted (#435). The runtime selects
+ // the first value and generates a MultiValuedAttributeTruncated RPEI warning.
}
}
diff --git a/src/JIM.Application/Servers/ExportEvaluationServer.cs b/src/JIM.Application/Servers/ExportEvaluationServer.cs
index b19b4de20..0e5e1f504 100644
--- a/src/JIM.Application/Servers/ExportEvaluationServer.cs
+++ b/src/JIM.Application/Servers/ExportEvaluationServer.cs
@@ -1879,7 +1879,7 @@ internal static bool HasRelevantChangedAttributes(
///
private Dictionary BuildAttributeDictionary(MetaverseObject mvo)
{
- var attributes = new Dictionary();
+ var attributes = new Dictionary(StringComparer.OrdinalIgnoreCase);
if (mvo.Type == null)
{
diff --git a/src/JIM.Application/Servers/MetaverseServer.cs b/src/JIM.Application/Servers/MetaverseServer.cs
index 7054c6e08..080570dc4 100644
--- a/src/JIM.Application/Servers/MetaverseServer.cs
+++ b/src/JIM.Application/Servers/MetaverseServer.cs
@@ -396,6 +396,8 @@ internal static void AddMvoChangeAttributeValueObject(
attributeChange = new MetaverseObjectChangeAttribute
{
Attribute = metaverseObjectAttributeValue.Attribute,
+ AttributeName = metaverseObjectAttributeValue.Attribute.Name,
+ AttributeType = metaverseObjectAttributeValue.Attribute.Type,
MetaverseObjectChange = metaverseObjectChange
};
metaverseObjectChange.AttributeChanges.Add(attributeChange);
@@ -436,9 +438,20 @@ internal static void AddMvoChangeAttributeValueObject(
attributeChange.ValueChanges.Add(new MetaverseObjectChangeAttributeValue(
attributeChange, valueChangeType, metaverseObjectAttributeValue.ReferenceValue));
break;
+ case AttributeDataType.Reference when metaverseObjectAttributeValue.ReferenceValueId.HasValue:
+ // Navigation property not loaded but FK is set — record the referenced MVO ID as a GUID.
+ // This happens when attribute values are snapshotted without eagerly loading navigation
+ // properties (e.g., during MVO deletion where the referenced MVOs may not be in the
+ // EF change tracker).
+ attributeChange.ValueChanges.Add(new MetaverseObjectChangeAttributeValue(
+ attributeChange, valueChangeType, metaverseObjectAttributeValue.ReferenceValueId.Value));
+ break;
case AttributeDataType.Reference when metaverseObjectAttributeValue.UnresolvedReferenceValue != null:
// Don't track unresolved references
break;
+ case AttributeDataType.Reference:
+ // Reference attribute with no resolved or unresolved value — nothing to track
+ break;
default:
throw new NotImplementedException(
$"Attribute data type {metaverseObjectAttributeValue.Attribute.Type} is not yet supported for MVO change tracking.");
diff --git a/src/JIM.Application/Servers/SyncEngine.AttributeFlow.cs b/src/JIM.Application/Servers/SyncEngine.AttributeFlow.cs
index 59be1b3c4..1f8611a57 100644
--- a/src/JIM.Application/Servers/SyncEngine.AttributeFlow.cs
+++ b/src/JIM.Application/Servers/SyncEngine.AttributeFlow.cs
@@ -3,6 +3,7 @@
using JIM.Models.Interfaces;
using JIM.Models.Logic;
using JIM.Models.Staging;
+using JIM.Models.Sync;
using JIM.Models.Utility;
using Serilog;
@@ -25,7 +26,8 @@ internal static void ProcessMapping(
bool skipReferenceAttributes = false,
bool onlyReferenceAttributes = false,
bool isFinalReferencePass = false,
- int? contributingSystemId = null)
+ int? contributingSystemId = null,
+ List? warnings = null)
{
if (cso.MetaverseObject == null)
{
@@ -56,6 +58,45 @@ internal static void ProcessMapping(
if (onlyReferenceAttributes && csotAttribute.Type != AttributeDataType.Reference)
continue;
+ // MVA -> SVA truncation: when multiple source values target a single-valued
+ // MV attribute, take only the first value and record a warning (#435)
+ var isMvaToSva = csoAttributeValues.Count > 1 &&
+ syncRuleMapping.TargetMetaverseAttribute.AttributePlurality == AttributePlurality.SingleValued;
+
+ if (isMvaToSva)
+ {
+ var firstValue = csoAttributeValues.First();
+ var selectedValueDescription = csotAttribute.Type switch
+ {
+ AttributeDataType.Text => firstValue.StringValue ?? "(null)",
+ AttributeDataType.Number => firstValue.IntValue?.ToString() ?? "(null)",
+ AttributeDataType.LongNumber => firstValue.LongValue?.ToString() ?? "(null)",
+ AttributeDataType.DateTime => firstValue.DateTimeValue?.ToString("O") ?? "(null)",
+ AttributeDataType.Boolean => firstValue.BoolValue?.ToString() ?? "(null)",
+ AttributeDataType.Guid => firstValue.GuidValue?.ToString() ?? "(null)",
+ AttributeDataType.Binary => firstValue.ByteValue != null
+ ? $"[{firstValue.ByteValue.Length} bytes]" : "(null)",
+ _ => "(unknown)"
+ };
+
+ Log.Warning(
+ "ProcessMapping: Multi-valued source attribute '{SourceAttr}' has {ValueCount} values but target " +
+ "attribute '{TargetAttr}' is single-valued. Using first value: '{SelectedValue}'. CSO {CsoId}",
+ csotAttribute.Name, csoAttributeValues.Count,
+ syncRuleMapping.TargetMetaverseAttribute.Name, selectedValueDescription, cso.Id);
+
+ warnings?.Add(new AttributeFlowWarning
+ {
+ SourceAttributeName = csotAttribute.Name,
+ TargetAttributeName = syncRuleMapping.TargetMetaverseAttribute.Name,
+ ValueCount = csoAttributeValues.Count,
+ SelectedValue = selectedValueDescription
+ });
+
+ // Truncate to the first value only
+ csoAttributeValues = new List { firstValue };
+ }
+
switch (csotAttribute.Type)
{
case AttributeDataType.Text:
@@ -211,8 +252,9 @@ private static void ProcessTextAttribute(
!csoAttributeValues.Any(csoav => csoav.StringValue != null && csoav.StringValue.Equals(mvoav.StringValue)));
mvo.PendingAttributeValueRemovals.AddRange(mvoObsoleteAttributeValues);
- var csoNewAttributeValues = cso.AttributeValues.Where(csoav =>
- csoav.AttributeId == sourceAttributeId &&
+ // Use the (possibly truncated) csoAttributeValues list rather than cso.AttributeValues
+ // to respect MVA->SVA first-value selection (#435)
+ var csoNewAttributeValues = csoAttributeValues.Where(csoav =>
!mvo.AttributeValues.Any(mvoav =>
mvoav.AttributeId == syncRuleMapping.TargetMetaverseAttribute!.Id &&
mvoav.StringValue != null && mvoav.StringValue.Equals(csoav.StringValue)));
@@ -249,8 +291,9 @@ private static void ProcessNumberAttribute(
!csoAttributeValues.Any(csoav => csoav.IntValue != null && csoav.IntValue.Equals(mvoav.IntValue)));
mvo.PendingAttributeValueRemovals.AddRange(mvoObsoleteAttributeValues);
- var csoNewAttributeValues = cso.AttributeValues.Where(csoav =>
- csoav.AttributeId == sourceAttributeId &&
+ // Use the (possibly truncated) csoAttributeValues list rather than cso.AttributeValues
+ // to respect MVA->SVA first-value selection (#435)
+ var csoNewAttributeValues = csoAttributeValues.Where(csoav =>
!mvo.AttributeValues.Any(mvoav =>
mvoav.AttributeId == syncRuleMapping.TargetMetaverseAttribute!.Id &&
mvoav.IntValue != null && mvoav.IntValue.Equals(csoav.IntValue)));
@@ -320,8 +363,9 @@ private static void ProcessBinaryAttribute(
csoav.ByteValue != null && JIM.Utilities.Utilities.AreByteArraysTheSame(csoav.ByteValue, mvoav.ByteValue)));
mvo.PendingAttributeValueRemovals.AddRange(mvoObsoleteAttributeValues);
- var csoNewAttributeValues = cso.AttributeValues.Where(csoav =>
- csoav.AttributeId == sourceAttributeId &&
+ // Use the (possibly truncated) csoAttributeValues list rather than cso.AttributeValues
+ // to respect MVA->SVA first-value selection (#435)
+ var csoNewAttributeValues = csoAttributeValues.Where(csoav =>
!mvo.AttributeValues.Any(mvoav =>
mvoav.AttributeId == syncRuleMapping.TargetMetaverseAttribute!.Id &&
JIM.Utilities.Utilities.AreByteArraysTheSame(mvoav.ByteValue, csoav.ByteValue)));
@@ -348,15 +392,21 @@ private static void ProcessReferenceAttribute(
bool isFinalReferencePass,
int? contributingSystemId)
{
+ // Use ResolvedReferenceMetaverseObjectId (populated via direct SQL) as the primary
+ // source, falling back to navigation properties for compatibility with in-memory tests.
static Guid? GetReferencedMvoId(ConnectedSystemObjectAttributeValue csoav)
{
+ if (csoav.ResolvedReferenceMetaverseObjectId.HasValue)
+ return csoav.ResolvedReferenceMetaverseObjectId;
if (csoav.ReferenceValue == null)
return null;
return csoav.ReferenceValue.MetaverseObjectId ?? csoav.ReferenceValue.MetaverseObject?.Id;
}
+ // A reference is resolved if we can determine the MVO it points to — either via
+ // ResolvedReferenceMetaverseObjectId (direct SQL) or ReferenceValue navigation (in-memory tests).
static bool IsResolved(ConnectedSystemObjectAttributeValue csoav)
- => csoav.ReferenceValue != null && GetReferencedMvoId(csoav).HasValue;
+ => (csoav.ReferenceValueId.HasValue || csoav.ReferenceValue != null) && GetReferencedMvoId(csoav).HasValue;
var unresolvedReferenceValues = csoAttributeValues.Where(csoav =>
!IsResolved(csoav) &&
@@ -366,25 +416,20 @@ static bool IsResolved(ConnectedSystemObjectAttributeValue csoav)
{
foreach (var unresolved in unresolvedReferenceValues)
{
- if (unresolved.ReferenceValue == null && unresolved.ReferenceValueId != null)
- {
- Log.Warning("SyncEngine: CSO {CsoId} has reference attribute {AttrName} with ReferenceValueId {RefId} but ReferenceValue navigation is null. " +
- "This indicates the EF Core query is missing .Include(av => av.ReferenceValue). The reference will not flow to the MVO.",
- cso.Id, source.ConnectedSystemAttribute?.Name ?? "unknown", unresolved.ReferenceValueId);
- }
- else if (unresolved.ReferenceValue != null && !unresolved.ReferenceValue.MetaverseObjectId.HasValue)
+ if (unresolved.ReferenceValueId.HasValue && !unresolved.ResolvedReferenceMetaverseObjectId.HasValue)
{
+ // Referenced CSO exists but isn't joined to an MVO yet.
if (isFinalReferencePass)
{
Log.Warning("SyncEngine: CSO {CsoId} has reference attribute {AttrName} pointing to CSO {RefCsoId} which is not joined to an MVO. " +
"Ensure referenced objects are synced before referencing objects. The reference will not flow to the MVO.",
- cso.Id, source.ConnectedSystemAttribute?.Name ?? "unknown", unresolved.ReferenceValue.Id);
+ cso.Id, source.ConnectedSystemAttribute?.Name ?? "unknown", unresolved.ReferenceValueId);
}
else
{
Log.Debug("SyncEngine: CSO {CsoId} has reference attribute {AttrName} pointing to CSO {RefCsoId} which is not yet joined to an MVO. " +
"This will be retried during cross-page reference resolution.",
- cso.Id, source.ConnectedSystemAttribute?.Name ?? "unknown", unresolved.ReferenceValue.Id);
+ cso.Id, source.ConnectedSystemAttribute?.Name ?? "unknown", unresolved.ReferenceValueId);
}
}
}
@@ -441,6 +486,10 @@ static bool IsResolved(ConnectedSystemObjectAttributeValue csoav)
ContributedBySystemId = contributingSystemId
};
+ // When the referenced MVO is available as a tracked navigation (same-page reference),
+ // set the navigation so EF handles insert ordering (the MVO may not be persisted yet).
+ // For cross-page references (navigation unavailable), set the scalar FK directly —
+ // the referenced MVO already exists in the database.
if (newCsoNewAttributeValue.ReferenceValue?.MetaverseObject != null)
newMvoAv.ReferenceValue = newCsoNewAttributeValue.ReferenceValue.MetaverseObject;
else
@@ -463,8 +512,9 @@ private static void ProcessGuidAttribute(
!csoAttributeValues.Any(csoav => csoav.GuidValue.HasValue && csoav.GuidValue.Equals(mvoav.GuidValue)));
mvo.PendingAttributeValueRemovals.AddRange(mvoObsoleteAttributeValues);
- var csoNewAttributeValues = cso.AttributeValues.Where(csoav =>
- csoav.AttributeId == sourceAttributeId &&
+ // Use the (possibly truncated) csoAttributeValues list rather than cso.AttributeValues
+ // to respect MVA->SVA first-value selection (#435)
+ var csoNewAttributeValues = csoAttributeValues.Where(csoav =>
!mvo.AttributeValues.Any(mvoav =>
mvoav.AttributeId == syncRuleMapping.TargetMetaverseAttribute!.Id &&
mvoav.GuidValue.HasValue && mvoav.GuidValue.Equals(csoav.GuidValue)));
@@ -524,7 +574,7 @@ private static void ProcessBooleanAttribute(
ConnectedSystemObject cso,
ConnectedSystemObjectType csoType)
{
- var attributes = new Dictionary();
+ var attributes = new Dictionary(StringComparer.OrdinalIgnoreCase);
foreach (var attributeValue in cso.AttributeValues)
{
diff --git a/src/JIM.Application/Servers/SyncEngine.Reconciliation.cs b/src/JIM.Application/Servers/SyncEngine.Reconciliation.cs
new file mode 100644
index 000000000..087c236e4
--- /dev/null
+++ b/src/JIM.Application/Servers/SyncEngine.Reconciliation.cs
@@ -0,0 +1,353 @@
+using JIM.Models.Core;
+using JIM.Models.Staging;
+using JIM.Models.Transactional;
+using Serilog;
+
+namespace JIM.Application.Servers;
+
+///
+/// Pending export reconciliation logic — pure, stateless methods for comparing
+/// imported CSO attribute values against pending export assertions.
+/// Consolidated from PendingExportReconciliationService to ensure both
+/// the sync path and import path use identical, comprehensive attribute matching.
+///
+public partial class SyncEngine
+{
+ ///
+ /// Default maximum number of export attempts before marking an attribute change as Failed.
+ ///
+ public const int DefaultMaxRetries = 5;
+
+ ///
+ /// Reconciles a Connected System Object against a pre-loaded pending export.
+ /// This method does NOT perform any database operations — the caller is responsible for persistence.
+ ///
+ /// The CSO that was just imported/updated.
+ /// The pre-loaded pending export for this CSO (or null if none).
+ /// The result object to populate with reconciliation outcomes.
+ public void ReconcileCsoAgainstPendingExport(
+ ConnectedSystemObject connectedSystemObject,
+ PendingExport? pendingExport,
+ PendingExportReconciliationResult result)
+ {
+ if (pendingExport == null)
+ {
+ Log.Debug("ReconcileCsoAgainstPendingExport: No pending export for CSO {CsoId}", connectedSystemObject.Id);
+ return;
+ }
+
+ // Only process exports that have been executed and are awaiting confirmation
+ if (pendingExport.Status != PendingExportStatus.Exported &&
+ pendingExport.Status != PendingExportStatus.ExportNotConfirmed)
+ {
+ Log.Debug("ReconcileCsoAgainstPendingExport: PendingExport {ExportId} status is {Status}, not awaiting confirmation. Skipping.",
+ pendingExport.Id, pendingExport.Status);
+ return;
+ }
+
+ Log.Debug("ReconcileCsoAgainstPendingExport: Found pending export {ExportId} with {Count} attribute changes for CSO {CsoId}",
+ pendingExport.Id, pendingExport.AttributeValueChanges.Count, connectedSystemObject.Id);
+
+ // Process each attribute change that is awaiting confirmation
+ var changesAwaitingConfirmation = pendingExport.AttributeValueChanges
+ .Where(ac => ac.Status == PendingExportAttributeChangeStatus.ExportedPendingConfirmation ||
+ ac.Status == PendingExportAttributeChangeStatus.ExportedNotConfirmed)
+ .ToList();
+
+ foreach (var attrChange in changesAwaitingConfirmation)
+ {
+ var confirmed = IsAttributeChangeConfirmed(connectedSystemObject, attrChange);
+
+ Log.Verbose("ReconcileCsoAgainstPendingExport: Comparing attribute {AttrName} (ChangeType: {ChangeType}) for CSO {CsoId}. " +
+ "Expected: '{ExpectedValue}', Found: '{ActualValue}', Confirmed: {Confirmed}",
+ attrChange.Attribute?.Name ?? "unknown",
+ attrChange.ChangeType,
+ connectedSystemObject.Id,
+ GetExpectedValueAsString(attrChange),
+ GetImportedValueAsString(connectedSystemObject, attrChange),
+ confirmed);
+
+ if (confirmed)
+ {
+ result.ConfirmedChanges.Add(attrChange);
+ Log.Debug("ReconcileCsoAgainstPendingExport: Attribute change {AttrChangeId} (Attr: {AttrName}) confirmed",
+ attrChange.Id, attrChange.Attribute?.Name ?? "unknown");
+ }
+ else
+ {
+ attrChange.Status = ShouldMarkAsFailed(attrChange)
+ ? PendingExportAttributeChangeStatus.Failed
+ : PendingExportAttributeChangeStatus.ExportedNotConfirmed;
+
+ attrChange.LastImportedValue = GetImportedValueAsString(connectedSystemObject, attrChange);
+ var expectedValue = GetExpectedValueAsString(attrChange);
+
+ if (attrChange.Status == PendingExportAttributeChangeStatus.Failed)
+ {
+ result.FailedChanges.Add(attrChange);
+ Log.Warning("ReconcileCsoAgainstPendingExport: Attribute change {AttrChangeId} (Attr: {AttrName}) failed after {Attempts} attempts. " +
+ "Expected: '{ExpectedValue}', Actual: '{ImportedValue}'",
+ attrChange.Id, attrChange.Attribute?.Name ?? "unknown", attrChange.ExportAttemptCount,
+ expectedValue, attrChange.LastImportedValue);
+ }
+ else
+ {
+ result.RetryChanges.Add(attrChange);
+ Log.Information("ReconcileCsoAgainstPendingExport: Attribute change {AttrChangeId} (Attr: {AttrName}) not confirmed, will retry (attempt {Attempt}). " +
+ "Expected: '{ExpectedValue}', Actual: '{ImportedValue}'",
+ attrChange.Id, attrChange.Attribute?.Name ?? "unknown", attrChange.ExportAttemptCount,
+ expectedValue, attrChange.LastImportedValue);
+ }
+ }
+ }
+
+ // Remove confirmed changes from the pending export
+ foreach (var confirmed in result.ConfirmedChanges)
+ pendingExport.AttributeValueChanges.Remove(confirmed);
+
+ // If this was a Create and the Secondary External ID was confirmed, transition to Update
+ TransitionCreateToUpdateIfSecondaryExternalIdConfirmed(pendingExport, result);
+
+ // Determine if the pending export should be deleted or updated
+ var hasRemainingChanges = pendingExport.AttributeValueChanges.Any(ac =>
+ ac.Status == PendingExportAttributeChangeStatus.Pending ||
+ ac.Status == PendingExportAttributeChangeStatus.ExportedPendingConfirmation ||
+ ac.Status == PendingExportAttributeChangeStatus.ExportedNotConfirmed ||
+ ac.Status == PendingExportAttributeChangeStatus.Failed);
+
+ if (!hasRemainingChanges)
+ {
+ result.PendingExportDeleted = true;
+ result.PendingExportToDelete = pendingExport;
+ }
+ else
+ {
+ UpdatePendingExportStatus(pendingExport);
+ result.PendingExportToUpdate = pendingExport;
+ }
+ }
+
+ ///
+ /// Determines if an attribute change has been confirmed by comparing the exported value
+ /// against the imported CSO attribute value. Handles all attribute data types comprehensively.
+ ///
+ public bool IsAttributeChangeConfirmed(ConnectedSystemObject cso, PendingExportAttributeValueChange attrChange)
+ {
+ if (attrChange.Attribute == null)
+ return false;
+
+ var csoAttrValues = cso.AttributeValues
+ .Where(av => av.AttributeId == attrChange.AttributeId)
+ .ToList();
+
+ switch (attrChange.ChangeType)
+ {
+ case PendingExportAttributeChangeType.Add:
+ case PendingExportAttributeChangeType.Update:
+ // For Add/Update with a null/empty value (clearing a single-valued attribute),
+ // confirmation means the CSO should have no values for this attribute.
+ if (IsPendingChangeEmpty(attrChange))
+ return csoAttrValues.Count == 0;
+
+ // For Add/Update with a real value, the value should exist on the CSO
+ return ValueExistsOnCso(csoAttrValues, attrChange);
+
+ case PendingExportAttributeChangeType.Remove:
+ // For Remove, the value should NOT exist on the CSO
+ return !ValueExistsOnCso(csoAttrValues, attrChange);
+
+ case PendingExportAttributeChangeType.RemoveAll:
+ // For RemoveAll, there should be no values for this attribute
+ return csoAttrValues.Count == 0;
+
+ default:
+ return false;
+ }
+ }
+
+ ///
+ /// Checks if the pending export attribute change value exists in the CSO's attribute values.
+ /// Uses type-aware comparison based on the attribute's data type.
+ ///
+ public static bool ValueExistsOnCso(List csoValues, PendingExportAttributeValueChange attrChange)
+ {
+ if (csoValues.Count == 0)
+ return false;
+
+ var attrType = attrChange.Attribute?.Type ?? AttributeDataType.NotSet;
+
+ return attrType switch
+ {
+ AttributeDataType.Text =>
+ !string.IsNullOrEmpty(attrChange.StringValue) &&
+ csoValues.Any(v => string.Equals(v.StringValue, attrChange.StringValue, StringComparison.Ordinal)),
+
+ AttributeDataType.Number =>
+ attrChange.IntValue.HasValue &&
+ csoValues.Any(v => v.IntValue == attrChange.IntValue),
+
+ AttributeDataType.LongNumber =>
+ attrChange.LongValue.HasValue &&
+ csoValues.Any(v => v.LongValue == attrChange.LongValue),
+
+ AttributeDataType.DateTime =>
+ attrChange.DateTimeValue.HasValue &&
+ csoValues.Any(v => v.DateTimeValue == attrChange.DateTimeValue),
+
+ AttributeDataType.Binary =>
+ attrChange.ByteValue != null &&
+ csoValues.Any(v => v.ByteValue != null && v.ByteValue.SequenceEqual(attrChange.ByteValue)),
+
+ AttributeDataType.Boolean =>
+ attrChange.BoolValue.HasValue &&
+ csoValues.Any(v => v.BoolValue == attrChange.BoolValue),
+
+ AttributeDataType.Guid =>
+ attrChange.GuidValue.HasValue &&
+ csoValues.Any(v => v.GuidValue == attrChange.GuidValue),
+
+ AttributeDataType.Reference =>
+ // Reference attributes can be stored in two ways:
+ // 1. UnresolvedReferenceValue — the raw DN string (before or after resolution)
+ // 2. StringValue — set during export resolution (when UnresolvedReferenceValue is cleared)
+ // We need to check both the pending export value AND the CSO values to handle both cases
+ (!string.IsNullOrEmpty(attrChange.UnresolvedReferenceValue) &&
+ csoValues.Any(v => string.Equals(v.UnresolvedReferenceValue, attrChange.UnresolvedReferenceValue, StringComparison.Ordinal))) ||
+ (!string.IsNullOrEmpty(attrChange.StringValue) &&
+ csoValues.Any(v => string.Equals(v.UnresolvedReferenceValue, attrChange.StringValue, StringComparison.Ordinal))),
+
+ _ => false
+ };
+ }
+
+ ///
+ /// Checks if a pending export attribute value change represents an empty/null value
+ /// (i.e., clearing a single-valued attribute).
+ ///
+ public static bool IsPendingChangeEmpty(PendingExportAttributeValueChange change)
+ {
+ return change.StringValue == null &&
+ !change.IntValue.HasValue &&
+ !change.LongValue.HasValue &&
+ !change.DateTimeValue.HasValue &&
+ !change.BoolValue.HasValue &&
+ !change.GuidValue.HasValue &&
+ change.ByteValue == null &&
+ change.UnresolvedReferenceValue == null;
+ }
+
+ ///
+ /// Determines if an attribute change should be marked as Failed based on retry count.
+ ///
+ public static bool ShouldMarkAsFailed(PendingExportAttributeValueChange attrChange)
+ {
+ return attrChange.ExportAttemptCount >= DefaultMaxRetries;
+ }
+
+ ///
+ /// If the pending export was a Create and the Secondary External ID attribute has been confirmed,
+ /// transition it to an Update. Once an object is created, remaining unconfirmed attribute changes
+ /// should be applied as updates. Connectors require the Secondary External ID (e.g., distinguishedName
+ /// for LDAP) in the attribute changes for Create operations, but once confirmed, it is removed.
+ /// Without this transition, retry attempts would fail because the connector cannot determine
+ /// where to create the object.
+ ///
+ public static void TransitionCreateToUpdateIfSecondaryExternalIdConfirmed(PendingExport pendingExport, PendingExportReconciliationResult result)
+ {
+ if (pendingExport.ChangeType != PendingExportChangeType.Create)
+ return;
+
+ var secondaryExternalIdWasConfirmed = result.ConfirmedChanges.Any(ac =>
+ ac.Attribute?.IsSecondaryExternalId == true);
+
+ if (!secondaryExternalIdWasConfirmed)
+ return;
+
+ if (pendingExport.AttributeValueChanges.Count > 0)
+ {
+ var confirmedAttrName = result.ConfirmedChanges
+ .FirstOrDefault(ac => ac.Attribute?.IsSecondaryExternalId == true)?.Attribute?.Name ?? "unknown";
+
+ pendingExport.ChangeType = PendingExportChangeType.Update;
+ Log.Information("ReconcileCsoAgainstPendingExport: Transitioned pending export {ExportId} from Create to Update. " +
+ "Secondary External ID attribute '{AttributeName}' was confirmed but {RemainingCount} attribute changes remain.",
+ pendingExport.Id, confirmedAttrName, pendingExport.AttributeValueChanges.Count);
+ }
+ }
+
+ ///
+ /// Updates the PendingExport status based on its attribute change statuses.
+ ///
+ public static void UpdatePendingExportStatus(PendingExport pendingExport)
+ {
+ var allFailed = pendingExport.AttributeValueChanges.All(ac => ac.Status == PendingExportAttributeChangeStatus.Failed);
+ var anyFailed = pendingExport.AttributeValueChanges.Any(ac => ac.Status == PendingExportAttributeChangeStatus.Failed);
+ var anyPendingOrRetry = pendingExport.AttributeValueChanges.Any(ac =>
+ ac.Status == PendingExportAttributeChangeStatus.Pending ||
+ ac.Status == PendingExportAttributeChangeStatus.ExportedNotConfirmed);
+
+ if (allFailed)
+ {
+ pendingExport.Status = PendingExportStatus.Failed;
+ }
+ else if (anyPendingOrRetry)
+ {
+ pendingExport.Status = PendingExportStatus.ExportNotConfirmed;
+ }
+ else if (anyFailed)
+ {
+ pendingExport.Status = PendingExportStatus.Exported;
+ }
+ }
+
+ ///
+ /// Gets the imported value as a string for debugging purposes.
+ ///
+ private static string? GetImportedValueAsString(ConnectedSystemObject cso, PendingExportAttributeValueChange attrChange)
+ {
+ var csoAttrValues = cso.AttributeValues
+ .Where(av => av.AttributeId == attrChange.AttributeId)
+ .ToList();
+
+ if (csoAttrValues.Count == 0)
+ return "(no values)";
+
+ var attrType = attrChange.Attribute?.Type ?? AttributeDataType.NotSet;
+
+ var values = attrType switch
+ {
+ AttributeDataType.Text => csoAttrValues.Select(v => v.StringValue).Where(v => v != null),
+ AttributeDataType.Number => csoAttrValues.Select(v => v.IntValue?.ToString()).Where(v => v != null),
+ AttributeDataType.LongNumber => csoAttrValues.Select(v => v.LongValue?.ToString()).Where(v => v != null),
+ AttributeDataType.DateTime => csoAttrValues.Select(v => v.DateTimeValue?.ToString("O")).Where(v => v != null),
+ AttributeDataType.Boolean => csoAttrValues.Select(v => v.BoolValue?.ToString()).Where(v => v != null),
+ AttributeDataType.Guid => csoAttrValues.Select(v => v.GuidValue?.ToString()).Where(v => v != null),
+ AttributeDataType.Reference => csoAttrValues.Select(v => v.UnresolvedReferenceValue).Where(v => v != null),
+ _ => Enumerable.Empty()
+ };
+
+ var valueList = values.ToList();
+ return valueList.Count > 0 ? string.Join(", ", valueList) : "(no matching type values)";
+ }
+
+ ///
+ /// Gets the expected (exported) value as a string for debugging purposes.
+ ///
+ private static string? GetExpectedValueAsString(PendingExportAttributeValueChange attrChange)
+ {
+ var attrType = attrChange.Attribute?.Type ?? AttributeDataType.NotSet;
+
+ return attrType switch
+ {
+ AttributeDataType.Text => attrChange.StringValue ?? "(null)",
+ AttributeDataType.Number => attrChange.IntValue?.ToString() ?? "(null)",
+ AttributeDataType.LongNumber => attrChange.LongValue?.ToString() ?? "(null)",
+ AttributeDataType.DateTime => attrChange.DateTimeValue?.ToString("O") ?? "(null)",
+ AttributeDataType.Boolean => attrChange.BoolValue?.ToString() ?? "(null)",
+ AttributeDataType.Guid => attrChange.GuidValue?.ToString() ?? "(null)",
+ AttributeDataType.Reference => attrChange.UnresolvedReferenceValue ?? "(null)",
+ AttributeDataType.Binary => attrChange.ByteValue != null ? $"(binary, {attrChange.ByteValue.Length} bytes)" : "(null)",
+ _ => "(unknown type)"
+ };
+ }
+}
diff --git a/src/JIM.Application/Servers/SyncEngine.cs b/src/JIM.Application/Servers/SyncEngine.cs
index 9b5d2c485..0b80fda07 100644
--- a/src/JIM.Application/Servers/SyncEngine.cs
+++ b/src/JIM.Application/Servers/SyncEngine.cs
@@ -32,7 +32,7 @@ public ProjectionDecision EvaluateProjection(
}
///
- public void FlowInboundAttributes(
+ public List FlowInboundAttributes(
ConnectedSystemObject cso,
SyncRule syncRule,
IReadOnlyList objectTypes,
@@ -41,10 +41,12 @@ public void FlowInboundAttributes(
bool onlyReferenceAttributes = false,
bool isFinalReferencePass = false)
{
+ var warnings = new List();
+
if (cso.MetaverseObject == null)
{
Log.Error("FlowInboundAttributes: CSO ({Cso}) has no MVO!", cso);
- return;
+ return warnings;
}
foreach (var syncRuleMapping in syncRule.AttributeFlowRules)
@@ -54,8 +56,10 @@ public void FlowInboundAttributes(
ProcessMapping(cso, syncRuleMapping, objectTypes, expressionEvaluator,
skipReferenceAttributes, onlyReferenceAttributes, isFinalReferencePass,
- cso.ConnectedSystemId);
+ cso.ConnectedSystemId, warnings);
}
+
+ return warnings;
}
///
@@ -82,7 +86,7 @@ public PendingExportConfirmationResult EvaluatePendingExportConfirmation(
continue;
}
- // Skip pending exports awaiting confirmation via PendingExportReconciliationService
+ // Skip pending exports awaiting confirmation via confirming import
if (pendingExport.Status == PendingExportStatus.Exported)
{
Log.Verbose("EvaluatePendingExportConfirmation: Skipping pending export {PeId} - awaiting confirmation via import (Status=Exported).", pendingExport.Id);
@@ -94,23 +98,8 @@ public PendingExportConfirmationResult EvaluatePendingExportConfirmation(
foreach (var attributeChange in pendingExport.AttributeValueChanges)
{
- var csoAttributeValue = cso.AttributeValues
- .FirstOrDefault(av => av.AttributeId == attributeChange.AttributeId);
-
- var changeMatches = attributeChange.ChangeType switch
- {
- PendingExportAttributeChangeType.Add or
- PendingExportAttributeChangeType.Update =>
- csoAttributeValue != null && AttributeValuesMatch(csoAttributeValue, attributeChange),
-
- PendingExportAttributeChangeType.Remove or
- PendingExportAttributeChangeType.RemoveAll =>
- csoAttributeValue == null || string.IsNullOrEmpty(csoAttributeValue.StringValue),
-
- _ => false
- };
-
- if (changeMatches)
+ // Use the comprehensive type-aware comparison
+ if (IsAttributeChangeConfirmed(cso, attributeChange))
successfulChanges.Add(attributeChange);
else
failedChanges.Add(attributeChange);
@@ -257,29 +246,6 @@ public InboundOutOfScopeAction DetermineOutOfScopeAction(
return importSyncRule.InboundOutOfScopeAction;
}
- ///
- public bool AttributeValuesMatch(
- ConnectedSystemObjectAttributeValue csoValue,
- PendingExportAttributeValueChange pendingChange)
- {
- if (pendingChange.StringValue != null && csoValue.StringValue != pendingChange.StringValue)
- return false;
-
- if (pendingChange.IntValue.HasValue && csoValue.IntValue != pendingChange.IntValue)
- return false;
-
- if (pendingChange.DateTimeValue.HasValue && csoValue.DateTimeValue != pendingChange.DateTimeValue)
- return false;
-
- if (pendingChange.ByteValue != null && !JIM.Utilities.Utilities.AreByteArraysTheSame(csoValue.ByteValue, pendingChange.ByteValue))
- return false;
-
- if (pendingChange.UnresolvedReferenceValue != null && csoValue.UnresolvedReferenceValue != pendingChange.UnresolvedReferenceValue)
- return false;
-
- return true;
- }
-
///
/// Evaluates the grace period for an MVO deletion decision.
///
diff --git a/src/JIM.Application/Services/DriftDetectionService.cs b/src/JIM.Application/Services/DriftDetectionService.cs
index d151b121b..a9e753541 100644
--- a/src/JIM.Application/Services/DriftDetectionService.cs
+++ b/src/JIM.Application/Services/DriftDetectionService.cs
@@ -455,7 +455,7 @@ private static bool IsContributorForExpressionAttributes(
///
private static Dictionary BuildAttributeDictionary(MetaverseObject mvo)
{
- var attributes = new Dictionary();
+ var attributes = new Dictionary(StringComparer.OrdinalIgnoreCase);
if (mvo.Type == null)
{
@@ -560,34 +560,13 @@ private static bool IsContributorForExpressionAttributes(
AttributeDataType.Guid => av.GuidValue,
AttributeDataType.Binary => av.ByteValue,
// For references, return the MVO ID that the referenced CSO is joined to.
- // This enables comparison with the expected MVO reference ID from GetExpectedValue.
- AttributeDataType.Reference => GetCsoReferenceMetaverseObjectId(av),
+ // ResolvedReferenceMetaverseObjectId (direct SQL) is the primary source;
+ // fall back to ReferenceValue navigation for in-memory test compatibility.
+ AttributeDataType.Reference => av.ResolvedReferenceMetaverseObjectId ?? av.ReferenceValue?.MetaverseObjectId,
_ => null
};
}
- ///
- /// Gets the MetaverseObjectId for a CSO reference attribute value.
- /// Uses the ReferenceValue navigation if loaded; logs a warning if the navigation
- /// is null despite ReferenceValueId being set (indicates a missing Include in the
- /// repository query).
- ///
- private static Guid? GetCsoReferenceMetaverseObjectId(ConnectedSystemObjectAttributeValue av)
- {
- if (av.ReferenceValue != null)
- return av.ReferenceValue.MetaverseObjectId;
-
- if (av.ReferenceValueId.HasValue)
- {
- Log.Warning("GetCsoReferenceMetaverseObjectId: ReferenceValueId {RefId} is set but " +
- "ReferenceValue navigation is null on attribute value {AvId}. " +
- "This indicates a missing .Include(av => av.ReferenceValue) in the repository query",
- av.ReferenceValueId.Value, av.Id);
- }
-
- return null;
- }
-
///
/// Compares two attribute values for equality.
/// Handles both single values and HashSets (for multi-valued attributes).
diff --git a/src/JIM.Application/Services/PendingExportReconciliationService.cs b/src/JIM.Application/Services/PendingExportReconciliationService.cs
index 37941dd62..cb7f81848 100644
--- a/src/JIM.Application/Services/PendingExportReconciliationService.cs
+++ b/src/JIM.Application/Services/PendingExportReconciliationService.cs
@@ -1,5 +1,5 @@
+using JIM.Application.Interfaces;
using JIM.Data.Repositories;
-using JIM.Models.Core;
using JIM.Models.Staging;
using JIM.Models.Transactional;
using Serilog;
@@ -7,27 +7,26 @@
namespace JIM.Application.Services;
///
-/// Service responsible for reconciling PendingExport attribute changes during import.
-/// Compares imported CSO values against pending export assertions to confirm or mark for retry.
+/// Thin orchestration wrapper for pending export reconciliation.
+/// Loads data from the database, delegates to for pure reconciliation logic,
+/// and persists the results. All decision-making logic lives in SyncEngine.
///
public class PendingExportReconciliationService
{
private readonly ISyncRepository _syncRepo;
+ private readonly ISyncEngine _syncEngine;
- ///
- /// Default maximum number of export attempts before marking an attribute change as Failed.
- ///
- public const int DefaultMaxRetries = 5;
-
- public PendingExportReconciliationService(ISyncRepository syncRepo)
+ public PendingExportReconciliationService(ISyncRepository syncRepo, ISyncEngine syncEngine)
{
_syncRepo = syncRepo;
+ _syncEngine = syncEngine;
}
///
/// Reconciles a Connected System Object's imported attribute values against any pending exports.
- /// This should be called after CSO attribute values are updated during import.
- /// Note: This method performs database operations per CSO. For bulk operations, use ReconcileWithPreloadedExportAsync instead.
+ /// Loads the pending export from the database, delegates reconciliation to SyncEngine,
+ /// and persists the outcome. For bulk operations, use SyncEngine.ReconcileCsoAgainstPendingExport directly
+ /// with pre-loaded data and batch persistence.
///
/// The CSO that was just imported/updated.
/// A result indicating what reconciliation actions were taken.
@@ -35,7 +34,6 @@ public async Task ReconcileAsync(ConnectedSys
{
var result = new PendingExportReconciliationResult();
- // Get any pending export for this CSO
var pendingExport = await _syncRepo.GetPendingExportByConnectedSystemObjectIdAsync(connectedSystemObject.Id);
if (pendingExport == null)
@@ -44,10 +42,10 @@ public async Task ReconcileAsync(ConnectedSys
return result;
}
- // Perform reconciliation (in-memory only)
- ReconcileCsoAgainstPendingExport(connectedSystemObject, pendingExport, result);
+ // Delegate to SyncEngine for pure in-memory reconciliation
+ _syncEngine.ReconcileCsoAgainstPendingExport(connectedSystemObject, pendingExport, result);
- // Persist changes immediately (non-batched mode)
+ // Persist changes
if (result.PendingExportDeleted)
{
await _syncRepo.DeletePendingExportAsync(pendingExport);
@@ -62,360 +60,4 @@ public async Task ReconcileAsync(ConnectedSys
return result;
}
-
- ///
- /// Reconciles a Connected System Object against a pre-loaded pending export.
- /// This method does NOT perform any database operations - caller is responsible for batching persistence.
- /// Use this method when processing multiple CSOs to batch database operations for better performance.
- ///
- /// The CSO that was just imported/updated.
- /// The pre-loaded pending export for this CSO (or null if none).
- /// The result object to populate with reconciliation outcomes.
- public void ReconcileCsoAgainstPendingExport(
- ConnectedSystemObject connectedSystemObject,
- PendingExport? pendingExport,
- PendingExportReconciliationResult result)
- {
- if (pendingExport == null)
- {
- Log.Debug("ReconcileCsoAgainstPendingExport: No pending export for CSO {CsoId}", connectedSystemObject.Id);
- return;
- }
-
- // Only process exports that have been executed and are awaiting confirmation
- // This includes:
- // - Exported: Just executed, awaiting first confirmation
- // - ExportNotConfirmed: Previously executed but some attributes weren't confirmed, awaiting re-confirmation
- if (pendingExport.Status != PendingExportStatus.Exported &&
- pendingExport.Status != PendingExportStatus.ExportNotConfirmed)
- {
- Log.Debug("ReconcileCsoAgainstPendingExport: PendingExport {ExportId} status is {Status}, not awaiting confirmation. Skipping.",
- pendingExport.Id, pendingExport.Status);
- return;
- }
-
- Log.Debug("ReconcileCsoAgainstPendingExport: Found pending export {ExportId} with {Count} attribute changes for CSO {CsoId}",
- pendingExport.Id, pendingExport.AttributeValueChanges.Count, connectedSystemObject.Id);
-
- // Process each attribute change that is awaiting confirmation
- // This includes:
- // - ExportedPendingConfirmation: Just exported, awaiting first confirmation
- // - ExportedNotConfirmed: Previously not confirmed, awaiting re-confirmation
- var changesAwaitingConfirmation = pendingExport.AttributeValueChanges
- .Where(ac => ac.Status == PendingExportAttributeChangeStatus.ExportedPendingConfirmation ||
- ac.Status == PendingExportAttributeChangeStatus.ExportedNotConfirmed)
- .ToList();
-
- foreach (var attrChange in changesAwaitingConfirmation)
- {
- var confirmed = IsAttributeChangeConfirmed(connectedSystemObject, attrChange);
-
- // Verbose logging for detailed troubleshooting - shows all comparison details
- Log.Verbose("ReconcileCsoAgainstPendingExport: Comparing attribute {AttrName} (ChangeType: {ChangeType}) for CSO {CsoId}. " +
- "Expected: '{ExpectedValue}', Found: '{ActualValue}', Confirmed: {Confirmed}",
- attrChange.Attribute?.Name ?? "unknown",
- attrChange.ChangeType,
- connectedSystemObject.Id,
- GetExpectedValueAsString(attrChange),
- GetImportedValueAsString(connectedSystemObject, attrChange),
- confirmed);
-
- if (confirmed)
- {
- result.ConfirmedChanges.Add(attrChange);
- Log.Debug("ReconcileCsoAgainstPendingExport: Attribute change {AttrChangeId} (Attr: {AttrName}) confirmed",
- attrChange.Id, attrChange.Attribute?.Name ?? "unknown");
- }
- else
- {
- // Not confirmed - mark for retry or fail
- attrChange.Status = ShouldMarkAsFailed(attrChange)
- ? PendingExportAttributeChangeStatus.Failed
- : PendingExportAttributeChangeStatus.ExportedNotConfirmed;
-
- // Capture what was imported for debugging
- attrChange.LastImportedValue = GetImportedValueAsString(connectedSystemObject, attrChange);
-
- var expectedValue = GetExpectedValueAsString(attrChange);
-
- if (attrChange.Status == PendingExportAttributeChangeStatus.Failed)
- {
- result.FailedChanges.Add(attrChange);
- Log.Warning("ReconcileCsoAgainstPendingExport: Attribute change {AttrChangeId} (Attr: {AttrName}) failed after {Attempts} attempts. " +
- "Expected: '{ExpectedValue}', Actual: '{ImportedValue}'",
- attrChange.Id, attrChange.Attribute?.Name ?? "unknown", attrChange.ExportAttemptCount,
- expectedValue, attrChange.LastImportedValue);
- }
- else
- {
- result.RetryChanges.Add(attrChange);
- Log.Information("ReconcileCsoAgainstPendingExport: Attribute change {AttrChangeId} (Attr: {AttrName}) not confirmed, will retry (attempt {Attempt}). " +
- "Expected: '{ExpectedValue}', Actual: '{ImportedValue}'",
- attrChange.Id, attrChange.Attribute?.Name ?? "unknown", attrChange.ExportAttemptCount,
- expectedValue, attrChange.LastImportedValue);
- }
- }
- }
-
- // Remove confirmed changes from the pending export
- foreach (var confirmed in result.ConfirmedChanges)
- {
- pendingExport.AttributeValueChanges.Remove(confirmed);
- }
-
- // If this was a Create and the Secondary External ID attribute was confirmed, transition to Update
- // This ensures remaining attribute changes are processed as updates, not creates
- TransitionCreateToUpdateIfSecondaryExternalIdConfirmed(pendingExport, result);
-
- // Determine if the pending export should be deleted or updated
- var hasRemainingChanges = pendingExport.AttributeValueChanges.Any(ac =>
- ac.Status == PendingExportAttributeChangeStatus.Pending ||
- ac.Status == PendingExportAttributeChangeStatus.ExportedPendingConfirmation ||
- ac.Status == PendingExportAttributeChangeStatus.ExportedNotConfirmed ||
- ac.Status == PendingExportAttributeChangeStatus.Failed);
-
- if (!hasRemainingChanges)
- {
- result.PendingExportDeleted = true;
- result.PendingExportToDelete = pendingExport;
- }
- else
- {
- // Update the pending export status based on attribute change statuses
- UpdatePendingExportStatus(pendingExport);
- result.PendingExportToUpdate = pendingExport;
- }
- }
-
- ///
- /// Determines if an attribute change has been confirmed by comparing the exported value
- /// against the imported CSO attribute value.
- ///
- private static bool IsAttributeChangeConfirmed(ConnectedSystemObject cso, PendingExportAttributeValueChange attrChange)
- {
- if (attrChange.Attribute == null)
- return false;
-
- // Find the corresponding attribute value on the CSO
- var csoAttrValues = cso.AttributeValues
- .Where(av => av.AttributeId == attrChange.AttributeId)
- .ToList();
-
- switch (attrChange.ChangeType)
- {
- case PendingExportAttributeChangeType.Add:
- case PendingExportAttributeChangeType.Update:
- // For Add/Update with a null/empty value (clearing a single-valued attribute),
- // confirmation means the CSO should have no values for this attribute.
- if (IsPendingChangeEmpty(attrChange))
- return csoAttrValues.Count == 0;
-
- // For Add/Update with a real value, the value should exist on the CSO
- return ValueExistsOnCso(csoAttrValues, attrChange);
-
- case PendingExportAttributeChangeType.Remove:
- // For Remove, the value should NOT exist on the CSO
- return !ValueExistsOnCso(csoAttrValues, attrChange);
-
- case PendingExportAttributeChangeType.RemoveAll:
- // For RemoveAll, there should be no values for this attribute
- return csoAttrValues.Count == 0;
-
- default:
- return false;
- }
- }
-
- ///
- /// Checks if the pending export attribute change value exists in the CSO's attribute values.
- ///
- private static bool ValueExistsOnCso(List csoValues, PendingExportAttributeValueChange attrChange)
- {
- if (csoValues.Count == 0)
- return false;
-
- // Check based on the data type of the attribute
- var attrType = attrChange.Attribute?.Type ?? AttributeDataType.NotSet;
-
- return attrType switch
- {
- AttributeDataType.Text =>
- !string.IsNullOrEmpty(attrChange.StringValue) &&
- csoValues.Any(v => string.Equals(v.StringValue, attrChange.StringValue, StringComparison.Ordinal)),
-
- AttributeDataType.Number =>
- attrChange.IntValue.HasValue &&
- csoValues.Any(v => v.IntValue == attrChange.IntValue),
-
- AttributeDataType.LongNumber =>
- attrChange.LongValue.HasValue &&
- csoValues.Any(v => v.LongValue == attrChange.LongValue),
-
- AttributeDataType.DateTime =>
- attrChange.DateTimeValue.HasValue &&
- csoValues.Any(v => v.DateTimeValue == attrChange.DateTimeValue),
-
- AttributeDataType.Binary =>
- attrChange.ByteValue != null &&
- csoValues.Any(v => v.ByteValue != null && v.ByteValue.SequenceEqual(attrChange.ByteValue)),
-
- AttributeDataType.Boolean =>
- attrChange.BoolValue.HasValue &&
- csoValues.Any(v => v.BoolValue == attrChange.BoolValue),
-
- AttributeDataType.Guid =>
- attrChange.GuidValue.HasValue &&
- csoValues.Any(v => v.GuidValue == attrChange.GuidValue),
-
- AttributeDataType.Reference =>
- // Reference attributes can be stored in two ways:
- // 1. UnresolvedReferenceValue - the raw DN string (before or after resolution)
- // 2. StringValue - set during export resolution (when UnresolvedReferenceValue is cleared)
- // We need to check both the pending export value AND the CSO values to handle both cases
- (!string.IsNullOrEmpty(attrChange.UnresolvedReferenceValue) &&
- csoValues.Any(v => string.Equals(v.UnresolvedReferenceValue, attrChange.UnresolvedReferenceValue, StringComparison.Ordinal))) ||
- (!string.IsNullOrEmpty(attrChange.StringValue) &&
- csoValues.Any(v => string.Equals(v.UnresolvedReferenceValue, attrChange.StringValue, StringComparison.Ordinal))),
-
- _ => false
- };
- }
-
- ///
- /// Checks if a pending export attribute value change represents an empty/null value
- /// (i.e., clearing a single-valued attribute).
- ///
- private static bool IsPendingChangeEmpty(PendingExportAttributeValueChange change)
- {
- return change.StringValue == null &&
- !change.IntValue.HasValue &&
- !change.LongValue.HasValue &&
- !change.DateTimeValue.HasValue &&
- !change.BoolValue.HasValue &&
- !change.GuidValue.HasValue &&
- change.ByteValue == null &&
- change.UnresolvedReferenceValue == null;
- }
-
- ///
- /// Gets the imported value as a string for debugging purposes.
- ///
- private static string? GetImportedValueAsString(ConnectedSystemObject cso, PendingExportAttributeValueChange attrChange)
- {
- var csoAttrValues = cso.AttributeValues
- .Where(av => av.AttributeId == attrChange.AttributeId)
- .ToList();
-
- if (csoAttrValues.Count == 0)
- return "(no values)";
-
- var attrType = attrChange.Attribute?.Type ?? AttributeDataType.NotSet;
-
- var values = attrType switch
- {
- AttributeDataType.Text => csoAttrValues.Select(v => v.StringValue).Where(v => v != null),
- AttributeDataType.Number => csoAttrValues.Select(v => v.IntValue?.ToString()).Where(v => v != null),
- AttributeDataType.LongNumber => csoAttrValues.Select(v => v.LongValue?.ToString()).Where(v => v != null),
- AttributeDataType.DateTime => csoAttrValues.Select(v => v.DateTimeValue?.ToString("O")).Where(v => v != null),
- AttributeDataType.Boolean => csoAttrValues.Select(v => v.BoolValue?.ToString()).Where(v => v != null),
- AttributeDataType.Guid => csoAttrValues.Select(v => v.GuidValue?.ToString()).Where(v => v != null),
- AttributeDataType.Reference => csoAttrValues.Select(v => v.UnresolvedReferenceValue).Where(v => v != null),
- _ => Enumerable.Empty()
- };
-
- var valueList = values.ToList();
- return valueList.Count > 0 ? string.Join(", ", valueList) : "(no matching type values)";
- }
-
- ///
- /// Gets the expected (exported) value as a string for debugging purposes.
- ///
- private static string? GetExpectedValueAsString(PendingExportAttributeValueChange attrChange)
- {
- var attrType = attrChange.Attribute?.Type ?? AttributeDataType.NotSet;
-
- return attrType switch
- {
- AttributeDataType.Text => attrChange.StringValue ?? "(null)",
- AttributeDataType.Number => attrChange.IntValue?.ToString() ?? "(null)",
- AttributeDataType.LongNumber => attrChange.LongValue?.ToString() ?? "(null)",
- AttributeDataType.DateTime => attrChange.DateTimeValue?.ToString("O") ?? "(null)",
- AttributeDataType.Boolean => attrChange.BoolValue?.ToString() ?? "(null)",
- AttributeDataType.Guid => attrChange.GuidValue?.ToString() ?? "(null)",
- AttributeDataType.Reference => attrChange.UnresolvedReferenceValue ?? "(null)",
- AttributeDataType.Binary => attrChange.ByteValue != null ? $"(binary, {attrChange.ByteValue.Length} bytes)" : "(null)",
- _ => "(unknown type)"
- };
- }
-
- ///
- /// Determines if an attribute change should be marked as Failed based on retry count.
- ///
- private static bool ShouldMarkAsFailed(PendingExportAttributeValueChange attrChange)
- {
- // ExportAttemptCount was already incremented during export, so compare directly
- return attrChange.ExportAttemptCount >= DefaultMaxRetries;
- }
-
- ///
- /// If the pending export was a Create and the Secondary External ID attribute has been confirmed,
- /// transition it to an Update. This is necessary because once an object is created, any remaining
- /// unconfirmed attribute changes should be applied as updates, not as part of a create operation.
- /// Connectors require the Secondary External ID (e.g., distinguishedName for LDAP) in the attribute
- /// changes for Create operations, but once confirmed, it is removed. Without this transition, retry
- /// attempts would fail because the connector cannot determine where to create the object.
- ///
- private static void TransitionCreateToUpdateIfSecondaryExternalIdConfirmed(PendingExport pendingExport, PendingExportReconciliationResult result)
- {
- // Only applies to Create pending exports
- if (pendingExport.ChangeType != PendingExportChangeType.Create)
- return;
-
- // Check if the Secondary External ID attribute was among the confirmed changes
- var secondaryExternalIdWasConfirmed = result.ConfirmedChanges.Any(ac =>
- ac.Attribute?.IsSecondaryExternalId == true);
-
- if (!secondaryExternalIdWasConfirmed)
- return;
-
- // The object was successfully created (Secondary External ID confirmed), but there are remaining
- // attribute changes that weren't confirmed. Transition to Update so these can be applied as modifications.
- if (pendingExport.AttributeValueChanges.Count > 0)
- {
- var confirmedAttrName = result.ConfirmedChanges
- .FirstOrDefault(ac => ac.Attribute?.IsSecondaryExternalId == true)?.Attribute?.Name ?? "unknown";
-
- pendingExport.ChangeType = PendingExportChangeType.Update;
- Log.Information("ReconcileAsync: Transitioned pending export {ExportId} from Create to Update. " +
- "Secondary External ID attribute '{AttributeName}' was confirmed but {RemainingCount} attribute changes remain.",
- pendingExport.Id, confirmedAttrName, pendingExport.AttributeValueChanges.Count);
- }
- }
-
- ///
- /// Updates the PendingExport status based on its attribute change statuses.
- ///
- private static void UpdatePendingExportStatus(PendingExport pendingExport)
- {
- var allFailed = pendingExport.AttributeValueChanges.All(ac => ac.Status == PendingExportAttributeChangeStatus.Failed);
- var anyFailed = pendingExport.AttributeValueChanges.Any(ac => ac.Status == PendingExportAttributeChangeStatus.Failed);
- var anyPendingOrRetry = pendingExport.AttributeValueChanges.Any(ac =>
- ac.Status == PendingExportAttributeChangeStatus.Pending ||
- ac.Status == PendingExportAttributeChangeStatus.ExportedNotConfirmed);
-
- if (allFailed)
- {
- pendingExport.Status = PendingExportStatus.Failed;
- }
- else if (anyPendingOrRetry)
- {
- // There are changes that need to be exported/re-exported
- pendingExport.Status = PendingExportStatus.ExportNotConfirmed;
- }
- else if (anyFailed)
- {
- // Some failed, but some are still pending confirmation
- pendingExport.Status = PendingExportStatus.Exported;
- }
- }
}
diff --git a/src/JIM.Application/Utilities/ExportChangeHistoryBuilder.cs b/src/JIM.Application/Utilities/ExportChangeHistoryBuilder.cs
index 65a7cdee8..67aa27bb2 100644
--- a/src/JIM.Application/Utilities/ExportChangeHistoryBuilder.cs
+++ b/src/JIM.Application/Utilities/ExportChangeHistoryBuilder.cs
@@ -106,6 +106,8 @@ internal static void MapAttributeValueChanges(
attributeChange = new ConnectedSystemObjectChangeAttribute
{
Attribute = peChange.Attribute,
+ AttributeName = peChange.Attribute.Name,
+ AttributeType = peChange.Attribute.Type,
ConnectedSystemChange = change
};
change.AttributeChanges.Add(attributeChange);
diff --git a/src/JIM.Connectors/LDAP/LdapConnector.cs b/src/JIM.Connectors/LDAP/LdapConnector.cs
index bb1e7cdff..6c0a3088e 100644
--- a/src/JIM.Connectors/LDAP/LdapConnector.cs
+++ b/src/JIM.Connectors/LDAP/LdapConnector.cs
@@ -11,6 +11,8 @@ namespace JIM.Connectors.LDAP;
public class LdapConnector : IConnector, IConnectorCapabilities, IConnectorSettings, IConnectorSchema, IConnectorPartitions, IConnectorImportUsingCalls, IConnectorExportUsingCalls, IConnectorCertificateAware, IConnectorCredentialAware, IConnectorContainerCreation, IDisposable
{
private LdapConnection? _connection;
+ private Func? _connectionFactory;
+ private LdapDirectoryType _directoryType = LdapDirectoryType.Generic;
private bool _disposed;
private ICertificateProvider? _certificateProvider;
private ICredentialProtection? _credentialProtection;
@@ -55,14 +57,21 @@ public class LdapConnector : IConnector, IConnectorCapabilities, IConnectorSetti
private readonly string _settingMaxRetries = "Maximum Retries";
private readonly string _settingRetryDelay = "Retry Delay (ms)";
+ // Schema settings
+ private readonly string _settingIncludeAuxiliaryClasses = "Include Auxiliary Classes";
+
// Hierarchy settings
private readonly string _settingSkipHiddenPartitions = "Skip Hidden Partitions";
+ // Import settings
+ private readonly string _settingImportConcurrency = "Import Concurrency";
+
// Export settings
private readonly string _settingDeleteBehaviour = "Delete Behaviour";
private readonly string _settingDisableAttribute = "Disable Attribute";
private readonly string _settingExportConcurrency = "Export Concurrency";
private readonly string _settingModifyBatchSize = "Modify Batch Size";
+ private readonly string _settingGroupPlaceholderMemberDn = LdapConnectorConstants.SETTING_GROUP_PLACEHOLDER_MEMBER_DN;
public List GetSettings()
{
@@ -85,11 +94,15 @@ public List GetSettings()
new() { Name = "Import Settings", Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.Heading },
new() { Name = _settingSearchTimeout, Required = false, Description = "Maximum time in seconds to wait for LDAP search results. Default is 300 (5 minutes).", DefaultIntValue = 300, Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.Integer },
+ new() { Name = _settingImportConcurrency, Required = false, Description = "Maximum number of parallel LDAP connections used during full imports from OpenLDAP and Generic directories. Each connection handles one container and object type combination independently, avoiding RFC 2696 paging cookie limitations. Not used for Active Directory. Default is 4. Recommended range: 2-8.", DefaultIntValue = LdapConnectorConstants.DEFAULT_IMPORT_CONCURRENCY, Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.Integer },
new() { Name = "Retry Settings", Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.Heading },
new() { Name = _settingMaxRetries, Required = false, Description = "Maximum number of retry attempts for transient failures. Default is 3.", DefaultIntValue = LdapConnectorConstants.DEFAULT_MAX_RETRIES, Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.Integer },
new() { Name = _settingRetryDelay, Required = false, Description = "Initial delay between retries in milliseconds. Uses exponential backoff. Default is 1000ms.", DefaultIntValue = LdapConnectorConstants.DEFAULT_RETRY_DELAY_MS, Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.Integer },
+ new() { Name = "Schema Discovery", Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.Heading },
+ new() { Name = _settingIncludeAuxiliaryClasses, Description = "When enabled, auxiliary object classes are included in schema discovery alongside structural classes. Enable this if you need to import or export objects whose primary class is declared as auxiliary in the directory schema.", DefaultCheckboxValue = false, Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.CheckBox },
+
new() { Name = "Container Provisioning", Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.Heading },
new() { Name = _settingCreateContainersAsNeeded, Description = "i.e. create OUs as needed when provisioning new objects.", DefaultCheckboxValue = false, Category = ConnectedSystemSettingCategory.General, Type = ConnectedSystemSettingType.CheckBox },
@@ -101,7 +114,10 @@ public List GetSettings()
new() { Name = _settingDeleteBehaviour, Required = false, Description = "How to handle object deletions.", Type = ConnectedSystemSettingType.DropDown, DropDownValues = new() { LdapConnectorConstants.DELETE_BEHAVIOUR_DELETE, LdapConnectorConstants.DELETE_BEHAVIOUR_DISABLE }, Category = ConnectedSystemSettingCategory.Export },
new() { Name = _settingDisableAttribute, Required = false, Description = "Attribute to set when disabling objects (e.g., userAccountControl for AD). Only used when Delete Behaviour is 'Disable'.", DefaultStringValue = "userAccountControl", Category = ConnectedSystemSettingCategory.Export, Type = ConnectedSystemSettingType.String },
new() { Name = _settingExportConcurrency, Required = false, Description = "Maximum number of concurrent LDAP operations during export. Higher values improve throughput but increase load on the target directory. Default is 4. Recommended range: 2-8. Values above 8 show diminishing returns and may overwhelm the directory server.", DefaultIntValue = LdapConnectorConstants.DEFAULT_EXPORT_CONCURRENCY, Category = ConnectedSystemSettingCategory.Export, Type = ConnectedSystemSettingType.Integer },
- new() { Name = _settingModifyBatchSize, Required = false, Description = "Maximum number of values per multi-valued attribute modification in a single LDAP request. When adding or removing many values from a multi-valued attribute (e.g., group members), changes are split into batches of this size. Lower values improve compatibility with constrained LDAP servers; higher values improve throughput. Default is 100. Recommended range: 50-500.", DefaultIntValue = LdapConnectorConstants.DEFAULT_MODIFY_BATCH_SIZE, Category = ConnectedSystemSettingCategory.Export, Type = ConnectedSystemSettingType.Integer }
+ new() { Name = _settingModifyBatchSize, Required = false, Description = "Maximum number of values per multi-valued attribute modification in a single LDAP request. When adding or removing many values from a multi-valued attribute (e.g., group members), changes are split into batches of this size. Lower values improve compatibility with constrained LDAP servers; higher values improve throughput. Default is 100. Recommended range: 50-500.", DefaultIntValue = LdapConnectorConstants.DEFAULT_MODIFY_BATCH_SIZE, Category = ConnectedSystemSettingCategory.Export, Type = ConnectedSystemSettingType.Integer },
+
+ new() { Name = "Group Membership", Category = ConnectedSystemSettingCategory.Export, Type = ConnectedSystemSettingType.Heading },
+ new() { Name = _settingGroupPlaceholderMemberDn, Required = false, Description = "Placeholder member DN used for group object classes that require at least one member (e.g. groupOfNames). When a group has no real members, this value is added to satisfy the schema constraint. It is automatically filtered out during import. Only applies to non-AD directories. Default: cn=placeholder. If your directory has referential integrity enabled, set this to an existing entry's DN.", DefaultStringValue = LdapConnectorConstants.DEFAULT_GROUP_PLACEHOLDER_MEMBER_DN, Category = ConnectedSystemSettingCategory.Export, Type = ConnectedSystemSettingType.String }
};
}
@@ -143,8 +159,10 @@ public async Task GetSchemaAsync(List q.Setting.Name == _settingIncludeAuxiliaryClasses)?.CheckboxValue ?? false;
+
var rootDse = LdapConnectorUtilities.GetBasicRootDseInformation(_connection, logger);
- var ldapConnectorSchema = new LdapConnectorSchema(_connection, logger, rootDse);
+ var ldapConnectorSchema = new LdapConnectorSchema(_connection, logger, rootDse, includeAuxiliaryClasses);
var schema = await ldapConnectorSchema.GetSchemaAsync();
CloseImportConnection();
return schema;
@@ -160,7 +178,10 @@ public async Task> GetPartitionsAsync(List q.Setting.Name == _settingSkipHiddenPartitions)?.CheckboxValue ?? true;
- var ldapConnectorPartitions = new LdapConnectorPartitions(_connection, logger);
+ // Detect directory type so partition discovery can use the appropriate mechanism
+ var rootDse = LdapConnectorUtilities.GetBasicRootDseInformation(_connection, logger);
+
+ var ldapConnectorPartitions = new LdapConnectorPartitions(_connection, logger, rootDse.DirectoryType);
var partitions = await ldapConnectorPartitions.GetPartitionsAsync(skipHiddenPartitions);
CloseImportConnection();
return partitions;
@@ -221,42 +242,66 @@ public void OpenImportConnection(List settingValues
else if (authTypeSettingValueString == LdapConnectorConstants.SETTING_AUTH_TYPE_NTLM)
authTypeEnumValue = AuthType.Ntlm;
+ // Build a reusable connection factory so LdapConnectorImport can create additional
+ // connections for parallel imports (one connection per container+objectType combo).
+ // Captured values are immutable for the duration of the import session.
+ _connectionFactory = () => CreateConnection(identifier, credential, authTypeEnumValue,
+ TimeSpan.FromSeconds(timeoutSeconds.IntValue.Value), useSsl, skipCertValidation, logger);
+
// Execute connection with retry logic
ExecuteWithRetry(() =>
{
- _connection = new LdapConnection(identifier, credential, authTypeEnumValue);
- _connection.SessionOptions.ProtocolVersion = 3;
- _connection.Timeout = TimeSpan.FromSeconds(timeoutSeconds.IntValue.Value);
+ _connection = _connectionFactory();
+ }, maxRetries, retryDelayMs, logger);
+ }
- // Configure LDAPS if enabled
- if (useSsl)
- {
- _connection.SessionOptions.SecureSocketLayer = true;
+ ///
+ /// Creates a new bound LdapConnection with the specified parameters.
+ /// Used both for the primary import connection and for parallel import connections
+ /// in OpenLDAP/Generic directories where each paged search needs its own connection.
+ ///
+ private LdapConnection CreateConnection(
+ LdapDirectoryIdentifier identifier,
+ NetworkCredential credential,
+ AuthType authType,
+ TimeSpan timeout,
+ bool useSsl,
+ bool skipCertValidation,
+ ILogger logger)
+ {
+ var connection = new LdapConnection(identifier, credential, authType);
+ connection.SessionOptions.ProtocolVersion = 3;
+ connection.Timeout = timeout;
+
+ // Configure LDAPS if enabled
+ if (useSsl)
+ {
+ connection.SessionOptions.SecureSocketLayer = true;
- if (skipCertValidation)
+ if (skipCertValidation)
+ {
+ logger.Warning("Certificate validation is disabled. This is not recommended for production environments.");
+ // On Linux, setting VerifyServerCertificate can fail. Use LDAPTLS_REQCERT=never
+ // environment variable instead. On Windows, set the callback directly.
+ if (OperatingSystem.IsWindows())
{
- logger.Warning("Certificate validation is disabled. This is not recommended for production environments.");
- // On Linux, setting VerifyServerCertificate can fail. Use LDAPTLS_REQCERT=never
- // environment variable instead. On Windows, set the callback directly.
- if (OperatingSystem.IsWindows())
- {
- _connection.SessionOptions.VerifyServerCertificate = (connection, certificate) => true;
- }
- else
- {
- logger.Debug("Skipping VerifyServerCertificate callback on Linux - using LDAPTLS_REQCERT environment variable");
- }
+ connection.SessionOptions.VerifyServerCertificate = (_, _) => true;
}
- else if (_trustedCertificates != null && _trustedCertificates.Count > 0)
+ else
{
- // Full validation with JIM certificates supplementing system store
- _connection.SessionOptions.VerifyServerCertificate = ValidateServerCertificate;
+ logger.Debug("Skipping VerifyServerCertificate callback on Linux - using LDAPTLS_REQCERT environment variable");
}
- // else: use system default validation only
}
+ else if (_trustedCertificates != null && _trustedCertificates.Count > 0)
+ {
+ // Full validation with JIM certificates supplementing system store
+ connection.SessionOptions.VerifyServerCertificate = ValidateServerCertificate;
+ }
+ // else: use system default validation only
+ }
- _connection.Bind();
- }, maxRetries, retryDelayMs, logger);
+ connection.Bind();
+ return connection;
}
///
@@ -315,7 +360,11 @@ public Task ImportAsync(ConnectedSystem connectedSy
// needs to filter by attributes
// needs to be able to stop processing at convenient points if cancellation has been requested
- var import = new LdapConnectorImport(connectedSystem, runProfile, _connection, paginationTokens, persistedConnectorData, logger, cancellationToken);
+ var importConcurrency = connectedSystem.SettingValues
+ .SingleOrDefault(s => s.Setting.Name == _settingImportConcurrency)?.IntValue
+ ?? LdapConnectorConstants.DEFAULT_IMPORT_CONCURRENCY;
+
+ var import = new LdapConnectorImport(connectedSystem, runProfile, _connection, _connectionFactory, importConcurrency, paginationTokens, persistedConnectorData, logger, cancellationToken);
switch (runProfile.RunType)
{
@@ -348,6 +397,13 @@ public void OpenExportConnection(IList settings)
// Reuse the same connection logic as import
OpenImportConnection(settings.ToList(), Log.Logger);
+
+ // Detect directory type for export operations (external ID fetching, etc.)
+ if (_connection != null)
+ {
+ var rootDse = LdapConnectorUtilities.GetBasicRootDseInformation(_connection, Log.Logger);
+ _directoryType = rootDse.DirectoryType;
+ }
}
public Task> ExportAsync(IList pendingExports, CancellationToken cancellationToken)
@@ -366,8 +422,12 @@ public Task> ExportAsync(IList
.FirstOrDefault(s => s.Setting.Name == _settingModifyBatchSize)?.IntValue
?? LdapConnectorConstants.DEFAULT_MODIFY_BATCH_SIZE;
+ var placeholderMemberDn = _exportSettings
+ .FirstOrDefault(s => s.Setting.Name == _settingGroupPlaceholderMemberDn)?.StringValue
+ ?? LdapConnectorConstants.DEFAULT_GROUP_PLACEHOLDER_MEMBER_DN;
+
var executor = new LdapOperationExecutor(_connection);
- _currentExport = new LdapConnectorExport(executor, _exportSettings, Log.Logger, concurrency, modifyBatchSize);
+ _currentExport = new LdapConnectorExport(executor, _exportSettings, Log.Logger, concurrency, modifyBatchSize, _directoryType, placeholderMemberDn);
return _currentExport.ExecuteAsync(pendingExports, cancellationToken);
}
diff --git a/src/JIM.Connectors/LDAP/LdapConnectorConstants.cs b/src/JIM.Connectors/LDAP/LdapConnectorConstants.cs
index 346f3d0ea..f2247f57c 100644
--- a/src/JIM.Connectors/LDAP/LdapConnectorConstants.cs
+++ b/src/JIM.Connectors/LDAP/LdapConnectorConstants.cs
@@ -54,6 +54,15 @@ internal static class LdapConnectorConstants
internal const int DEFAULT_MAX_RETRIES = 3;
internal const int DEFAULT_RETRY_DELAY_MS = 1000;
+ // Import concurrency settings
+ // Controls the number of parallel LDAP connections used during OpenLDAP/Generic directory imports.
+ // Each connection handles one container+objectType combo independently, bypassing the RFC 2696
+ // connection-scoped paging cookie limitation. Not used for AD directories (which multiplex on
+ // a single connection). Typical deployments have 2-6 combos, so even low concurrency values
+ // eliminate the serialisation bottleneck. Higher values add connection overhead with diminishing returns.
+ internal const int DEFAULT_IMPORT_CONCURRENCY = 4;
+ internal const int MAX_IMPORT_CONCURRENCY = 8;
+
// Export concurrency settings
internal const int DEFAULT_EXPORT_CONCURRENCY = 4;
internal const int MAX_EXPORT_CONCURRENCY = 8;
@@ -111,4 +120,22 @@ internal static class LdapConnectorConstants
"samDomain",
"samServer"
};
+
+ // Group placeholder member settings
+ // The groupOfNames object class (RFC 4519) requires at least one member value (MUST constraint).
+ // When a group has no real members, a placeholder DN is used to satisfy this constraint.
+ // This applies to OpenLDAP and Generic directories — AD/Samba AD use the 'group' class which has no such constraint.
+ internal const string DEFAULT_GROUP_PLACEHOLDER_MEMBER_DN = "cn=placeholder";
+ internal const string SETTING_GROUP_PLACEHOLDER_MEMBER_DN = "Group Placeholder Member DN";
+
+ ///
+ /// LDAP object classes that require at least one member value (MUST constraint on the member attribute).
+ /// When exporting to directories using these classes, a placeholder member is injected
+ /// to satisfy the schema constraint when the group would otherwise be empty.
+ ///
+ internal static readonly HashSet MUST_MEMBER_OBJECT_CLASSES = new(StringComparer.OrdinalIgnoreCase)
+ {
+ "groupOfNames",
+ "groupOfUniqueNames"
+ };
}
\ No newline at end of file
diff --git a/src/JIM.Connectors/LDAP/LdapConnectorEnums.cs b/src/JIM.Connectors/LDAP/LdapConnectorEnums.cs
new file mode 100644
index 000000000..9748f5270
--- /dev/null
+++ b/src/JIM.Connectors/LDAP/LdapConnectorEnums.cs
@@ -0,0 +1,35 @@
+namespace JIM.Connectors.LDAP;
+
+///
+/// The type of LDAP directory server detected via rootDSE capabilities.
+/// Determines directory-specific behaviour: schema discovery, external ID attribute,
+/// delta import strategy, and attribute semantics.
+///
+internal enum LdapDirectoryType
+{
+ ///
+ /// Microsoft Active Directory (AD-DS) or Active Directory Lightweight Directory Services (AD-LDS).
+ /// Detected via supportedCapabilities OIDs on rootDSE.
+ ///
+ ActiveDirectory,
+
+ ///
+ /// Samba Active Directory Domain Controller.
+ /// Advertises AD capability OIDs but has behavioural differences: paged search returns duplicates,
+ /// different error codes for missing objects, and different backend tooling (ldbadd vs ldapmodify).
+ /// Detected via AD capability OIDs combined with vendorName containing "Samba".
+ ///
+ SambaAD,
+
+ ///
+ /// OpenLDAP directory server.
+ /// Detected via vendorName or vendorVersion on rootDSE.
+ ///
+ OpenLDAP,
+
+ ///
+ /// Unrecognised directory server. Uses RFC-standard LDAP behaviour.
+ /// Falls back to OpenLDAP-compatible defaults (entryUUID, changelog delta, RFC 4512 schema).
+ ///
+ Generic
+}
diff --git a/src/JIM.Connectors/LDAP/LdapConnectorExport.cs b/src/JIM.Connectors/LDAP/LdapConnectorExport.cs
index d3588c9da..816e25f98 100644
--- a/src/JIM.Connectors/LDAP/LdapConnectorExport.cs
+++ b/src/JIM.Connectors/LDAP/LdapConnectorExport.cs
@@ -17,6 +17,8 @@ internal class LdapConnectorExport
private readonly ILogger _logger;
private readonly int _exportConcurrency;
private readonly int _modifyBatchSize;
+ private readonly LdapDirectoryType _directoryType;
+ private readonly string _placeholderMemberDn;
// Setting names
private const string SettingDeleteBehaviour = "Delete Behaviour";
@@ -81,13 +83,17 @@ internal LdapConnectorExport(
IList settings,
ILogger logger,
int exportConcurrency = LdapConnectorConstants.DEFAULT_EXPORT_CONCURRENCY,
- int modifyBatchSize = LdapConnectorConstants.DEFAULT_MODIFY_BATCH_SIZE)
+ int modifyBatchSize = LdapConnectorConstants.DEFAULT_MODIFY_BATCH_SIZE,
+ LdapDirectoryType directoryType = LdapDirectoryType.ActiveDirectory,
+ string? placeholderMemberDn = null)
{
_executor = executor;
_settings = settings;
_logger = logger;
+ _directoryType = directoryType;
_exportConcurrency = Math.Clamp(exportConcurrency, 1, LdapConnectorConstants.MAX_EXPORT_CONCURRENCY);
_modifyBatchSize = Math.Clamp(modifyBatchSize, LdapConnectorConstants.MIN_MODIFY_BATCH_SIZE, LdapConnectorConstants.MAX_MODIFY_BATCH_SIZE);
+ _placeholderMemberDn = placeholderMemberDn ?? LdapConnectorConstants.DEFAULT_GROUP_PLACEHOLDER_MEMBER_DN;
if (exportConcurrency > LdapConnectorConstants.MAX_EXPORT_CONCURRENCY)
{
@@ -196,12 +202,22 @@ private ConnectedSystemExportResult ProcessCreate(PendingExport pendingExport)
EnsureParentContainersExist(dn);
}
+ // Track whether we injected a placeholder so we can provide a specific error if it's rejected
+ var hasPlaceholder = RequiresPlaceholderMember(pendingExport);
+
var (addRequest, overflowModifyRequests) = BuildAddRequestWithOverflow(pendingExport, dn);
- var response = (AddResponse)_executor.SendRequest(addRequest);
- if (response.ResultCode != ResultCode.Success)
+ try
{
- ThrowAddFailure(addRequest, dn, response);
+ var response = (AddResponse)_executor.SendRequest(addRequest);
+ if (response.ResultCode != ResultCode.Success)
+ {
+ ThrowAddFailure(addRequest, dn, response);
+ }
+ }
+ catch (DirectoryOperationException ex) when (hasPlaceholder && IsPlaceholderConstraintViolation(ex))
+ {
+ return HandlePlaceholderRejection(dn, ex);
}
// Send any overflow ModifyRequests for multi-valued attributes that exceeded the batch size
@@ -215,37 +231,40 @@ private ConnectedSystemExportResult ProcessCreate(PendingExport pendingExport)
_logger.Debug("LdapConnectorExport.ProcessCreate: Successfully created object at '{Dn}'", dn);
- // After successful create, fetch the system-assigned objectGUID
- var objectGuid = FetchObjectGuid(dn);
- if (objectGuid != null)
+ // After successful create, fetch the system-assigned external ID (objectGUID for AD, entryUUID for OpenLDAP)
+ var rootDse = new LdapConnectorRootDse { DirectoryType = _directoryType };
+ var externalId = FetchExternalId(dn, rootDse);
+ if (externalId != null)
{
- _logger.Debug("LdapConnectorExport.ProcessCreate: Retrieved objectGUID {ObjectGuid} for '{Dn}'", objectGuid, dn);
- return ConnectedSystemExportResult.Succeeded(objectGuid, dn);
+ _logger.Debug("LdapConnectorExport.ProcessCreate: Retrieved external ID {ExternalId} for '{Dn}'", externalId, dn);
+ return ConnectedSystemExportResult.Succeeded(externalId, dn);
}
- // objectGUID not available, return success without external ID
+ // External ID not available, return success without it
return ConnectedSystemExportResult.Succeeded(null, dn);
}
///
- /// Fetches the objectGUID for a newly created object.
+ /// Fetches the external ID attribute for a newly created object.
+ /// For AD this is objectGUID (binary GUID); for OpenLDAP this is entryUUID (string UUID).
///
- private string? FetchObjectGuid(string dn)
+ private string? FetchExternalId(string dn, LdapConnectorRootDse rootDse)
{
try
{
+ var externalIdAttr = rootDse.ExternalIdAttributeName;
var searchRequest = new SearchRequest(
dn,
"(objectClass=*)",
SearchScope.Base,
- "objectGUID");
+ externalIdAttr);
var searchResponse = (SearchResponse)_executor.SendRequest(searchRequest);
- return ParseObjectGuidFromResponse(searchResponse, dn);
+ return ParseExternalIdFromResponse(searchResponse, dn, rootDse);
}
catch (Exception ex)
{
- _logger.Warning(ex, "LdapConnectorExport.FetchObjectGuid: Error fetching objectGUID for '{Dn}'", dn);
+ _logger.Warning(ex, "LdapConnectorExport.FetchExternalId: Error fetching external ID for '{Dn}'", dn);
return null;
}
}
@@ -278,6 +297,9 @@ private ConnectedSystemExportResult ProcessUpdate(PendingExport pendingExport)
wasRenamed = true;
}
+ // Track whether placeholder modifications were injected for error handling
+ var hasPlaceholderModifications = RequiresPlaceholderMember(pendingExport);
+
var modifyRequests = BuildModifyRequests(pendingExport, workingDn);
if (modifyRequests.Count == 0)
@@ -297,8 +319,15 @@ private ConnectedSystemExportResult ProcessUpdate(PendingExport pendingExport)
i + 1, modifyRequests.Count, request.Modifications.Count, workingDn);
}
- var response = (ModifyResponse)_executor.SendRequest(request);
- lastResult = HandleModifyResponse(response, request, workingDn, wasRenamed);
+ try
+ {
+ var response = (ModifyResponse)_executor.SendRequest(request);
+ lastResult = HandleModifyResponse(response, request, workingDn, wasRenamed);
+ }
+ catch (DirectoryOperationException ex) when (hasPlaceholderModifications && IsPlaceholderConstraintViolation(ex))
+ {
+ return HandlePlaceholderRejection(workingDn, ex);
+ }
}
return lastResult;
@@ -652,32 +681,34 @@ private async Task ProcessCreateAsync(PendingExport
_logger.Debug("LdapConnectorExport.ProcessCreateAsync: Successfully created object at '{Dn}'", dn);
- var objectGuid = await FetchObjectGuidAsync(dn);
- if (objectGuid != null)
+ var rootDse = new LdapConnectorRootDse { DirectoryType = _directoryType };
+ var externalId = await FetchExternalIdAsync(dn, rootDse);
+ if (externalId != null)
{
- _logger.Debug("LdapConnectorExport.ProcessCreateAsync: Retrieved objectGUID {ObjectGuid} for '{Dn}'", objectGuid, dn);
- return ConnectedSystemExportResult.Succeeded(objectGuid, dn);
+ _logger.Debug("LdapConnectorExport.ProcessCreateAsync: Retrieved external ID {ExternalId} for '{Dn}'", externalId, dn);
+ return ConnectedSystemExportResult.Succeeded(externalId, dn);
}
return ConnectedSystemExportResult.Succeeded(null, dn);
}
- private async Task FetchObjectGuidAsync(string dn)
+ private async Task FetchExternalIdAsync(string dn, LdapConnectorRootDse rootDse)
{
try
{
+ var externalIdAttr = rootDse.ExternalIdAttributeName;
var searchRequest = new SearchRequest(
dn,
"(objectClass=*)",
SearchScope.Base,
- "objectGUID");
+ externalIdAttr);
var searchResponse = (SearchResponse)await _executor.SendRequestAsync(searchRequest);
- return ParseObjectGuidFromResponse(searchResponse, dn);
+ return ParseExternalIdFromResponse(searchResponse, dn, rootDse);
}
catch (Exception ex)
{
- _logger.Warning(ex, "LdapConnectorExport.FetchObjectGuidAsync: Error fetching objectGUID for '{Dn}'", dn);
+ _logger.Warning(ex, "LdapConnectorExport.FetchExternalIdAsync: Error fetching external ID for '{Dn}'", dn);
return null;
}
}
@@ -994,6 +1025,21 @@ private async Task CreateContainerAsync(string containerDn)
}
}
+ // Placeholder member injection for groupOfNames/groupOfUniqueNames on non-AD directories.
+ // If this is a MUST-member object class and no member values are present, inject the placeholder.
+ if (RequiresPlaceholderMember(pendingExport) && !string.IsNullOrEmpty(objectClass))
+ {
+ var memberAttrName = GetMemberAttributeName(objectClass);
+ if (!attributeGroups.ContainsKey(memberAttrName) || attributeGroups[memberAttrName].Count == 0)
+ {
+ _logger.Information(
+ "LdapConnectorExport.BuildAddRequestWithOverflow: Object class '{ObjectClass}' requires at least one member. " +
+ "Injecting placeholder member '{Placeholder}' for '{Dn}'.",
+ objectClass, _placeholderMemberDn, dn);
+ attributeGroups[memberAttrName] = new List
- internal static List ConsolidateModifications(PendingExport pendingExport)
+ internal static List ConsolidateModifications(PendingExport pendingExport, string? workingDn = null)
{
// Group changes by (attribute name, operation type), preserving the original attribute changes
// for protected attribute handling
@@ -1135,9 +1204,10 @@ internal static List ConsolidateModifications(PendingE
var attrName = attrChange.Attribute.Name;
- // Skip RDN (Relative Distinguished Name) attributes - they cannot be modified via LDAP ModifyRequest.
+ // Skip RDN (Relative Distinguished Name) attributes — they cannot be modified via LDAP ModifyRequest.
// These require a ModifyDNRequest (rename operation) instead, which is handled separately.
- if (IsRdnAttribute(attrName))
+ // The actual RDN attribute is determined from the object's DN (e.g., CN for AD, uid for OpenLDAP).
+ if (IsRdnAttribute(attrName, workingDn))
continue;
var operation = attrChange.ChangeType switch
@@ -1324,14 +1394,34 @@ private List BuildConsolidatedModification(Conso
///
/// Checks if an attribute name is an RDN (Relative Distinguished Name) attribute.
- /// RDN attributes cannot be modified via LDAP ModifyRequest - they require ModifyDNRequest.
+ /// RDN attributes cannot be modified via LDAP ModifyRequest — they require ModifyDNRequest.
+ /// The actual RDN attribute varies by directory and object type (e.g., CN for AD users, uid for OpenLDAP).
+ /// We determine the RDN by parsing it from the object's current DN.
///
- private static bool IsRdnAttribute(string attrName)
+ /// The attribute name to check.
+ /// The current DN of the object, used to determine the actual RDN attribute.
+ private static bool IsRdnAttribute(string attrName, string? dn = null)
{
- return attrName.Equals("distinguishedName", StringComparison.OrdinalIgnoreCase) ||
- attrName.Equals("cn", StringComparison.OrdinalIgnoreCase) ||
+ // distinguishedName is always skipped — it's the full DN, not a modifiable attribute
+ if (attrName.Equals("distinguishedName", StringComparison.OrdinalIgnoreCase))
+ return true;
+
+ // If we have the DN, determine the actual RDN attribute by parsing the first component
+ if (!string.IsNullOrEmpty(dn))
+ {
+ var equalsIndex = dn.IndexOf('=');
+ if (equalsIndex > 0)
+ {
+ var rdnAttr = dn[..equalsIndex].Trim();
+ return attrName.Equals(rdnAttr, StringComparison.OrdinalIgnoreCase);
+ }
+ }
+
+ // Fallback: common RDN attributes (when DN is not available)
+ return attrName.Equals("cn", StringComparison.OrdinalIgnoreCase) ||
attrName.Equals("ou", StringComparison.OrdinalIgnoreCase) ||
- attrName.Equals("dc", StringComparison.OrdinalIgnoreCase);
+ attrName.Equals("dc", StringComparison.OrdinalIgnoreCase) ||
+ attrName.Equals("uid", StringComparison.OrdinalIgnoreCase);
}
///
@@ -1723,6 +1813,195 @@ private static void UpdateAttributeChangeWithSubstitutedValue(PendingExportAttri
return firstAttr?.Attribute?.ConnectedSystemObjectType?.Name;
}
+ ///
+ /// Checks whether a DirectoryOperationException represents a constraint violation that could be
+ /// caused by the directory rejecting the placeholder member DN (e.g. referential integrity enabled).
+ ///
+ private static bool IsPlaceholderConstraintViolation(DirectoryOperationException ex)
+ {
+ var response = ex.Response;
+ if (response == null)
+ return false;
+
+ return response.ResultCode is ResultCode.ConstraintViolation or ResultCode.NoSuchObject
+ or ResultCode.UnwillingToPerform;
+ }
+
+ ///
+ /// Returns a structured export error when the placeholder member DN is rejected by the directory.
+ /// This typically means the directory has referential integrity enabled and the placeholder DN
+ /// does not reference an existing entry.
+ ///
+ private ConnectedSystemExportResult HandlePlaceholderRejection(string dn, DirectoryOperationException ex)
+ {
+ _logger.Warning(ex,
+ "LdapConnectorExport: Placeholder member '{Placeholder}' was rejected by the directory for '{Dn}'. " +
+ "The directory may have referential integrity enabled. " +
+ "Update the '{Setting}' connector setting to point to an existing entry.",
+ _placeholderMemberDn, dn, LdapConnectorConstants.SETTING_GROUP_PLACEHOLDER_MEMBER_DN);
+
+ return ConnectedSystemExportResult.Failed(
+ $"Failed to add placeholder member '{_placeholderMemberDn}' to group '{dn}' — " +
+ $"the directory may have referential integrity enabled. " +
+ $"Update the '{LdapConnectorConstants.SETTING_GROUP_PLACEHOLDER_MEMBER_DN}' connector setting to point to an existing entry in the directory.",
+ ConnectedSystemExportErrorType.PlaceholderMemberConstraintViolation);
+ }
+
+ ///
+ /// Inspects the consolidated modifications for a groupOfNames/groupOfUniqueNames export and injects
+ /// placeholder member add/remove operations as needed:
+ /// - If member removals would leave the group empty, adds the placeholder DN (to satisfy the MUST constraint).
+ /// - If members are being added to a group that currently only has the placeholder, removes the placeholder.
+ ///
+ private void InjectPlaceholderModificationsIfNeeded(
+ PendingExport pendingExport,
+ List consolidatedModifications)
+ {
+ var objectClass = GetObjectClass(pendingExport);
+ if (string.IsNullOrEmpty(objectClass))
+ return;
+
+ var memberAttrName = GetMemberAttributeName(objectClass);
+
+ // Count current members on the CSO (the directory's current state as JIM knows it)
+ var currentMemberCount = GetCurrentMemberCount(pendingExport, memberAttrName);
+
+ // Count how many members are being added and removed in this export
+ var addCount = 0;
+ var removeCount = 0;
+ foreach (var mod in consolidatedModifications)
+ {
+ if (!mod.AttributeName.Equals(memberAttrName, StringComparison.OrdinalIgnoreCase))
+ continue;
+
+ if (mod.Operation == DirectoryAttributeOperation.Add)
+ addCount += mod.AttributeChanges.Count;
+ else if (mod.Operation == DirectoryAttributeOperation.Delete)
+ removeCount += mod.AttributeChanges.Count;
+ }
+
+ // Check if the group currently only has the placeholder member
+ var hasOnlyPlaceholder = currentMemberCount == 1 && GroupHasOnlyPlaceholderMember(pendingExport, memberAttrName);
+
+ // Scenario 1: Removing members would leave the group empty — inject placeholder
+ if (removeCount > 0 && !hasOnlyPlaceholder)
+ {
+ var effectiveMemberCount = currentMemberCount - removeCount + addCount;
+ if (effectiveMemberCount <= 0)
+ {
+ _logger.Information(
+ "LdapConnectorExport.InjectPlaceholderModificationsIfNeeded: Removing all members from '{ObjectClass}' group. " +
+ "Injecting placeholder member '{Placeholder}' to satisfy MUST constraint.",
+ objectClass, _placeholderMemberDn);
+
+ var placeholderAttr = new ConnectedSystemObjectTypeAttribute
+ {
+ Name = memberAttrName,
+ ConnectedSystemObjectType = pendingExport.ConnectedSystemObject?.Type
+ };
+
+ consolidatedModifications.Add(new ConsolidatedModification
+ {
+ AttributeName = memberAttrName,
+ Operation = DirectoryAttributeOperation.Add,
+ AttributeChanges = { new PendingExportAttributeValueChange
+ {
+ Attribute = placeholderAttr,
+ ChangeType = PendingExportAttributeChangeType.Add,
+ StringValue = _placeholderMemberDn
+ }}
+ });
+ }
+ }
+
+ // Scenario 2: Adding members to a placeholder-only group — remove the placeholder
+ if (addCount > 0 && hasOnlyPlaceholder)
+ {
+ _logger.Information(
+ "LdapConnectorExport.InjectPlaceholderModificationsIfNeeded: Adding real members to placeholder-only '{ObjectClass}' group. " +
+ "Removing placeholder member '{Placeholder}'.",
+ objectClass, _placeholderMemberDn);
+
+ var placeholderAttr = new ConnectedSystemObjectTypeAttribute
+ {
+ Name = memberAttrName,
+ ConnectedSystemObjectType = pendingExport.ConnectedSystemObject?.Type
+ };
+
+ consolidatedModifications.Add(new ConsolidatedModification
+ {
+ AttributeName = memberAttrName,
+ Operation = DirectoryAttributeOperation.Delete,
+ AttributeChanges = { new PendingExportAttributeValueChange
+ {
+ Attribute = placeholderAttr,
+ ChangeType = PendingExportAttributeChangeType.Remove,
+ StringValue = _placeholderMemberDn
+ }}
+ });
+ }
+ }
+
+ ///
+ /// Gets the number of current member values on the CSO for the specified member attribute.
+ ///
+ private static int GetCurrentMemberCount(PendingExport pendingExport, string memberAttrName)
+ {
+ var cso = pendingExport.ConnectedSystemObject;
+ if (cso?.AttributeValues == null)
+ return 0;
+
+ return cso.AttributeValues.Count(av =>
+ av.Attribute?.Name.Equals(memberAttrName, StringComparison.OrdinalIgnoreCase) == true);
+ }
+
+ ///
+ /// Checks whether the group's only current member is the placeholder DN.
+ ///
+ private bool GroupHasOnlyPlaceholderMember(PendingExport pendingExport, string memberAttrName)
+ {
+ var cso = pendingExport.ConnectedSystemObject;
+ if (cso?.AttributeValues == null)
+ return false;
+
+ var memberValues = cso.AttributeValues
+ .Where(av => av.Attribute?.Name.Equals(memberAttrName, StringComparison.OrdinalIgnoreCase) == true)
+ .ToList();
+
+ if (memberValues.Count != 1)
+ return false;
+
+ var memberDn = memberValues[0].UnresolvedReferenceValue ?? memberValues[0].StringValue;
+ return _placeholderMemberDn.Equals(memberDn, StringComparison.OrdinalIgnoreCase);
+ }
+
+ ///
+ /// Determines whether placeholder member handling is required for the given pending export.
+ /// Placeholder handling is needed when:
+ /// 1. The directory is not Active Directory or Samba AD (which use 'group' class with no MUST member constraint)
+ /// 2. The object class is one that requires at least one member value (e.g. groupOfNames, groupOfUniqueNames)
+ ///
+ internal bool RequiresPlaceholderMember(PendingExport pendingExport)
+ {
+ // AD and Samba AD use the 'group' class which allows empty groups — no placeholder needed
+ if (_directoryType is LdapDirectoryType.ActiveDirectory or LdapDirectoryType.SambaAD)
+ return false;
+
+ var objectClass = GetObjectClass(pendingExport);
+ return objectClass != null && LdapConnectorConstants.MUST_MEMBER_OBJECT_CLASSES.Contains(objectClass);
+ }
+
+ ///
+ /// Gets the member attribute name for the object class. Returns "member" for groupOfNames
+ /// and "uniqueMember" for groupOfUniqueNames.
+ ///
+ internal static string GetMemberAttributeName(string objectClass)
+ {
+ return objectClass.Equals("groupOfUniqueNames", StringComparison.OrdinalIgnoreCase)
+ ? "uniqueMember"
+ : "member";
+ }
+
private string? GetSettingValue(string settingName)
{
return _settings.SingleOrDefault(s => s.Setting.Name == settingName)?.StringValue;
@@ -1735,8 +2014,8 @@ private static void UpdateAttributeChangeWithSubstitutedValue(PendingExportAttri
///
/// Builds a list of container DNs from root to the specified container.
- /// For example, for "OU=Engineering,OU=Users,DC=subatomic,DC=local", returns:
- /// ["OU=Users,DC=subatomic,DC=local", "OU=Engineering,OU=Users,DC=subatomic,DC=local"]
+ /// For example, for "OU=Engineering,OU=Users,DC=panoply,DC=local", returns:
+ /// ["OU=Users,DC=panoply,DC=local", "OU=Engineering,OU=Users,DC=panoply,DC=local"]
///
internal static List BuildContainerChain(string containerDn)
{
diff --git a/src/JIM.Connectors/LDAP/LdapConnectorImport.cs b/src/JIM.Connectors/LDAP/LdapConnectorImport.cs
index 76c046756..36b381281 100644
--- a/src/JIM.Connectors/LDAP/LdapConnectorImport.cs
+++ b/src/JIM.Connectors/LDAP/LdapConnectorImport.cs
@@ -1,4 +1,5 @@
-using JIM.Models.Core;
+using JIM.Models.Activities;
+using JIM.Models.Core;
using JIM.Models.Enums;
using JIM.Models.Exceptions;
using JIM.Models.Staging;
@@ -19,9 +20,12 @@ internal class LdapConnectorImport
private readonly ConnectedSystemRunProfile _connectedSystemRunProfile;
private readonly ILogger _logger;
private readonly LdapConnection _connection;
+ private readonly Func? _connectionFactory;
+ private readonly int _importConcurrency;
private readonly List _paginationTokens;
private readonly string? _persistedConnectorData;
private readonly TimeSpan _searchTimeout;
+ private readonly string _placeholderMemberDn;
private LdapConnectorRootDse? _previousRootDse;
private LdapConnectorRootDse? _currentRootDse;
@@ -29,6 +33,8 @@ internal LdapConnectorImport(
ConnectedSystem connectedSystem,
ConnectedSystemRunProfile runProfile,
LdapConnection connection,
+ Func? connectionFactory,
+ int importConcurrency,
List paginationTokens,
string? persistedConnectorData,
ILogger logger,
@@ -37,6 +43,8 @@ internal LdapConnectorImport(
_connectedSystem = connectedSystem;
_connectedSystemRunProfile = runProfile;
_connection = connection;
+ _connectionFactory = connectionFactory;
+ _importConcurrency = Math.Clamp(importConcurrency, 1, LdapConnectorConstants.MAX_IMPORT_CONCURRENCY);
_paginationTokens = paginationTokens;
_persistedConnectorData = persistedConnectorData;
_logger = logger;
@@ -48,6 +56,11 @@ internal LdapConnectorImport(
var searchTimeoutSeconds = searchTimeoutSetting?.IntValue ?? DefaultSearchTimeoutSeconds;
_searchTimeout = TimeSpan.FromSeconds(searchTimeoutSeconds);
+ // Get placeholder member DN for filtering during import
+ var placeholderSetting = connectedSystem.SettingValues
+ .SingleOrDefault(s => s.Setting.Name == LdapConnectorConstants.SETTING_GROUP_PLACEHOLDER_MEMBER_DN);
+ _placeholderMemberDn = placeholderSetting?.StringValue ?? LdapConnectorConstants.DEFAULT_GROUP_PLACEHOLDER_MEMBER_DN;
+
// If we have persisted connector data from a previous page, deserialise it to get capabilities
// This allows subsequent pages to know the directory capabilities without re-querying
if (!string.IsNullOrEmpty(persistedConnectorData) && paginationTokens.Count > 0)
@@ -92,25 +105,76 @@ internal ConnectedSystemImportResult GetFullImportObjects()
result.PersistedConnectorData = JsonSerializer.Serialize(_currentRootDse);
}
- // enumerate target partitions (scoped to run profile partition if set, otherwise all selected)
+ // OpenLDAP's RFC 2696 paging cookies are connection-scoped: any new search on the same
+ // connection invalidates all outstanding paging cursors. To work around this, we give each
+ // container+objectType combo its own dedicated LdapConnection and run combos in parallel,
+ // capped by the Import Concurrency setting. Each connection fully drains its paged search
+ // before being disposed, so there are no cross-call pagination tokens for this path — all
+ // data is fetched within this single call.
+ //
+ // When the connection factory is unavailable or concurrency is 1, we fall back to the
+ // original serialised approach using the primary connection (one combo at a time).
+ //
+ // For AD directories, this block is skipped entirely — AD supports multiple concurrent
+ // paged searches on a single connection, so the original multi-combo-per-page logic below
+ // is used instead.
+ var isConnectionScopedPaging = _currentRootDse?.DirectoryType is LdapDirectoryType.OpenLDAP or LdapDirectoryType.Generic;
+
+ if (isConnectionScopedPaging)
+ {
+ // Build the ordered list of all container+objectType combos
+ var combos = new List<(ConnectedSystemContainer Container, ConnectedSystemObjectType ObjectType)>();
+ foreach (var selectedPartition in GetTargetPartitions())
+ {
+ foreach (var selectedContainer in ConnectedSystemUtilities.GetTopLevelSelectedContainers(selectedPartition))
+ {
+ foreach (var selectedObjectType in _connectedSystem.ObjectTypes.Where(ot => ot.Selected))
+ {
+ combos.Add((selectedContainer, selectedObjectType));
+ }
+ }
+ }
+
+ if (combos.Count == 0)
+ return result;
+
+ _logger.Debug("GetFullImportObjects: OpenLDAP/Generic directory detected. Processing {ComboCount} container+objectType combos with concurrency {Concurrency}",
+ combos.Count, _connectionFactory != null ? _importConcurrency : 1);
+
+ if (_connectionFactory != null && _importConcurrency > 1)
+ {
+ // Parallel path: one dedicated connection per combo, capped by semaphore.
+ // Each combo fully drains all pages on its own connection, so no pagination
+ // tokens are returned — the import processor sees this as a single-page result.
+ GetFullImportObjectsParallel(result, combos);
+ }
+ else
+ {
+ // Sequential fallback: use the primary connection, one combo at a time.
+ // Each combo is fully drained before moving to the next.
+ GetFullImportObjectsSequential(result, combos);
+ }
+
+ return result;
+ }
+
+ // Non-OpenLDAP: original behaviour — query all combos on every page
foreach (var selectedPartition in GetTargetPartitions())
{
- // enumerate top-level selected containers in this partition
- // Use GetTopLevelSelectedContainers to avoid duplicates when both parent and child containers are selected
- // (subtree search on parent already includes children)
foreach (var selectedContainer in ConnectedSystemUtilities.GetTopLevelSelectedContainers(selectedPartition))
{
- // we need to perform a query per object type, so that we can have distinct attribute lists per LDAP request
foreach (var selectedObjectType in _connectedSystem.ObjectTypes.Where(ot => ot.Selected))
{
- // if this is the subsequent page for this container, use this when getting the results for the next page
var paginationTokenName = LdapConnectorUtilities.GetPaginationTokenName(selectedContainer, selectedObjectType);
var paginationToken = _paginationTokens.SingleOrDefault(pt => pt.Name == paginationTokenName);
var lastRunsCookie = paginationToken?.ByteValue;
+ if (_paginationTokens.Count > 0 && paginationToken == null)
+ continue;
+
if (_cancellationToken.IsCancellationRequested)
{
- _logger.Debug("GetFullImportObjects: O2 Cancellation requested. Stopping");
+ _logger.Debug("GetFullImportObjects: Cancellation requested. Stopping");
return result;
}
@@ -166,7 +230,7 @@ internal ConnectedSystemImportResult GetDeltaImportObjects()
}
// Determine which delta strategy to use
- if (_previousRootDse.IsActiveDirectory)
+ if (_previousRootDse.UseUsnDeltaImport)
{
if (!_previousRootDse.HighestCommittedUsn.HasValue)
{
@@ -194,6 +258,10 @@ internal ConnectedSystemImportResult GetDeltaImportObjects()
var paginationToken = _paginationTokens.SingleOrDefault(pt => pt.Name == paginationTokenName);
var lastRunsCookie = paginationToken?.ByteValue;
+ // On subsequent pages, skip combos with no pagination token (see full import comment)
+ if (_paginationTokens.Count > 0 && paginationToken == null)
+ continue;
+
GetDeltaResultsUsingUsn(result, selectedContainer, selectedObjectType, _previousRootDse.HighestCommittedUsn.Value, lastRunsCookie);
}
}
@@ -210,9 +278,44 @@ internal ConnectedSystemImportResult GetDeltaImportObjects()
GetDeletedObjectsUsingUsn(result, selectedPartition, _previousRootDse.HighestCommittedUsn.Value);
}
}
+ else if (_previousRootDse.UseAccesslogDeltaImport)
+ {
+ // For OpenLDAP with accesslog overlay
+ if (string.IsNullOrEmpty(_previousRootDse.LastAccesslogTimestamp))
+ {
+ // The accesslog watermark is not available. This can happen when:
+ // - The accesslog has more entries than the server's olcSizeLimit (default 500)
+ // and the bind account cannot bypass the limit (not the accesslog DB rootDN)
+ // - The accesslog overlay is not enabled or not accessible
+ // - The previous full import failed to capture the watermark
+ //
+ // Rather than failing, fall back to a full import which will correctly import
+ // all objects AND establish the watermark for future delta imports.
+ _logger.Warning("GetDeltaImportObjects: Accesslog watermark not available. " +
+ "Falling back to full import to establish baseline. " +
+ "Future delta imports should work normally after this full import completes.");
+
+ result = GetFullImportObjects();
+ result.WarningMessage = "Delta import was requested but the accesslog watermark was not available " +
+ "(the cn=accesslog database may have exceeded the server's size limit for the bind account). " +
+ "A full import was performed instead. The watermark has been established and future " +
+ "delta imports should succeed normally.";
+ result.WarningErrorType = ActivityRunProfileExecutionItemErrorType.DeltaImportFallbackToFullImport;
+ return result;
+ }
+
+ _logger.Debug("GetDeltaImportObjects: Using accesslog-based delta import. Previous timestamp: {PreviousTimestamp}",
+ _previousRootDse.LastAccesslogTimestamp);
+
+ // Pass the target partitions so the method can filter accesslog entries by DN suffix.
+ // OpenLDAP uses a shared cn=accesslog for all databases, so entries from other suffixes
+ // (e.g., Target) must be excluded when importing from Source.
+ var targetPartitions = GetTargetPartitions().ToList();
+ GetDeltaResultsUsingAccesslog(result, _previousRootDse.LastAccesslogTimestamp, targetPartitions);
+ }
else
{
- // For changelog-based directories
+ // For generic changelog-based directories (Oracle, 389DS, etc.)
if (!_previousRootDse.LastChangeNumber.HasValue)
{
throw new CannotPerformDeltaImportException("Previous changelog number not available. Run a full import first.");
@@ -244,6 +347,149 @@ private IEnumerable GetTargetPartitions()
return _connectedSystem.Partitions!.Where(p => p.Selected);
}
+ ///
+ /// Processes all container+objectType combos in parallel using a dedicated LdapConnection per combo.
+ /// Each combo fully drains all pages on its own connection, avoiding the RFC 2696 connection-scoped
+ /// paging cookie limitation. Concurrency is capped by .
+ ///
+ private void GetFullImportObjectsParallel(
+ ConnectedSystemImportResult result,
+ List<(ConnectedSystemContainer Container, ConnectedSystemObjectType ObjectType)> combos)
+ {
+ var stopwatch = Stopwatch.StartNew();
+
+ // Each combo gets its own result to avoid contention on shared collections.
+ // Results are merged after all combos complete.
+ var comboResults = new ConnectedSystemImportResult[combos.Count];
+ for (var i = 0; i < comboResults.Length; i++)
+ comboResults[i] = new ConnectedSystemImportResult();
+
+ using var semaphore = new SemaphoreSlim(_importConcurrency);
+ var tasks = new Task[combos.Count];
+
+ for (var i = 0; i < combos.Count; i++)
+ {
+ _cancellationToken.ThrowIfCancellationRequested();
+
+ var index = i;
+ var (container, objectType) = combos[i];
+
+ tasks[i] = Task.Run(async () =>
+ {
+ await semaphore.WaitAsync(_cancellationToken);
+ LdapConnection? comboConnection = null;
+ try
+ {
+ comboConnection = _connectionFactory!();
+ _logger.Debug("GetFullImportObjectsParallel: Started combo {Index}/{Total} — container={Container}, objectType={ObjectType}",
+ index + 1, combos.Count, container.Name, objectType.Name);
+
+ // Fully drain all pages for this combo on its dedicated connection
+ DrainAllPages(comboResults[index], comboConnection, container, objectType);
+ }
+ catch (OperationCanceledException)
+ {
+ _logger.Debug("GetFullImportObjectsParallel: Combo {Index} cancelled", index + 1);
+ }
+ catch (Exception ex)
+ {
+ _logger.Error(ex, "GetFullImportObjectsParallel: Combo {Index} failed — container={Container}, objectType={ObjectType}",
+ index + 1, container.Name, objectType.Name);
+ throw;
+ }
+ finally
+ {
+ comboConnection?.Dispose();
+ semaphore.Release();
+ }
+ }, _cancellationToken);
+ }
+
+ try
+ {
+ Task.WaitAll(tasks, _cancellationToken);
+ }
+ catch (OperationCanceledException)
+ {
+ _logger.Debug("GetFullImportObjectsParallel: Cancelled while waiting for combos to complete");
+ return;
+ }
+ catch (AggregateException ae)
+ {
+ // Unwrap and rethrow the first real exception so the import processor sees it
+ var inner = ae.Flatten().InnerExceptions.FirstOrDefault(e => e is not OperationCanceledException);
+ if (inner != null)
+ throw inner;
+ return;
+ }
+
+ // Merge results from all combos
+ foreach (var comboResult in comboResults)
+ {
+ result.ImportObjects.AddRange(comboResult.ImportObjects);
+ }
+
+ stopwatch.Stop();
+ _logger.Information("GetFullImportObjectsParallel: Completed {ComboCount} combos in {Elapsed}. Total objects: {ObjectCount}",
+ combos.Count, stopwatch.Elapsed, result.ImportObjects.Count);
+ }
+
+ ///
+ /// Processes all container+objectType combos sequentially on the primary connection.
+ /// Each combo is fully drained (all pages) before moving to the next.
+ /// Used as a fallback when the connection factory is unavailable or concurrency is 1.
+ ///
+ private void GetFullImportObjectsSequential(
+ ConnectedSystemImportResult result,
+ List<(ConnectedSystemContainer Container, ConnectedSystemObjectType ObjectType)> combos)
+ {
+ foreach (var (container, objectType) in combos)
+ {
+ if (_cancellationToken.IsCancellationRequested)
+ {
+ _logger.Debug("GetFullImportObjectsSequential: Cancellation requested. Stopping");
+ return;
+ }
+
+ DrainAllPages(result, _connection, container, objectType);
+ }
+ }
+
+ ///
+ /// Fully drains all pages for a single container+objectType combination on the given connection.
+ /// Keeps issuing paged search requests until the server returns an empty paging cookie.
+ ///
+ private void DrainAllPages(
+ ConnectedSystemImportResult result,
+ LdapConnection connection,
+ ConnectedSystemContainer container,
+ ConnectedSystemObjectType objectType)
+ {
+ byte[]? pagingCookie = null;
+
+ while (true)
+ {
+ if (_cancellationToken.IsCancellationRequested)
+ return;
+
+ var comboResult = new ConnectedSystemImportResult();
+ GetFisoResults(comboResult, connection, container, objectType, pagingCookie);
+
+ result.ImportObjects.AddRange(comboResult.ImportObjects);
+
+ // Check if there are more pages
+ if (comboResult.PaginationTokens.Count > 0)
+ {
+ pagingCookie = comboResult.PaginationTokens[0].ByteValue;
+ }
+ else
+ {
+ // No more pages — this combo is fully drained
+ break;
+ }
+ }
+ }
+
#region private methods
///
/// For directories that support changelog.
@@ -286,6 +532,185 @@ private IEnumerable GetTargetPartitions()
}
}
+ ///
+ /// Queries the OpenLDAP accesslog overlay (cn=accesslog) for the latest reqStart timestamp.
+ /// This establishes the watermark for the next delta import.
+ ///
+ /// Strategy:
+ /// 1. Try server-side sort (reverse by reqStart) with SizeLimit=1 to get only the latest entry.
+ /// This is the most efficient approach but requires the sssvlv overlay to be enabled.
+ /// 2. If sort is not supported, fall back to a simple query that handles size limit exceeded
+ /// by extracting partial results from the exception response.
+ ///
+ /// OpenLDAP enforces olcSizeLimit (default 500) as a hard cap for non-rootDN clients, even
+ /// with paging controls. The bind account used by the connector is typically not the rootDN
+ /// of the cn=accesslog database, so paging alone cannot bypass the limit. The strategies
+ /// above are designed to work within this constraint.
+ ///
+ private string? QueryAccesslogForLatestTimestamp()
+ {
+ try
+ {
+ // Strategy 1: Server-side sort (reverse) with SizeLimit=1
+ // This gets only the single latest entry, avoiding size limit issues entirely.
+ var result = QueryAccesslogWithServerSideSort();
+ if (result != null)
+ return result;
+
+ // Strategy 2: Simple query with size limit exceeded handling.
+ // If the accesslog has fewer entries than olcSizeLimit, this returns all entries normally.
+ // If it exceeds the limit, we catch the DirectoryOperationException and extract
+ // the latest timestamp from the partial results in the exception's response.
+ return QueryAccesslogWithSizeLimitHandling();
+ }
+ catch (Exception ex)
+ {
+ _logger.Warning(ex, "QueryAccesslogForLatestTimestamp: Failed to query accesslog. " +
+ "The directory may not have the accesslog overlay enabled.");
+ return null;
+ }
+ }
+
+ ///
+ /// Attempts to query the accesslog using server-side sorting (reverse by reqStart) with
+ /// SizeLimit=1 to retrieve only the latest entry. This requires the sssvlv overlay.
+ /// Returns null if server-side sorting is not supported.
+ ///
+ private string? QueryAccesslogWithServerSideSort()
+ {
+ try
+ {
+ var request = new SearchRequest("cn=accesslog",
+ "(&(objectClass=auditWriteObject)(reqResult=0))",
+ SearchScope.OneLevel,
+ "reqStart");
+
+ // Request reverse sort by reqStart so the latest entry comes first
+ var sortControl = new SortRequestControl(new SortKey("reqStart", "caseIgnoreOrderingMatch", true));
+ sortControl.IsCritical = true;
+ request.Controls.Add(sortControl);
+ request.SizeLimit = 1;
+
+ var response = (SearchResponse)_connection.SendRequest(request, _searchTimeout);
+ if (response?.Entries.Count > 0)
+ {
+ var timestamp = LdapConnectorUtilities.GetEntryAttributeStringValue(response.Entries[0], "reqStart");
+ if (timestamp != null)
+ {
+ _logger.Debug("QueryAccesslogWithServerSideSort: Latest accesslog timestamp: {Timestamp} (via server-side sort)", timestamp);
+ return timestamp;
+ }
+ }
+
+ _logger.Debug("QueryAccesslogWithServerSideSort: No accesslog entries found via server-side sort");
+ return null;
+ }
+ catch (DirectoryOperationException ex) when (ex.Response is SearchResponse { ResultCode: ResultCode.UnavailableCriticalExtension or ResultCode.UnwillingToPerform })
+ {
+ _logger.Debug("QueryAccesslogWithServerSideSort: Server-side sorting not supported (sssvlv overlay not enabled). Falling back to size-limit-aware query.");
+ return null;
+ }
+ catch (DirectoryOperationException ex) when (ex.Response is SearchResponse { ResultCode: ResultCode.InappropriateMatching })
+ {
+ // Matching rule not supported for this attribute — fall back
+ _logger.Debug("QueryAccesslogWithServerSideSort: Sort matching rule not supported for reqStart. Falling back to size-limit-aware query.");
+ return null;
+ }
+ }
+
+ ///
+ /// Queries the accesslog using an iterative approach that works within the server's size limit.
+ /// When the size limit is exceeded, extracts the latest timestamp from partial results and
+ /// re-queries with a narrower filter (reqStart >= latest_seen) to walk forward through the
+ /// accesslog until all entries have been scanned. This effectively implements manual paging
+ /// without requiring paging controls to bypass the size limit.
+ ///
+ private string? QueryAccesslogWithSizeLimitHandling()
+ {
+ string? latestTimestamp = null;
+ var totalEntries = 0;
+ var iterations = 0;
+ const int maxIterations = 100; // Safety limit to prevent infinite loops
+
+ // Start with an unfiltered query to get the first batch
+ var currentFilter = "(&(objectClass=auditWriteObject)(reqResult=0))";
+
+ while (iterations < maxIterations)
+ {
+ iterations++;
+ string? batchLatest;
+ int batchCount;
+ var hitSizeLimit = false;
+
+ try
+ {
+ var request = new SearchRequest("cn=accesslog", currentFilter,
+ SearchScope.OneLevel, "reqStart");
+
+ var response = (SearchResponse)_connection.SendRequest(request, _searchTimeout);
+ (batchLatest, batchCount) = ExtractLatestTimestamp(response);
+ }
+ catch (DirectoryOperationException ex) when (ex.Response is SearchResponse partialResponse
+ && partialResponse.ResultCode == ResultCode.SizeLimitExceeded)
+ {
+ // The server hit its size limit but returned partial results.
+ (batchLatest, batchCount) = ExtractLatestTimestamp(partialResponse);
+ hitSizeLimit = true;
+ }
+
+ totalEntries += batchCount;
+
+ if (batchCount == 0 || batchLatest == null)
+ break;
+
+ // Update the overall latest timestamp
+ if (latestTimestamp == null || string.Compare(batchLatest, latestTimestamp, StringComparison.Ordinal) > 0)
+ latestTimestamp = batchLatest;
+
+ if (!hitSizeLimit)
+ break; // Got all results without hitting the limit — done
+
+ // Size limit was hit. The partial results contain the earliest entries (OpenLDAP
+ // returns in insertion order). Re-query starting after the latest timestamp we've
+ // seen to walk forward through the remaining entries.
+ _logger.Debug("QueryAccesslogWithSizeLimitHandling: Size limit exceeded on iteration {Iteration}. " +
+ "Latest timestamp so far: {Timestamp}. Re-querying from that point.",
+ iterations, latestTimestamp);
+
+ currentFilter = $"(&(objectClass=auditWriteObject)(reqResult=0)(reqStart>={latestTimestamp}))";
+ }
+
+ if (totalEntries == 0)
+ {
+ _logger.Debug("QueryAccesslogWithSizeLimitHandling: No accesslog entries found");
+ return null;
+ }
+
+ _logger.Debug("QueryAccesslogWithSizeLimitHandling: Latest accesslog timestamp: {Timestamp} " +
+ "(scanned {Count} entries in {Iterations} iterations)",
+ latestTimestamp, totalEntries, iterations);
+ return latestTimestamp;
+ }
+
+ ///
+ /// Extracts the latest reqStart timestamp from a search response containing accesslog entries.
+ ///
+ private static (string? latestTimestamp, int entryCount) ExtractLatestTimestamp(SearchResponse response)
+ {
+ string? latestTimestamp = null;
+ var count = 0;
+
+ foreach (SearchResultEntry entry in response.Entries)
+ {
+ var reqStart = LdapConnectorUtilities.GetEntryAttributeStringValue(entry, "reqStart");
+ if (reqStart != null && (latestTimestamp == null || string.Compare(reqStart, latestTimestamp, StringComparison.Ordinal) > 0))
+ latestTimestamp = reqStart;
+ count++;
+ }
+
+ return (latestTimestamp, count);
+ }
+
private LdapConnectorRootDse GetRootDseInformation()
{
var request = new SearchRequest()
@@ -297,7 +722,8 @@ private LdapConnectorRootDse GetRootDseInformation()
"DNSHostName",
"HighestCommittedUSN",
"supportedCapabilities",
- "vendorName"
+ "vendorName",
+ "structuralObjectClass"
});
var response = (SearchResponse)_connection.SendRequest(request);
@@ -313,43 +739,60 @@ private LdapConnectorRootDse GetRootDseInformation()
var rootDseEntry = response.Entries[0];
- // Check if this is Active Directory by looking for AD capability OIDs
+ // Detect directory type from rootDSE capabilities
var capabilities = LdapConnectorUtilities.GetEntryAttributeStringValues(rootDseEntry, "supportedCapabilities");
- var isActiveDirectory = capabilities != null &&
- (capabilities.Contains(LdapConnectorConstants.LDAP_CAP_ACTIVE_DIRECTORY_OID) ||
- capabilities.Contains(LdapConnectorConstants.LDAP_CAP_ACTIVE_DIRECTORY_ADAM_OID));
-
- // Get vendor name for capability detection
var vendorName = LdapConnectorUtilities.GetEntryAttributeStringValue(rootDseEntry, "vendorName");
-
- // Determine paging support based on directory type
- // Samba AD claims AD compatibility but doesn't properly support paged searches
- // (it returns paging cookies but then returns the same results on subsequent pages)
- var isSambaAd = vendorName != null &&
- vendorName.Contains("Samba", StringComparison.OrdinalIgnoreCase);
- var supportsPaging = isActiveDirectory && !isSambaAd;
+ var structuralObjectClass = LdapConnectorUtilities.GetEntryAttributeStringValue(rootDseEntry, "structuralObjectClass");
+ var directoryType = LdapConnectorUtilities.DetectDirectoryType(capabilities, vendorName, structuralObjectClass);
var rootDse = new LdapConnectorRootDse
{
DnsHostName = LdapConnectorUtilities.GetEntryAttributeStringValue(rootDseEntry, "DNSHostName"),
HighestCommittedUsn = LdapConnectorUtilities.GetEntryAttributeLongValue(rootDseEntry, "HighestCommittedUSN"),
- IsActiveDirectory = isActiveDirectory,
- VendorName = vendorName,
- SupportsPaging = supportsPaging
+ DirectoryType = directoryType,
+ VendorName = vendorName
};
- // For non-AD directories, try to get the last change number from the changelog
- if (!isActiveDirectory)
+ // For non-AD directories, capture the current delta watermark.
+ // This must run during BOTH full and delta imports:
+ // - Full import: establishes the baseline watermark for the first delta import
+ // - Delta import: captures the current position so the next delta starts from here
+ if (!rootDse.UseUsnDeltaImport)
{
- rootDse.LastChangeNumber = QueryDirectoryForLastChangeNumber(0);
+ if (rootDse.UseAccesslogDeltaImport)
+ {
+ // OpenLDAP: query cn=accesslog for the latest reqStart timestamp
+ rootDse.LastAccesslogTimestamp = QueryAccesslogForLatestTimestamp();
+
+ // If the accesslog is empty (e.g., after snapshot restore clears stale data),
+ // generate a fallback timestamp so the watermark is never null. This prevents
+ // the next delta import from falling back to a full import unnecessarily.
+ if (string.IsNullOrEmpty(rootDse.LastAccesslogTimestamp))
+ {
+ rootDse.LastAccesslogTimestamp = LdapConnectorUtilities.GenerateAccesslogFallbackTimestamp();
+ _logger.Information("GetRootDseInformation: Accesslog is empty — using fallback timestamp {Timestamp} as watermark",
+ rootDse.LastAccesslogTimestamp);
+ }
+ }
+ else
+ {
+ // Generic/Oracle: query cn=changelog for the latest changeNumber
+ rootDse.LastChangeNumber = QueryDirectoryForLastChangeNumber(0);
+ }
}
- _logger.Information("GetRootDseInformation: Directory capabilities detected. IsActiveDirectory={IsAd}, VendorName={VendorName}, SupportsPaging={SupportsPaging}, HighestUSN={Usn}, LastChangeNumber={ChangeNum}",
- rootDse.IsActiveDirectory, rootDse.VendorName ?? "(not set)", rootDse.SupportsPaging, rootDse.HighestCommittedUsn, rootDse.LastChangeNumber);
+ _logger.Information("GetRootDseInformation: Directory capabilities detected. DirectoryType={DirectoryType}, VendorName={VendorName}, SupportsPaging={SupportsPaging}, HighestUSN={Usn}, LastChangeNumber={ChangeNum}, LastAccesslogTimestamp={AccesslogTs}",
+ rootDse.DirectoryType, rootDse.VendorName ?? "(not set)", rootDse.SupportsPaging, rootDse.HighestCommittedUsn, rootDse.LastChangeNumber, rootDse.LastAccesslogTimestamp ?? "(not set)");
return rootDse;
}
+ ///
+ /// Overload that uses the primary connection. Called by the AD (non-connection-scoped) path and delta imports.
+ ///
private void GetFisoResults(ConnectedSystemImportResult connectedSystemImportResult, ConnectedSystemContainer connectedSystemContainer, ConnectedSystemObjectType connectedSystemObjectType, byte[]? lastRunsCookie)
+ => GetFisoResults(connectedSystemImportResult, _connection, connectedSystemContainer, connectedSystemObjectType, lastRunsCookie);
+
+ private void GetFisoResults(ConnectedSystemImportResult connectedSystemImportResult, LdapConnection connection, ConnectedSystemContainer connectedSystemContainer, ConnectedSystemObjectType connectedSystemObjectType, byte[]? lastRunsCookie)
{
if (_cancellationToken.IsCancellationRequested)
{
@@ -402,7 +845,7 @@ private void GetFisoResults(ConnectedSystemImportResult connectedSystemImportRes
SearchResponse searchResponse;
try
{
- searchResponse = (SearchResponse)_connection.SendRequest(searchRequest, _searchTimeout);
+ searchResponse = (SearchResponse)connection.SendRequest(searchRequest, _searchTimeout);
}
catch (DirectoryOperationException ex) when (lastRunsCookie is { Length: > 0 } &&
ex.Message.Contains("does not support the control", StringComparison.OrdinalIgnoreCase))
@@ -763,8 +1206,294 @@ private void GetDeltaResultsUsingChangelog(ConnectedSystemImportResult result, i
}
///
- /// Fetches a single object by its DN for changelog-based delta imports.
+ /// Gets delta results using OpenLDAP's accesslog overlay (slapo-accesslog).
+ /// Queries cn=accesslog for write operations that occurred after the previous watermark timestamp.
+ /// For each change, fetches the current state of the affected object.
+ ///
+ /// Handles the server-side size limit (olcSizeLimit, default 500) by iterating through batches:
+ /// when the size limit is exceeded, the latest timestamp from partial results is used to narrow
+ /// the next query, effectively walking forward through the accesslog until all changes are found.
///
+ private void GetDeltaResultsUsingAccesslog(ConnectedSystemImportResult result, string previousTimestamp,
+ List targetPartitions)
+ {
+ _logger.Debug("GetDeltaResultsUsingAccesslog: Querying for changes since {PreviousTimestamp}", previousTimestamp);
+
+ var currentTimestamp = previousTimestamp;
+ var totalEntries = 0;
+ var skippedOutOfScope = 0;
+ var iterations = 0;
+ const int maxIterations = 100; // Safety limit
+ // Track processed DNs+timestamps to avoid duplicates when iterating with >= filters
+ var processedEntries = new HashSet(StringComparer.Ordinal);
+ // Track processed DNs to avoid importing the same object multiple times when the
+ // accesslog has multiple changes for the same DN (e.g., 3 member modifications to
+ // the same group). Since we fetch current state rather than replaying individual
+ // changes, only the first occurrence needs to be processed.
+ var processedDns = new HashSet(StringComparer.OrdinalIgnoreCase);
+ // Build partition suffix list for filtering — OpenLDAP shares cn=accesslog across
+ // all databases, so we must exclude entries from other suffixes.
+ var partitionSuffixes = targetPartitions
+ .Select(p => p.ExternalId)
+ .Where(id => !string.IsNullOrEmpty(id))
+ .ToList();
+
+ while (iterations < maxIterations)
+ {
+ iterations++;
+
+ if (_cancellationToken.IsCancellationRequested)
+ {
+ _logger.Debug("GetDeltaResultsUsingAccesslog: Cancellation requested. Stopping");
+ return;
+ }
+
+ // LDAP only supports >= (not >), so we use >= and skip already-processed entries in code.
+ var ldapFilter = $"(&(objectClass=auditWriteObject)(reqResult=0)(reqStart>={currentTimestamp}))";
+ // Request reqOld for delete entries — contains the old objectClass and entryUUID
+ // which are needed to construct proper delete import objects.
+ var request = new SearchRequest("cn=accesslog", ldapFilter, SearchScope.OneLevel,
+ "reqStart", "reqType", "reqDN", "reqOld", "reqEntryUUID");
+
+ SearchResponse response;
+ var hitSizeLimit = false;
+
+ try
+ {
+ response = (SearchResponse)_connection.SendRequest(request, _searchTimeout);
+
+ if (response == null || response.ResultCode != ResultCode.Success)
+ {
+ _logger.Warning("GetDeltaResultsUsingAccesslog: Failed to query accesslog. ResultCode: {ResultCode}",
+ response?.ResultCode);
+ return;
+ }
+ }
+ catch (DirectoryOperationException ex) when (ex.Response is SearchResponse partialResponse
+ && partialResponse.ResultCode == ResultCode.SizeLimitExceeded)
+ {
+ // Size limit exceeded — process the partial results we got
+ response = partialResponse;
+ hitSizeLimit = true;
+ _logger.Debug("GetDeltaResultsUsingAccesslog: Size limit exceeded on iteration {Iteration}. " +
+ "Processing {Count} partial results.", iterations, response.Entries.Count);
+ }
+ catch (Exception ex)
+ {
+ _logger.Error(ex, "GetDeltaResultsUsingAccesslog: Error querying accesslog");
+ return;
+ }
+
+ if (response.Entries.Count == 0)
+ break;
+
+ _logger.Debug("GetDeltaResultsUsingAccesslog: Processing {Count} accesslog entries (iteration {Iteration})",
+ response.Entries.Count, iterations);
+
+ string? batchLatestTimestamp = null;
+
+ foreach (SearchResultEntry entry in response.Entries)
+ {
+ if (_cancellationToken.IsCancellationRequested)
+ {
+ _logger.Debug("GetDeltaResultsUsingAccesslog: Cancellation requested. Stopping");
+ return;
+ }
+
+ var reqStart = LdapConnectorUtilities.GetEntryAttributeStringValue(entry, "reqStart");
+ var reqType = LdapConnectorUtilities.GetEntryAttributeStringValue(entry, "reqType");
+ var reqDn = LdapConnectorUtilities.GetEntryAttributeStringValue(entry, "reqDN");
+
+ if (string.IsNullOrEmpty(reqDn) || string.IsNullOrEmpty(reqStart))
+ continue;
+
+ // Filter by partition scope — only process entries whose DN falls within
+ // the connected system's selected partitions. OpenLDAP's shared cn=accesslog
+ // records changes from ALL databases (suffixes), so we must exclude entries
+ // from other suffixes to avoid importing objects from the wrong partition.
+ if (partitionSuffixes.Count > 0 &&
+ !partitionSuffixes.Any(suffix => reqDn.EndsWith(suffix, StringComparison.OrdinalIgnoreCase)))
+ {
+ skippedOutOfScope++;
+ continue;
+ }
+
+ // Track the latest timestamp in this batch for the next iteration
+ if (batchLatestTimestamp == null || string.Compare(reqStart, batchLatestTimestamp, StringComparison.Ordinal) > 0)
+ batchLatestTimestamp = reqStart;
+
+ // Skip the exact timestamp match from the previous watermark
+ if (reqStart == previousTimestamp)
+ continue;
+
+ // Skip entries we've already processed (from overlapping >= queries)
+ var entryKey = $"{reqStart}|{reqDn}";
+ if (!processedEntries.Add(entryKey))
+ continue;
+
+ totalEntries++;
+
+ // Map accesslog reqType to ObjectChangeType
+ var objectChangeType = reqType?.ToLowerInvariant() switch
+ {
+ "add" => ObjectChangeType.Added,
+ "modify" => ObjectChangeType.Updated,
+ "delete" => ObjectChangeType.Deleted,
+ "modrdn" => ObjectChangeType.Updated,
+ _ => ObjectChangeType.NotSet
+ };
+
+ // Deduplicate by DN for non-delete operations. Multiple add/modify entries for
+ // the same DN would produce identical import objects (since we fetch current state),
+ // triggering duplicate detection errors. However, delete entries must ALWAYS be
+ // processed even if the DN was already seen — a group that was added then deleted
+ // in the same delta window must be processed as a delete (the final state).
+ if (objectChangeType != ObjectChangeType.Deleted && !processedDns.Add(reqDn))
+ continue;
+
+ if (objectChangeType == ObjectChangeType.Deleted)
+ {
+ // For deletes, extract object type and external ID from the accesslog entry.
+ // The reqOld attribute contains the old attribute values (key: value format),
+ // and reqEntryUUID contains the entryUUID of the deleted object.
+ var deleteObject = BuildDeleteImportObjectFromAccesslog(entry);
+ if (deleteObject != null)
+ result.ImportObjects.Add(deleteObject);
+ }
+ else
+ {
+ // For add/modify/modrdn, fetch the current state of the object
+ var currentObject = GetObjectByDn(reqDn, objectChangeType);
+ if (currentObject != null)
+ {
+ result.ImportObjects.Add(currentObject);
+ }
+ else
+ {
+ _logger.Warning("GetDeltaResultsUsingAccesslog: GetObjectByDn returned null for DN '{ReqDn}' " +
+ "(change type: {ChangeType}). The object may have been deleted or moved since the accesslog " +
+ "entry was recorded. Entry skipped.", reqDn, objectChangeType);
+ }
+ }
+ }
+
+ if (!hitSizeLimit)
+ break; // Got all results without hitting the limit — done
+
+ // Size limit was hit. Narrow the query to start from the latest timestamp we've seen
+ // to walk forward through the remaining entries.
+ if (batchLatestTimestamp == null || batchLatestTimestamp == currentTimestamp)
+ {
+ // No progress made — all entries have the same timestamp. Cannot narrow further.
+ _logger.Warning("GetDeltaResultsUsingAccesslog: Cannot narrow accesslog query further. " +
+ "All {Count} entries in this batch have timestamp {Timestamp}. Some changes may be missed.",
+ response.Entries.Count, currentTimestamp);
+ break;
+ }
+
+ currentTimestamp = batchLatestTimestamp;
+ }
+
+ _logger.Debug("GetDeltaResultsUsingAccesslog: Processed {TotalEntries} change entries in {Iterations} iterations. " +
+ "Skipped {SkippedOutOfScope} entries outside partition scope.",
+ totalEntries, iterations, skippedOutOfScope);
+ }
+
+ ///
+ /// Fetches a single object by its DN for changelog/accesslog-based delta imports.
+ ///
+ ///
+ /// Builds a delete import object from an accesslog auditDelete entry.
+ /// Extracts the objectClass and entryUUID from reqOld attributes to construct
+ /// a proper import object with ObjectType and external ID, matching what the
+ /// USN-based delete detection produces.
+ ///
+ private ConnectedSystemImportObject? BuildDeleteImportObjectFromAccesslog(SearchResultEntry accesslogEntry)
+ {
+ if (_connectedSystem.ObjectTypes == null)
+ return null;
+
+ // Extract entryUUID — try reqEntryUUID first (direct attribute), then reqOld
+ var entryUuid = LdapConnectorUtilities.GetEntryAttributeStringValue(accesslogEntry, "reqEntryUUID");
+
+ // Extract objectClass from reqOld values (format: "attributeName: value")
+ var reqOldValues = LdapConnectorUtilities.GetEntryAttributeStringValues(accesslogEntry, "reqOld");
+ string? objectClassName = null;
+
+ if (reqOldValues != null)
+ {
+ foreach (var oldValue in reqOldValues)
+ {
+ if (oldValue.StartsWith("objectClass: ", StringComparison.OrdinalIgnoreCase))
+ {
+ var className = oldValue["objectClass: ".Length..].Trim();
+ // Find the most specific matching object type (skip generic ones like 'top')
+ var matchedType = _connectedSystem.ObjectTypes
+ .FirstOrDefault(ot => ot.Selected && ot.Name.Equals(className, StringComparison.OrdinalIgnoreCase));
+ if (matchedType != null)
+ objectClassName = matchedType.Name;
+ }
+
+ // Also try to get entryUUID from reqOld if not found via reqEntryUUID
+ if (entryUuid == null && oldValue.StartsWith("entryUUID: ", StringComparison.OrdinalIgnoreCase))
+ {
+ entryUuid = oldValue["entryUUID: ".Length..].Trim();
+ }
+ }
+ }
+
+ if (string.IsNullOrEmpty(objectClassName))
+ {
+ var reqDn = LdapConnectorUtilities.GetEntryAttributeStringValue(accesslogEntry, "reqDN");
+ _logger.Warning("BuildDeleteImportObjectFromAccesslog: Could not determine object type for deleted object. " +
+ "DN: {Dn}. The accesslog entry may not contain reqOld attributes.", reqDn);
+ return null;
+ }
+
+ if (string.IsNullOrEmpty(entryUuid))
+ {
+ var reqDn = LdapConnectorUtilities.GetEntryAttributeStringValue(accesslogEntry, "reqDN");
+ _logger.Warning("BuildDeleteImportObjectFromAccesslog: Could not determine entryUUID for deleted object. " +
+ "DN: {Dn}. The accesslog entry may not contain reqEntryUUID or reqOld entryUUID.", reqDn);
+ return null;
+ }
+
+ var importObject = new ConnectedSystemImportObject
+ {
+ ObjectType = objectClassName,
+ ChangeType = ObjectChangeType.Deleted,
+ };
+
+ // Add entryUUID as an attribute so the import processor can match to the existing CSO
+ var externalIdAttribute = _connectedSystem.ObjectTypes
+ .First(ot => ot.Name.Equals(objectClassName, StringComparison.OrdinalIgnoreCase))
+ .Attributes.FirstOrDefault(a => a.IsExternalId);
+
+ if (externalIdAttribute != null)
+ {
+ importObject.Attributes.Add(new ConnectedSystemImportObjectAttribute
+ {
+ Name = externalIdAttribute.Name,
+ StringValues = [entryUuid]
+ });
+ }
+
+ // Add the DN as the secondary external ID (distinguishedName)
+ var reqDnValue = LdapConnectorUtilities.GetEntryAttributeStringValue(accesslogEntry, "reqDN");
+ if (!string.IsNullOrEmpty(reqDnValue))
+ {
+ importObject.Attributes.Add(new ConnectedSystemImportObjectAttribute
+ {
+ Name = "distinguishedName",
+ StringValues = [reqDnValue]
+ });
+ }
+
+ _logger.Debug("BuildDeleteImportObjectFromAccesslog: Built delete import for {ObjectType} with entryUUID {Uuid}, DN: {Dn}",
+ objectClassName, entryUuid, reqDnValue);
+ return importObject;
+ }
+
private ConnectedSystemImportObject? GetObjectByDn(string dn, ObjectChangeType changeType)
{
if (_connectedSystem.ObjectTypes == null)
@@ -922,7 +1651,18 @@ private IEnumerable ConvertLdapResults(SearchResult
case AttributeDataType.Reference:
var referenceValues = LdapConnectorUtilities.GetEntryAttributeStringValues(searchResult, attributeName);
if (referenceValues is { Count: > 0 })
- importObjectAttribute.ReferenceValues.AddRange(referenceValues);
+ {
+ // Filter out the placeholder member DN so it never enters the metaverse.
+ // The placeholder is injected by the connector during export to satisfy the
+ // groupOfNames MUST member constraint — it should be invisible to JIM.
+ var filteredValues = referenceValues.Where(v =>
+ !_placeholderMemberDn.Equals(v, StringComparison.OrdinalIgnoreCase)).ToList();
+ if (filteredValues.Count > 0)
+ importObjectAttribute.ReferenceValues.AddRange(filteredValues);
+ else if (referenceValues.Count > filteredValues.Count)
+ _logger.Debug("LdapConnectorImport: Filtered placeholder member '{Placeholder}' from attribute '{Attr}' on '{Dn}'",
+ _placeholderMemberDn, attributeName, searchResult.DistinguishedName);
+ }
break;
case AttributeDataType.NotSet:
default:
@@ -932,6 +1672,22 @@ private IEnumerable ConvertLdapResults(SearchResult
importObject.Attributes.Add(importObjectAttribute);
}
+ // Synthesise distinguishedName for directories that don't return it as an attribute.
+ // OpenLDAP (and most RFC-compliant directories) expose the DN as the entry's DistinguishedName
+ // property, not as a searchable/importable attribute. The connector schema synthesises
+ // distinguishedName as an attribute (for DN-based provisioning), so we need to populate it
+ // from the entry's DN during import for export confirmation to match correctly.
+ if (!importObject.Attributes.Any(a => a.Name.Equals("distinguishedName", StringComparison.OrdinalIgnoreCase))
+ && objectType.Attributes.Any(a => a.Name.Equals("distinguishedName", StringComparison.OrdinalIgnoreCase) && a.Selected))
+ {
+ importObject.Attributes.Add(new ConnectedSystemImportObjectAttribute
+ {
+ Name = "distinguishedName",
+ Type = AttributeDataType.Text,
+ StringValues = { searchResult.DistinguishedName }
+ });
+ }
+
importObjects.Add(importObject);
}
diff --git a/src/JIM.Connectors/LDAP/LdapConnectorPartitions.cs b/src/JIM.Connectors/LDAP/LdapConnectorPartitions.cs
index e25d2b1cf..135b4d159 100644
--- a/src/JIM.Connectors/LDAP/LdapConnectorPartitions.cs
+++ b/src/JIM.Connectors/LDAP/LdapConnectorPartitions.cs
@@ -8,15 +8,28 @@ internal class LdapConnectorPartitions
{
private readonly LdapConnection _connection;
private readonly ILogger _logger;
+ private readonly LdapDirectoryType _directoryType;
private string _partitionsDn = null!;
- internal LdapConnectorPartitions(LdapConnection ldapConnection, ILogger logger)
+ internal LdapConnectorPartitions(LdapConnection ldapConnection, ILogger logger, LdapDirectoryType directoryType)
{
_connection = ldapConnection;
_logger = logger;
+ _directoryType = directoryType;
}
internal async Task> GetPartitionsAsync(bool skipHiddenPartitions = true)
+ {
+ return _directoryType is LdapDirectoryType.ActiveDirectory or LdapDirectoryType.SambaAD
+ ? await GetActiveDirectoryPartitionsAsync(skipHiddenPartitions)
+ : await GetNamingContextPartitionsAsync();
+ }
+
+ ///
+ /// Discovers partitions using the AD-specific crossRef/systemFlags mechanism.
+ /// Works for both Microsoft AD and Samba AD.
+ ///
+ private async Task> GetActiveDirectoryPartitionsAsync(bool skipHiddenPartitions)
{
return await Task.Run(() =>
{
@@ -32,13 +45,13 @@ internal async Task> GetPartitionsAsync(bool skipHidden
var response = (SearchResponse)_connection.SendRequest(request);
var partitions = new List();
- _logger.Debug("GetPartitionsAsync: Found {Count} crossRef entries to process (skipHiddenPartitions={SkipHidden})",
+ _logger.Debug("GetActiveDirectoryPartitionsAsync: Found {Count} crossRef entries to process (skipHiddenPartitions={SkipHidden})",
response.Entries.Count, skipHiddenPartitions);
foreach (SearchResultEntry entry in response.Entries)
{
- // ncName is the actual naming context DN (e.g., "DC=subatomic,DC=local")
- // entry.DistinguishedName is the crossRef object DN (e.g., "CN=subatomic,CN=Partitions,CN=Configuration,DC=subatomic,DC=local")
+ // ncName is the actual naming context DN (e.g., "DC=panoply,DC=local")
+ // entry.DistinguishedName is the crossRef object DN (e.g., "CN=panoply,CN=Partitions,CN=Configuration,DC=panoply,DC=local")
// We use ncName as the Id because container DNs end with the naming context, not the crossRef DN
var ncName = LdapConnectorUtilities.GetEntryAttributeStringValue(entry, "ncname") ?? entry.DistinguishedName;
var systemFlags = LdapConnectorUtilities.GetEntryAttributeStringValue(entry, "systemflags");
@@ -54,13 +67,13 @@ internal async Task> GetPartitionsAsync(bool skipHidden
!ncName.Contains("ForestDnsZones", StringComparison.OrdinalIgnoreCase);
var isHidden = !isDomainPartitionByFlags && !isDomainPartitionByName;
- _logger.Debug("GetPartitionsAsync: Partition '{Name}' - systemFlags={SystemFlags}, isDomainByFlags={ByFlags}, isDomainByName={ByName}, isHidden={IsHidden}",
+ _logger.Debug("GetActiveDirectoryPartitionsAsync: Partition '{Name}' - systemFlags={SystemFlags}, isDomainByFlags={ByFlags}, isDomainByName={ByName}, isHidden={IsHidden}",
ncName, systemFlags ?? "(null)", isDomainPartitionByFlags, isDomainPartitionByName, isHidden);
// Skip hidden partitions early if configured - this avoids expensive LDAP subtree searches
if (skipHiddenPartitions && isHidden)
{
- _logger.Debug("GetPartitionsAsync: Skipping hidden partition '{Name}'", ncName);
+ _logger.Debug("GetActiveDirectoryPartitionsAsync: Skipping hidden partition '{Name}'", ncName);
continue;
}
@@ -76,7 +89,7 @@ internal async Task> GetPartitionsAsync(bool skipHidden
partition.Containers = GetPartitionContainers(partition);
partitionStopwatch.Stop();
- _logger.Debug("GetPartitionsAsync: Partition '{Name}' (Hidden={Hidden}) - {ContainerCount} containers retrieved in {ElapsedMs}ms",
+ _logger.Debug("GetActiveDirectoryPartitionsAsync: Partition '{Name}' (Hidden={Hidden}) - {ContainerCount} containers retrieved in {ElapsedMs}ms",
partition.Name, partition.Hidden, partition.Containers.Count, partitionStopwatch.ElapsedMilliseconds);
// only return partitions that have containers. Discard the rest.
@@ -85,7 +98,56 @@ internal async Task> GetPartitionsAsync(bool skipHidden
}
totalStopwatch.Stop();
- _logger.Information("GetPartitionsAsync: Completed - {PartitionCount} partitions with containers in {ElapsedMs}ms total",
+ _logger.Information("GetActiveDirectoryPartitionsAsync: Completed - {PartitionCount} partitions with containers in {ElapsedMs}ms total",
+ partitions.Count, totalStopwatch.ElapsedMilliseconds);
+
+ return partitions;
+ });
+ }
+
+ ///
+ /// Discovers partitions using the standard LDAP namingContexts rootDSE attribute (RFC 4512).
+ /// Works for OpenLDAP, 389 Directory Server, and other standards-compliant directories.
+ /// All naming contexts are returned as non-hidden partitions.
+ ///
+ private async Task> GetNamingContextPartitionsAsync()
+ {
+ return await Task.Run(() =>
+ {
+ var totalStopwatch = System.Diagnostics.Stopwatch.StartNew();
+
+ var namingContexts = GetNamingContexts();
+ if (namingContexts == null || namingContexts.Count == 0)
+ throw new Exception("Couldn't get any namingContexts from rootDSE. The directory may not expose partition information.");
+
+ _logger.Debug("GetNamingContextPartitionsAsync: Found {Count} naming contexts", namingContexts.Count);
+
+ var partitions = new List();
+
+ foreach (var namingContext in namingContexts)
+ {
+ var partitionStopwatch = System.Diagnostics.Stopwatch.StartNew();
+
+ var partition = new ConnectorPartition
+ {
+ Id = namingContext,
+ Name = namingContext,
+ Hidden = false,
+ };
+
+ partition.Containers = GetPartitionContainers(partition);
+ partitionStopwatch.Stop();
+
+ _logger.Debug("GetNamingContextPartitionsAsync: Partition '{Name}' - {ContainerCount} containers retrieved in {ElapsedMs}ms",
+ partition.Name, partition.Containers.Count, partitionStopwatch.ElapsedMilliseconds);
+
+ // only return partitions that have containers. Discard the rest.
+ if (partition.Containers.Count > 0)
+ partitions.Add(partition);
+ }
+
+ totalStopwatch.Stop();
+ _logger.Information("GetNamingContextPartitionsAsync: Completed - {PartitionCount} partitions with containers in {ElapsedMs}ms total",
partitions.Count, totalStopwatch.ElapsedMilliseconds);
return partitions;
@@ -195,6 +257,32 @@ private static void SortChildrenRecursively(List containers)
///
internal record ContainerEntry(string DistinguishedName, string Name);
+ ///
+ /// Retrieves the namingContexts attribute from the rootDSE (RFC 4512).
+ /// This is the standard mechanism for discovering partitions on non-AD directories.
+ ///
+ private List? GetNamingContexts()
+ {
+ var request = new SearchRequest { Scope = SearchScope.Base };
+ request.Attributes.Add("namingContexts");
+ var response = (SearchResponse)_connection.SendRequest(request);
+
+ if (response.ResultCode != ResultCode.Success)
+ {
+ _logger.Warning("GetNamingContexts: No success. Result code: {ResultCode}", response.ResultCode);
+ return null;
+ }
+
+ if (response.Entries.Count == 0)
+ {
+ _logger.Warning("GetNamingContexts: Didn't get any results from rootDSE!");
+ return null;
+ }
+
+ var entry = response.Entries[0];
+ return LdapConnectorUtilities.GetEntryAttributeStringValues(entry, "namingContexts");
+ }
+
private string? GetConfigurationNamingContext()
{
// get the configuration naming context from an attribute on the rootDSE
diff --git a/src/JIM.Connectors/LDAP/LdapConnectorRootDse.cs b/src/JIM.Connectors/LDAP/LdapConnectorRootDse.cs
index f2c8e1087..2b6bd18f5 100644
--- a/src/JIM.Connectors/LDAP/LdapConnectorRootDse.cs
+++ b/src/JIM.Connectors/LDAP/LdapConnectorRootDse.cs
@@ -1,4 +1,6 @@
-namespace JIM.Connectors.LDAP;
+using JIM.Models.Core;
+
+namespace JIM.Connectors.LDAP;
///
/// Holder of key synchronisation-related information about an LDAP directory from its rootDSE object.
@@ -18,26 +20,88 @@ internal class LdapConnectorRootDse
public long? HighestCommittedUsn { get; set; }
///
- /// For changelog-based directories (e.g., OpenLDAP, Oracle Directory): The last change number processed.
- /// Used for delta imports - we query the changelog for entries > this value.
+ /// For changelog-based directories (e.g., Oracle Directory): The last change number processed.
+ /// Used for delta imports — we query cn=changelog for entries with changeNumber > this value.
///
public int? LastChangeNumber { get; set; }
///
- /// Indicates if the connected directory is Active Directory (AD-DS or AD-LDS).
- /// Determines which delta import strategy to use.
+ /// For OpenLDAP with accesslog overlay: The reqStart timestamp of the last processed entry.
+ /// Used for delta imports — we query cn=accesslog for entries with reqStart > this value.
+ /// Format: Generalised time (e.g., "20260326183000.000000Z").
///
- public bool IsActiveDirectory { get; set; }
+ public string? LastAccesslogTimestamp { get; set; }
///
- /// The vendor name of the directory server (e.g., "Samba Team", "Microsoft").
- /// Used for capability detection when directories claim AD compatibility but have behavioural differences.
+ /// The detected directory server type, determined from rootDSE capabilities during connection.
+ /// Drives all directory-specific behaviour: schema discovery, external ID, delta strategy, etc.
+ ///
+ public LdapDirectoryType DirectoryType { get; set; } = LdapDirectoryType.Generic;
+
+ ///
+ /// The vendor name of the directory server (e.g., "Samba Team", "Microsoft", "OpenLDAP").
+ /// Retained for logging and diagnostics.
///
public string? VendorName { get; set; }
+ // -----------------------------------------------------------------------
+ // Computed properties — centralised directory-type-specific behaviour
+ // -----------------------------------------------------------------------
+
+ ///
+ /// The attribute name used as the unique, immutable external identifier for directory objects.
+ /// AD/Samba AD use objectGUID (binary GUID in Microsoft byte order); OpenLDAP uses entryUUID (RFC 4530, string format).
+ ///
+ public string ExternalIdAttributeName => DirectoryType switch
+ {
+ LdapDirectoryType.ActiveDirectory => "objectGUID",
+ LdapDirectoryType.SambaAD => "objectGUID",
+ LdapDirectoryType.OpenLDAP => "entryUUID",
+ LdapDirectoryType.Generic => "entryUUID",
+ _ => "entryUUID"
+ };
+
+ ///
+ /// The data type of the external ID attribute in JIM's attribute model.
+ /// AD/Samba AD objectGUID is a binary GUID; OpenLDAP entryUUID is a string representation of a UUID.
+ ///
+ public AttributeDataType ExternalIdDataType => DirectoryType switch
+ {
+ LdapDirectoryType.ActiveDirectory => AttributeDataType.Guid,
+ LdapDirectoryType.SambaAD => AttributeDataType.Guid,
+ LdapDirectoryType.OpenLDAP => AttributeDataType.Text,
+ LdapDirectoryType.Generic => AttributeDataType.Text,
+ _ => AttributeDataType.Text
+ };
+
+ ///
+ /// Whether delta imports should use USN-based change tracking (AD/Samba AD).
+ ///
+ public bool UseUsnDeltaImport => DirectoryType is LdapDirectoryType.ActiveDirectory or LdapDirectoryType.SambaAD;
+
+ ///
+ /// Whether delta imports should use the OpenLDAP accesslog overlay (cn=accesslog with reqStart timestamps).
+ /// Falls back to standard changelog (cn=changelog with changeNumber) for Generic directories.
+ ///
+ public bool UseAccesslogDeltaImport => DirectoryType is LdapDirectoryType.OpenLDAP;
+
+ ///
+ /// Whether the directory's SAM layer enforces single-valued semantics on certain multi-valued schema attributes
+ /// (e.g., 'description' on user/group objects). Applies to both Microsoft AD and Samba AD.
+ ///
+ public bool EnforcesSamSingleValuedRules => DirectoryType is LdapDirectoryType.ActiveDirectory or LdapDirectoryType.SambaAD;
+
///
- /// Indicates if the directory supports paged search results.
- /// True AD supports paging; Samba AD claims support but returns duplicate results, so we disable it.
+ /// Whether the directory supports paged search results.
+ /// Microsoft AD supports paging; Samba AD claims support but returns duplicate results.
+ /// OpenLDAP supports paging via Simple Paged Results control.
///
- public bool SupportsPaging { get; set; } = true;
-}
\ No newline at end of file
+ public bool SupportsPaging => DirectoryType switch
+ {
+ LdapDirectoryType.ActiveDirectory => true,
+ LdapDirectoryType.SambaAD => false,
+ LdapDirectoryType.OpenLDAP => true,
+ LdapDirectoryType.Generic => true,
+ _ => true
+ };
+}
diff --git a/src/JIM.Connectors/LDAP/LdapConnectorSchema.cs b/src/JIM.Connectors/LDAP/LdapConnectorSchema.cs
index 74d6b7cb0..7f5d36c4d 100644
--- a/src/JIM.Connectors/LDAP/LdapConnectorSchema.cs
+++ b/src/JIM.Connectors/LDAP/LdapConnectorSchema.cs
@@ -1,4 +1,4 @@
-using JIM.Models.Core;
+using JIM.Models.Core;
using JIM.Models.Staging;
using Serilog;
using System.DirectoryServices.Protocols;
@@ -9,18 +9,35 @@ internal class LdapConnectorSchema
private readonly LdapConnection _connection;
private readonly ILogger _logger;
private readonly LdapConnectorRootDse _rootDse;
+ private readonly bool _includeAuxiliaryClasses;
private readonly ConnectorSchema _schema;
private string _schemaNamingContext = null!;
- internal LdapConnectorSchema(LdapConnection ldapConnection, ILogger logger, LdapConnectorRootDse rootDse)
+ internal LdapConnectorSchema(LdapConnection ldapConnection, ILogger logger, LdapConnectorRootDse rootDse, bool includeAuxiliaryClasses = false)
{
_connection = ldapConnection;
_logger = logger;
_rootDse = rootDse;
+ _includeAuxiliaryClasses = includeAuxiliaryClasses;
_schema = new ConnectorSchema();
}
internal async Task GetSchemaAsync()
+ {
+ return _rootDse.DirectoryType is LdapDirectoryType.ActiveDirectory or LdapDirectoryType.SambaAD
+ ? await GetActiveDirectorySchemaAsync()
+ : await GetRfcSchemaAsync();
+ }
+
+ // -----------------------------------------------------------------------
+ // Active Directory schema discovery (classSchema / attributeSchema)
+ // -----------------------------------------------------------------------
+
+ ///
+ /// Discovers schema using the AD-specific classSchema/attributeSchema partition.
+ /// Works for both Microsoft AD and Samba AD.
+ ///
+ private async Task GetActiveDirectorySchemaAsync()
{
return await Task.Run(() =>
{
@@ -30,8 +47,12 @@ internal async Task GetSchemaAsync()
throw new Exception($"Couldn't get schema naming context from rootDSE.");
_schemaNamingContext = schemaNamingContext;
- // query: classes, structural, don't return hidden by default classes, exclude defunct classes
- var filter = "(&(objectClass=classSchema)(objectClassCategory=1)(defaultHidingValue=FALSE)(!(isDefunct=TRUE)))";
+ // query: classes, structural (and optionally auxiliary), don't return hidden by default classes, exclude defunct classes
+ // objectClassCategory: 1 = structural, 3 = auxiliary
+ var classFilter = _includeAuxiliaryClasses
+ ? "(|(objectClassCategory=1)(objectClassCategory=3))"
+ : "(objectClassCategory=1)";
+ var filter = $"(&(objectClass=classSchema){classFilter}(defaultHidingValue=FALSE)(!(isDefunct=TRUE)))";
var request = new SearchRequest(_schemaNamingContext, filter, SearchScope.Subtree);
var response = (SearchResponse)_connection.SendRequest(request);
@@ -50,25 +71,7 @@ internal async Task GetSchemaAsync()
// now go and work out which attributes the object type has and add them to the object type
if (AddObjectTypeAttributes(objectType))
{
- // make a recommendation on what unique identifier attribute(s) to use
- // for AD/AD LDS:
- var objectGuidSchemaAttribute = objectType.Attributes.Single(a => a.Name.Equals("objectguid", StringComparison.OrdinalIgnoreCase));
- objectType.RecommendedExternalIdAttribute = objectGuidSchemaAttribute;
-
- // say what the secondary external identifier needs to be for LDAP systems
- var dnSchemaAttribute = objectType.Attributes.Single(a => a.Name.Equals("distinguishedname", StringComparison.OrdinalIgnoreCase));
- objectType.RecommendedSecondaryExternalIdAttribute = dnSchemaAttribute;
-
- // override the data type for distinguishedName, we want to handle it as a string, not a reference type
- // we do this as a DN attribute on an object cannot be a reference to itself. that would make no sense.
- dnSchemaAttribute.Type = AttributeDataType.Text;
-
- // override writability for distinguishedName: AD marks it as systemOnly but the LDAP connector
- // needs it writable because the DN is provided by the client in Add requests to specify where
- // the object is created. It's not written as an attribute — it's the target of the LDAP Add operation.
- dnSchemaAttribute.Writability = AttributeWritability.Writable;
-
- // object type looks good to go, add it to the schema
+ ApplyExternalIdRecommendations(objectType);
_schema.ObjectTypes.Add(objectType);
}
}
@@ -164,7 +167,7 @@ private void AddAttributesFromSchemaProperty(SearchResultEntry objectClassEntry,
{
if (objectType.Attributes.All(q => q.Name != attributeName))
{
- var attr = GetSchemaAttribute(attributeName, objectClassName, required, objectType.Name);
+ var attr = GetAdSchemaAttribute(attributeName, objectClassName, required, objectType.Name);
if (attr != null)
objectType.Attributes.Add(attr);
}
@@ -172,10 +175,10 @@ private void AddAttributesFromSchemaProperty(SearchResultEntry objectClassEntry,
}
///
- /// Looks up the schema entry for an attribute and returns a ConnectorSchemaAttribute with full metadata.
+ /// Looks up the AD schema entry for an attribute and returns a ConnectorSchemaAttribute with full metadata.
/// Returns null if the attribute is defunct (should be excluded from the schema entirely).
///
- private ConnectorSchemaAttribute? GetSchemaAttribute(string attributeName, string objectClass, bool required, string objectTypeName)
+ private ConnectorSchemaAttribute? GetAdSchemaAttribute(string attributeName, string objectClass, bool required, string objectTypeName)
{
var attributeEntry = LdapConnectorUtilities.GetSchemaEntry(_connection, _schemaNamingContext, $"(ldapdisplayname={attributeName})") ??
throw new Exception($"Couldn't retrieve schema attribute: {attributeName}");
@@ -184,7 +187,7 @@ private void AddAttributesFromSchemaProperty(SearchResultEntry objectClassEntry,
var isDefunctRawValue = LdapConnectorUtilities.GetEntryAttributeStringValue(attributeEntry, "isdefunct");
if (bool.TryParse(isDefunctRawValue, out var isDefunct) && isDefunct)
{
- _logger.Debug("GetSchemaAttribute: Skipping defunct attribute '{AttributeName}'.", attributeName);
+ _logger.Debug("GetAdSchemaAttribute: Skipping defunct attribute '{AttributeName}'.", attributeName);
return null;
}
@@ -197,7 +200,7 @@ private void AddAttributesFromSchemaProperty(SearchResultEntry objectClassEntry,
if (!bool.TryParse(isSingleValuedRawValue, out var isSingleValued))
{
isSingleValued = true;
- _logger.Verbose("GetSchemaAttribute: Could not establish if SVA/MVA for attribute {AttributeName}. Assuming SVA. Raw value: '{RawValue}'", attributeName, isSingleValuedRawValue);
+ _logger.Verbose("GetAdSchemaAttribute: Could not establish if SVA/MVA for attribute {AttributeName}. Assuming SVA. Raw value: '{RawValue}'", attributeName, isSingleValuedRawValue);
}
var attributePlurality = isSingleValued ? AttributePlurality.SingleValued : AttributePlurality.MultiValued;
@@ -205,10 +208,10 @@ private void AddAttributesFromSchemaProperty(SearchResultEntry objectClassEntry,
// Active Directory SAM layer override: certain attributes are declared as multi-valued in the
// LDAP schema but the SAM layer enforces single-valued semantics on security principals.
// Override the plurality to match actual runtime behaviour so mapping validation works correctly.
- if (!isSingleValued && LdapConnectorUtilities.ShouldOverridePluralityToSingleValued(attributeName, objectTypeName, _rootDse.IsActiveDirectory))
+ if (!isSingleValued && LdapConnectorUtilities.ShouldOverridePluralityToSingleValued(attributeName, objectTypeName, _rootDse.DirectoryType))
{
attributePlurality = AttributePlurality.SingleValued;
- _logger.Debug("GetSchemaAttribute: Overriding '{AttributeName}' from multi-valued to single-valued on object type '{ObjectType}' — " +
+ _logger.Debug("GetAdSchemaAttribute: Overriding '{AttributeName}' from multi-valued to single-valued on object type '{ObjectType}' — " +
"Active Directory SAM layer enforces single-valued semantics on this attribute for security principals.",
attributeName, objectTypeName);
}
@@ -226,14 +229,15 @@ private void AddAttributesFromSchemaProperty(SearchResultEntry objectClassEntry,
catch (InvalidDataException)
{
// Unsupported omSyntax - default to Text
- _logger.Warning("GetSchemaAttribute: Unsupported omSyntax {OmSyntax} for attribute {AttributeName}. Defaulting to Text.", omSyntax.Value, attributeName);
+ _logger.Warning("GetAdSchemaAttribute: Unsupported omSyntax {OmSyntax} for attribute {AttributeName}. Defaulting to Text.", omSyntax.Value, attributeName);
}
}
- // handle exceptions:
- // the objectGUID is typed as a string in the schema, but the byte-array returned does not decode to a string, but does to a Guid. go figure.
- if (attributeName.Equals("objectguid", StringComparison.OrdinalIgnoreCase))
- attributeDataType = AttributeDataType.Guid;
+ // Override the data type for the external ID attribute based on directory type.
+ // AD's objectGUID is declared as octet string in the schema but actually returns a binary GUID.
+ // OpenLDAP's entryUUID is a string-formatted UUID (RFC 4530).
+ if (attributeName.Equals(_rootDse.ExternalIdAttributeName, StringComparison.OrdinalIgnoreCase))
+ attributeDataType = _rootDse.ExternalIdDataType;
// determine writability from schema metadata: systemOnly, systemFlags (constructed bit), and linkID (back-links)
var systemOnlyRawValue = LdapConnectorUtilities.GetEntryAttributeStringValue(attributeEntry, "systemonly");
@@ -274,4 +278,323 @@ private void AddAttributesFromSchemaProperty(SearchResultEntry objectClassEntry,
var entry = response.Entries[0];
return LdapConnectorUtilities.GetEntryAttributeStringValue(entry, "schemaNamingContext");
}
-}
\ No newline at end of file
+
+ // -----------------------------------------------------------------------
+ // RFC 4512 schema discovery (subschema subentry)
+ // -----------------------------------------------------------------------
+
+ ///
+ /// Discovers schema using the RFC 4512 subschema subentry mechanism.
+ /// Works for OpenLDAP, 389 Directory Server, and other standards-compliant directories.
+ /// Queries the subschemaSubentry for objectClasses and attributeTypes operational attributes
+ /// and parses the RFC 4512 description strings.
+ ///
+ private async Task GetRfcSchemaAsync()
+ {
+ return await Task.Run(() =>
+ {
+ var totalStopwatch = System.Diagnostics.Stopwatch.StartNew();
+
+ // Step 1: Get the subschema subentry DN from rootDSE
+ var subschemaDn = GetSubschemaSubentryDn();
+ if (string.IsNullOrEmpty(subschemaDn))
+ throw new Exception("Couldn't get subschemaSubentry DN from rootDSE. Schema discovery requires a subschema subentry.");
+
+ _logger.Debug("GetRfcSchemaAsync: Querying subschema subentry at '{SubschemaDn}'", subschemaDn);
+
+ // Step 2: Query the subschema subentry for objectClasses and attributeTypes
+ var request = new SearchRequest(subschemaDn, "(objectClass=subschema)", SearchScope.Base);
+ request.Attributes.AddRange(["objectClasses", "attributeTypes"]);
+ var response = (SearchResponse)_connection.SendRequest(request);
+
+ if (response.ResultCode != ResultCode.Success || response.Entries.Count == 0)
+ throw new Exception($"Failed to query subschema subentry at '{subschemaDn}'. Result code: {response.ResultCode}");
+
+ var subschemaEntry = response.Entries[0];
+
+ // Step 3: Parse all attribute type definitions into a lookup dictionary
+ var attributeTypeDefs = ParseAllAttributeTypes(subschemaEntry);
+ _logger.Debug("GetRfcSchemaAsync: Parsed {Count} attribute type definitions", attributeTypeDefs.Count);
+
+ // Step 4: Parse all object class definitions
+ var objectClassDefs = ParseAllObjectClasses(subschemaEntry);
+ _logger.Debug("GetRfcSchemaAsync: Parsed {Count} object class definitions", objectClassDefs.Count);
+
+ // Step 5: Build the connector schema from structural (and optionally auxiliary) object classes
+ foreach (var objectClassDef in objectClassDefs.Values)
+ {
+ // Only expose structural classes by default — these are the ones that can have objects instantiated.
+ // When _includeAuxiliaryClasses is enabled, also include auxiliary classes for directories
+ // where objects may use auxiliary classes as their primary structural class.
+ var isStructural = objectClassDef.Kind == Rfc4512ObjectClassKind.Structural;
+ var isAuxiliary = objectClassDef.Kind == Rfc4512ObjectClassKind.Auxiliary;
+ if (!isStructural && !(isAuxiliary && _includeAuxiliaryClasses))
+ continue;
+
+ var objectType = new ConnectorSchemaObjectType(objectClassDef.Name!);
+
+ // Collect attributes by walking the class hierarchy (SUP chain)
+ var allMust = new HashSet(StringComparer.OrdinalIgnoreCase);
+ var allMay = new HashSet(StringComparer.OrdinalIgnoreCase);
+ CollectClassAttributes(objectClassDef, objectClassDefs, allMust, allMay);
+
+ // Build ConnectorSchemaAttribute for each attribute
+ foreach (var attrName in allMust)
+ {
+ var attr = BuildRfcSchemaAttribute(attrName, attributeTypeDefs, objectClassDef.Name!, required: true, objectType.Name);
+ if (attr != null && objectType.Attributes.All(a => !a.Name.Equals(attr.Name, StringComparison.OrdinalIgnoreCase)))
+ objectType.Attributes.Add(attr);
+ }
+
+ foreach (var attrName in allMay)
+ {
+ // If the attribute is already in the MUST set, don't add it again as MAY
+ if (allMust.Contains(attrName))
+ continue;
+
+ var attr = BuildRfcSchemaAttribute(attrName, attributeTypeDefs, objectClassDef.Name!, required: false, objectType.Name);
+ if (attr != null && objectType.Attributes.All(a => !a.Name.Equals(attr.Name, StringComparison.OrdinalIgnoreCase)))
+ objectType.Attributes.Add(attr);
+ }
+
+ // Synthesise operational attributes that are not part of any object class's
+ // MUST/MAY but are needed by the connector for identity management.
+
+ // External ID attribute (e.g., entryUUID for OpenLDAP) — global operational attribute
+ // not listed in any class hierarchy. Required for JIM to uniquely identify objects.
+ var externalIdAttrName = _rootDse.ExternalIdAttributeName;
+ if (objectType.Attributes.All(a => !a.Name.Equals(externalIdAttrName, StringComparison.OrdinalIgnoreCase)))
+ {
+ objectType.Attributes.Add(new ConnectorSchemaAttribute(
+ externalIdAttrName, _rootDse.ExternalIdDataType, AttributePlurality.SingleValued,
+ required: false, className: objectClassDef.Name!, writability: AttributeWritability.ReadOnly)
+ {
+ Description = "System-assigned unique identifier for this entry"
+ });
+ }
+
+ // distinguishedName — OpenLDAP doesn't expose it in the subschema because it's
+ // the entry's DN, not a stored attribute. The LDAP connector needs it as a writable
+ // Text attribute for DN-based provisioning expressions.
+ if (objectType.Attributes.All(a => !a.Name.Equals("distinguishedName", StringComparison.OrdinalIgnoreCase)))
+ {
+ objectType.Attributes.Add(new ConnectorSchemaAttribute(
+ "distinguishedName", AttributeDataType.Text, AttributePlurality.SingleValued,
+ required: false, className: objectClassDef.Name!, writability: AttributeWritability.Writable)
+ {
+ Description = "The full distinguished name (DN) of this entry"
+ });
+ }
+
+ objectType.Attributes = objectType.Attributes.OrderBy(a => a.Name).ToList();
+
+ ApplyExternalIdRecommendations(objectType);
+ _schema.ObjectTypes.Add(objectType);
+ }
+
+ totalStopwatch.Stop();
+ _logger.Information("GetRfcSchemaAsync: Completed — {ObjectTypeCount} object types discovered in {ElapsedMs}ms",
+ _schema.ObjectTypes.Count, totalStopwatch.ElapsedMilliseconds);
+
+ return _schema;
+ });
+ }
+
+ ///
+ /// Recursively collects MUST and MAY attributes from an object class and all its superiors.
+ ///
+ private static void CollectClassAttributes(
+ Rfc4512ObjectClassDescription classDef,
+ Dictionary allClasses,
+ HashSet mustAttributes,
+ HashSet mayAttributes)
+ {
+ foreach (var attr in classDef.MustAttributes)
+ mustAttributes.Add(attr);
+ foreach (var attr in classDef.MayAttributes)
+ mayAttributes.Add(attr);
+
+ // Walk the superclass chain
+ if (classDef.SuperiorName != null &&
+ allClasses.TryGetValue(classDef.SuperiorName, out var superClass))
+ {
+ CollectClassAttributes(superClass, allClasses, mustAttributes, mayAttributes);
+ }
+ }
+
+ ///
+ /// Builds a ConnectorSchemaAttribute from an RFC 4512 attribute type definition.
+ /// Resolves the SYNTAX OID (walking the SUP chain if needed) and maps to JIM's data types.
+ ///
+ private ConnectorSchemaAttribute? BuildRfcSchemaAttribute(
+ string attributeName,
+ Dictionary attributeTypeDefs,
+ string objectClassName,
+ bool required,
+ string objectTypeName)
+ {
+ if (!attributeTypeDefs.TryGetValue(attributeName, out var attrDef))
+ {
+ _logger.Warning("GetRfcSchemaAsync: Attribute '{AttributeName}' referenced by object class '{ObjectClass}' " +
+ "not found in schema. Skipping.", attributeName, objectClassName);
+ return null;
+ }
+
+ // Resolve SYNTAX OID — walk the SUP chain if the attribute inherits its syntax
+ var syntaxOid = ResolveSyntaxOid(attrDef, attributeTypeDefs);
+ var attributeDataType = Rfc4512SchemaParser.GetRfcAttributeDataType(syntaxOid);
+
+ // Override data type for external ID attribute
+ if (attributeName.Equals(_rootDse.ExternalIdAttributeName, StringComparison.OrdinalIgnoreCase))
+ attributeDataType = _rootDse.ExternalIdDataType;
+
+ var plurality = attrDef.IsSingleValued ? AttributePlurality.SingleValued : AttributePlurality.MultiValued;
+ var writability = Rfc4512SchemaParser.DetermineRfcAttributeWritability(attrDef.Usage, attrDef.IsNoUserModification);
+
+ var attribute = new ConnectorSchemaAttribute(attributeName, attributeDataType, plurality, required, objectClassName, writability);
+
+ if (!string.IsNullOrEmpty(attrDef.Description))
+ attribute.Description = attrDef.Description;
+
+ return attribute;
+ }
+
+ ///
+ /// Resolves the SYNTAX OID for an attribute type, walking the SUP (superior) chain
+ /// if the attribute inherits its syntax from a parent attribute type.
+ ///
+ private static string? ResolveSyntaxOid(
+ Rfc4512AttributeTypeDescription attrDef,
+ Dictionary allAttributes)
+ {
+ // Direct SYNTAX specified — use it
+ if (attrDef.SyntaxOid != null)
+ return attrDef.SyntaxOid;
+
+ // Walk the SUP chain to find inherited SYNTAX (max 10 levels to prevent infinite loops)
+ var current = attrDef;
+ for (var depth = 0; depth < 10; depth++)
+ {
+ if (current.SuperiorName == null)
+ break;
+
+ if (!allAttributes.TryGetValue(current.SuperiorName, out var superAttr))
+ break;
+
+ if (superAttr.SyntaxOid != null)
+ return superAttr.SyntaxOid;
+
+ current = superAttr;
+ }
+
+ return null;
+ }
+
+ ///
+ /// Parses all attributeTypes values from the subschema entry into a name-keyed dictionary.
+ ///
+ private Dictionary ParseAllAttributeTypes(SearchResultEntry subschemaEntry)
+ {
+ var result = new Dictionary(StringComparer.OrdinalIgnoreCase);
+
+ if (!subschemaEntry.Attributes.Contains("attributeTypes"))
+ return result;
+
+ foreach (string definition in subschemaEntry.Attributes["attributeTypes"].GetValues(typeof(string)))
+ {
+ var parsed = Rfc4512SchemaParser.ParseAttributeTypeDescription(definition);
+ if (parsed?.Name != null && !result.ContainsKey(parsed.Name))
+ result[parsed.Name] = parsed;
+ }
+
+ return result;
+ }
+
+ ///
+ /// Parses all objectClasses values from the subschema entry into a name-keyed dictionary.
+ ///
+ private Dictionary ParseAllObjectClasses(SearchResultEntry subschemaEntry)
+ {
+ var result = new Dictionary(StringComparer.OrdinalIgnoreCase);
+
+ if (!subschemaEntry.Attributes.Contains("objectClasses"))
+ return result;
+
+ foreach (string definition in subschemaEntry.Attributes["objectClasses"].GetValues(typeof(string)))
+ {
+ var parsed = Rfc4512SchemaParser.ParseObjectClassDescription(definition);
+ if (parsed?.Name != null && !result.ContainsKey(parsed.Name))
+ result[parsed.Name] = parsed;
+ }
+
+ return result;
+ }
+
+ ///
+ /// Gets the subschemaSubentry DN from the rootDSE (RFC 4512 § 5.1).
+ ///
+ private string? GetSubschemaSubentryDn()
+ {
+ var request = new SearchRequest { Scope = SearchScope.Base };
+ request.Attributes.Add("subschemaSubentry");
+ var response = (SearchResponse)_connection.SendRequest(request);
+
+ if (response.ResultCode != ResultCode.Success)
+ {
+ _logger.Warning("GetSubschemaSubentryDn: No success. Result code: {ResultCode}", response.ResultCode);
+ return null;
+ }
+
+ if (response.Entries.Count == 0)
+ {
+ _logger.Warning("GetSubschemaSubentryDn: Didn't get any results from rootDSE!");
+ return null;
+ }
+
+ var entry = response.Entries[0];
+ return LdapConnectorUtilities.GetEntryAttributeStringValue(entry, "subschemaSubentry");
+ }
+
+ // -----------------------------------------------------------------------
+ // Shared helpers — used by both AD and RFC paths
+ // -----------------------------------------------------------------------
+
+ ///
+ /// Applies external ID and secondary external ID recommendations to an object type.
+ /// Shared between AD and RFC schema discovery paths.
+ ///
+ private void ApplyExternalIdRecommendations(ConnectorSchemaObjectType objectType)
+ {
+ // make a recommendation on what unique identifier attribute(s) to use
+ var externalIdAttrName = _rootDse.ExternalIdAttributeName;
+ var externalIdSchemaAttribute = objectType.Attributes.SingleOrDefault(
+ a => a.Name.Equals(externalIdAttrName, StringComparison.OrdinalIgnoreCase));
+
+ if (externalIdSchemaAttribute != null)
+ {
+ objectType.RecommendedExternalIdAttribute = externalIdSchemaAttribute;
+ }
+ else
+ {
+ _logger.Warning("Schema discovery: external ID attribute '{ExternalIdAttr}' not found on object type '{ObjectType}'. " +
+ "External ID recommendation will be unavailable — the administrator must select one manually.",
+ externalIdAttrName, objectType.Name);
+ }
+
+ // say what the secondary external identifier needs to be for LDAP systems
+ var dnSchemaAttribute = objectType.Attributes.SingleOrDefault(a => a.Name.Equals("distinguishedname", StringComparison.OrdinalIgnoreCase));
+ if (dnSchemaAttribute != null)
+ {
+ objectType.RecommendedSecondaryExternalIdAttribute = dnSchemaAttribute;
+
+ // override the data type for distinguishedName, we want to handle it as a string, not a reference type
+ // we do this as a DN attribute on an object cannot be a reference to itself. that would make no sense.
+ dnSchemaAttribute.Type = AttributeDataType.Text;
+
+ // override writability for distinguishedName: AD marks it as systemOnly but the LDAP connector
+ // needs it writable because the DN is provided by the client in Add requests to specify where
+ // the object is created. It's not written as an attribute — it's the target of the LDAP Add operation.
+ dnSchemaAttribute.Writability = AttributeWritability.Writable;
+ }
+ }
+}
diff --git a/src/JIM.Connectors/LDAP/LdapConnectorUtilities.cs b/src/JIM.Connectors/LDAP/LdapConnectorUtilities.cs
index 950037645..ba87b4dec 100644
--- a/src/JIM.Connectors/LDAP/LdapConnectorUtilities.cs
+++ b/src/JIM.Connectors/LDAP/LdapConnectorUtilities.cs
@@ -406,13 +406,50 @@ public int GetHashCode(byte[] obj)
}
///
- /// Queries the rootDSE to detect directory type (Active Directory, Samba AD, or generic LDAP).
+ /// Determines the from rootDSE capabilities and vendor information.
+ ///
+ /// OIDs from the rootDSE supportedCapabilities attribute.
+ /// The vendorName attribute from rootDSE (may be null).
+ /// The structuralObjectClass from rootDSE (may be null). OpenLDAP uses "OpenLDAProotDSE".
+ internal static LdapDirectoryType DetectDirectoryType(IEnumerable? supportedCapabilities, string? vendorName, string? structuralObjectClass = null)
+ {
+ var hasAdCapability = supportedCapabilities != null &&
+ (supportedCapabilities.Contains(LdapConnectorConstants.LDAP_CAP_ACTIVE_DIRECTORY_OID) ||
+ supportedCapabilities.Contains(LdapConnectorConstants.LDAP_CAP_ACTIVE_DIRECTORY_ADAM_OID));
+
+ if (hasAdCapability)
+ {
+ // Samba AD advertises the AD capability OID but has different behaviour
+ var isSamba = vendorName != null &&
+ vendorName.Contains("Samba", StringComparison.OrdinalIgnoreCase);
+ return isSamba ? LdapDirectoryType.SambaAD : LdapDirectoryType.ActiveDirectory;
+ }
+
+ // Check vendorName first (set by some OpenLDAP configurations)
+ if (vendorName != null &&
+ vendorName.Contains("OpenLDAP", StringComparison.OrdinalIgnoreCase))
+ {
+ return LdapDirectoryType.OpenLDAP;
+ }
+
+ // Fallback: check structuralObjectClass on rootDSE — OpenLDAP uses "OpenLDAProotDSE"
+ if (structuralObjectClass != null &&
+ structuralObjectClass.Contains("OpenLDAP", StringComparison.OrdinalIgnoreCase))
+ {
+ return LdapDirectoryType.OpenLDAP;
+ }
+
+ return LdapDirectoryType.Generic;
+ }
+
+ ///
+ /// Queries the rootDSE to detect directory type and basic capabilities.
/// Used by schema discovery to apply directory-specific attribute overrides.
///
internal static LdapConnectorRootDse GetBasicRootDseInformation(LdapConnection connection, ILogger logger)
{
var request = new SearchRequest { Scope = SearchScope.Base };
- request.Attributes.AddRange(["supportedCapabilities", "vendorName"]);
+ request.Attributes.AddRange(["supportedCapabilities", "vendorName", "structuralObjectClass"]);
var response = (SearchResponse)connection.SendRequest(request);
@@ -425,25 +462,19 @@ internal static LdapConnectorRootDse GetBasicRootDseInformation(LdapConnection c
var rootDseEntry = response.Entries[0];
var capabilities = GetEntryAttributeStringValues(rootDseEntry, "supportedCapabilities");
- var isActiveDirectory = capabilities != null &&
- (capabilities.Contains(LdapConnectorConstants.LDAP_CAP_ACTIVE_DIRECTORY_OID) ||
- capabilities.Contains(LdapConnectorConstants.LDAP_CAP_ACTIVE_DIRECTORY_ADAM_OID));
-
var vendorName = GetEntryAttributeStringValue(rootDseEntry, "vendorName");
+ var structuralObjectClass = GetEntryAttributeStringValue(rootDseEntry, "structuralObjectClass");
- var isSambaAd = vendorName != null &&
- vendorName.Contains("Samba", StringComparison.OrdinalIgnoreCase);
- var supportsPaging = isActiveDirectory && !isSambaAd;
+ var directoryType = DetectDirectoryType(capabilities, vendorName, structuralObjectClass);
var rootDse = new LdapConnectorRootDse
{
- IsActiveDirectory = isActiveDirectory,
- VendorName = vendorName,
- SupportsPaging = supportsPaging
+ DirectoryType = directoryType,
+ VendorName = vendorName
};
- logger.Debug("GetBasicRootDseInformation: IsActiveDirectory={IsAd}, VendorName={VendorName}",
- rootDse.IsActiveDirectory, rootDse.VendorName ?? "(not set)");
+ logger.Debug("GetBasicRootDseInformation: DirectoryType={DirectoryType}, VendorName={VendorName}",
+ rootDse.DirectoryType, rootDse.VendorName ?? "(not set)");
return rootDse;
}
@@ -466,11 +497,11 @@ internal static string GetPaginationTokenName(ConnectedSystemContainer connected
///
/// The LDAP attribute name (e.g., "description").
/// The structural object class name (e.g., "user", "group").
- /// Whether the directory is Active Directory (AD-DS, AD-LDS, or Samba AD).
+ /// The detected directory type.
/// True if the attribute should be treated as single-valued despite the LDAP schema declaring it as multi-valued.
- internal static bool ShouldOverridePluralityToSingleValued(string attributeName, string objectTypeName, bool isActiveDirectory)
+ internal static bool ShouldOverridePluralityToSingleValued(string attributeName, string objectTypeName, LdapDirectoryType directoryType)
{
- return isActiveDirectory &&
+ return directoryType is LdapDirectoryType.ActiveDirectory or LdapDirectoryType.SambaAD &&
LdapConnectorConstants.SAM_ENFORCED_SINGLE_VALUED_ATTRIBUTES.Contains(attributeName) &&
LdapConnectorConstants.SAM_MANAGED_OBJECT_CLASSES.Contains(objectTypeName);
}
@@ -592,4 +623,15 @@ internal static bool HasValidRdnValues(string dn)
return false;
}
}
+
+ ///
+ /// Generates a fallback accesslog timestamp for when the cn=accesslog database is empty
+ /// (e.g., after snapshot restore clears stale accesslog data). Returns the current UTC time
+ /// formatted as LDAP generalised time (YYYYMMDDHHmmSS.ffffffZ), which serves as the
+ /// watermark for the next delta import: "no changes happened before this point."
+ ///
+ internal static string GenerateAccesslogFallbackTimestamp()
+ {
+ return DateTime.UtcNow.ToString("yyyyMMddHHmmss.ffffffZ");
+ }
}
\ No newline at end of file
diff --git a/src/JIM.Connectors/LDAP/Rfc4512SchemaParser.cs b/src/JIM.Connectors/LDAP/Rfc4512SchemaParser.cs
new file mode 100644
index 000000000..7d20b52e3
--- /dev/null
+++ b/src/JIM.Connectors/LDAP/Rfc4512SchemaParser.cs
@@ -0,0 +1,445 @@
+using JIM.Models.Core;
+
+namespace JIM.Connectors.LDAP;
+
+///
+/// Parses RFC 4512 schema description strings from LDAP subschema subentry attributes.
+/// These are the objectClasses and attributeTypes operational attributes
+/// returned by querying the subschema subentry (typically cn=Subschema).
+///
+///
+/// RFC 4512 § 4.1.1 (Object Class Description):
+///
+/// ( OID NAME 'name' [DESC 'description'] [SUP superior] [ABSTRACT|STRUCTURAL|AUXILIARY] [MUST (...)] [MAY (...)] )
+///
+/// RFC 4512 § 4.1.2 (Attribute Type Description):
+///
+/// ( OID NAME 'name' [DESC 'description'] [SUP superior] [SYNTAX oid] [SINGLE-VALUE] [NO-USER-MODIFICATION] [USAGE usage] )
+///
+///
+internal static class Rfc4512SchemaParser
+{
+ ///
+ /// Parses an RFC 4512 objectClass description string into a structured representation.
+ ///
+ internal static Rfc4512ObjectClassDescription? ParseObjectClassDescription(string definition)
+ {
+ if (string.IsNullOrWhiteSpace(definition))
+ return null;
+
+ var tokens = Tokenise(definition);
+ if (tokens.Count == 0)
+ return null;
+
+ var result = new Rfc4512ObjectClassDescription();
+
+ for (var i = 0; i < tokens.Count; i++)
+ {
+ var token = tokens[i];
+
+ switch (token)
+ {
+ case "NAME":
+ result.Name = ReadNameValue(tokens, ref i);
+ break;
+ case "DESC":
+ result.Description = ReadQuotedString(tokens, ref i);
+ break;
+ case "SUP":
+ result.SuperiorName = ReadSingleValue(tokens, ref i);
+ break;
+ case "ABSTRACT":
+ result.Kind = Rfc4512ObjectClassKind.Abstract;
+ break;
+ case "STRUCTURAL":
+ result.Kind = Rfc4512ObjectClassKind.Structural;
+ break;
+ case "AUXILIARY":
+ result.Kind = Rfc4512ObjectClassKind.Auxiliary;
+ break;
+ case "MUST":
+ result.MustAttributes = ReadOidList(tokens, ref i);
+ break;
+ case "MAY":
+ result.MayAttributes = ReadOidList(tokens, ref i);
+ break;
+ }
+ }
+
+ return result.Name != null ? result : null;
+ }
+
+ ///
+ /// Parses an RFC 4512 attributeType description string into a structured representation.
+ ///
+ internal static Rfc4512AttributeTypeDescription? ParseAttributeTypeDescription(string definition)
+ {
+ if (string.IsNullOrWhiteSpace(definition))
+ return null;
+
+ var tokens = Tokenise(definition);
+ if (tokens.Count == 0)
+ return null;
+
+ var result = new Rfc4512AttributeTypeDescription();
+
+ for (var i = 0; i < tokens.Count; i++)
+ {
+ var token = tokens[i];
+
+ switch (token)
+ {
+ case "NAME":
+ result.Name = ReadNameValue(tokens, ref i);
+ break;
+ case "DESC":
+ result.Description = ReadQuotedString(tokens, ref i);
+ break;
+ case "SUP":
+ result.SuperiorName = ReadSingleValue(tokens, ref i);
+ break;
+ case "SYNTAX":
+ result.SyntaxOid = ReadSyntaxOid(tokens, ref i);
+ break;
+ case "SINGLE-VALUE":
+ result.IsSingleValued = true;
+ break;
+ case "NO-USER-MODIFICATION":
+ result.IsNoUserModification = true;
+ break;
+ case "USAGE":
+ result.Usage = ReadUsageValue(tokens, ref i);
+ break;
+ }
+ }
+
+ return result.Name != null ? result : null;
+ }
+
+ ///
+ /// Maps an RFC 4517 SYNTAX OID to JIM's .
+ /// Returns for unknown or null OIDs as a safe default.
+ ///
+ internal static AttributeDataType GetRfcAttributeDataType(string? syntaxOid)
+ {
+ if (syntaxOid == null)
+ return AttributeDataType.Text;
+
+ return syntaxOid switch
+ {
+ // Boolean (RFC 4517 § 3.3.3)
+ "1.3.6.1.4.1.1466.115.121.1.7" => AttributeDataType.Boolean,
+
+ // Integer (RFC 4517 § 3.3.16)
+ "1.3.6.1.4.1.1466.115.121.1.27" => AttributeDataType.Number,
+
+ // Generalised Time (RFC 4517 § 3.3.13)
+ "1.3.6.1.4.1.1466.115.121.1.24" => AttributeDataType.DateTime,
+
+ // UTC Time
+ "1.3.6.1.4.1.1466.115.121.1.53" => AttributeDataType.DateTime,
+
+ // Distinguished Name (RFC 4517 § 3.3.9) — reference to another LDAP object
+ "1.3.6.1.4.1.1466.115.121.1.12" => AttributeDataType.Reference,
+
+ // Name and Optional UID (RFC 4517 § 3.3.21) — DN with optional unique ID
+ "1.3.6.1.4.1.1466.115.121.1.34" => AttributeDataType.Reference,
+
+ // Octet String (RFC 4517 § 3.3.25)
+ "1.3.6.1.4.1.1466.115.121.1.40" => AttributeDataType.Binary,
+
+ // JPEG (RFC 4517 § 3.3.17)
+ "1.3.6.1.4.1.1466.115.121.1.28" => AttributeDataType.Binary,
+
+ // Certificate (RFC 4523)
+ "1.3.6.1.4.1.1466.115.121.1.8" => AttributeDataType.Binary,
+
+ // Certificate List (CRL)
+ "1.3.6.1.4.1.1466.115.121.1.9" => AttributeDataType.Binary,
+
+ // Certificate Pair
+ "1.3.6.1.4.1.1466.115.121.1.10" => AttributeDataType.Binary,
+
+ // All text-like syntaxes
+ "1.3.6.1.4.1.1466.115.121.1.15" => AttributeDataType.Text, // Directory String
+ "1.3.6.1.4.1.1466.115.121.1.26" => AttributeDataType.Text, // IA5 String
+ "1.3.6.1.4.1.1466.115.121.1.44" => AttributeDataType.Text, // Printable String
+ "1.3.6.1.4.1.1466.115.121.1.36" => AttributeDataType.Text, // Numeric String
+ "1.3.6.1.4.1.1466.115.121.1.38" => AttributeDataType.Text, // OID
+ "1.3.6.1.4.1.1466.115.121.1.50" => AttributeDataType.Text, // Telephone Number
+ "1.3.6.1.4.1.1466.115.121.1.11" => AttributeDataType.Text, // Country String
+ "1.3.6.1.4.1.1466.115.121.1.6" => AttributeDataType.Text, // Bit String
+ "1.3.6.1.4.1.1466.115.121.1.41" => AttributeDataType.Text, // Postal Address
+ "1.3.6.1.4.1.1466.115.121.1.22" => AttributeDataType.Text, // Facsimile Telephone Number
+ "1.3.6.1.4.1.1466.115.121.1.52" => AttributeDataType.Text, // Telex Number
+ "1.3.6.1.4.1.1466.115.121.1.14" => AttributeDataType.Text, // Delivery Method
+ "1.3.6.1.4.1.1466.115.121.1.39" => AttributeDataType.Text, // Other Mailbox
+ "1.3.6.1.1.16.1" => AttributeDataType.Text, // UUID (RFC 4530) — entryUUID
+
+ // Unknown — default to Text as a safe fallback
+ _ => AttributeDataType.Text
+ };
+ }
+
+ ///
+ /// Determines attribute writability from RFC 4512 schema metadata.
+ /// Operational attributes (non-userApplications USAGE) and NO-USER-MODIFICATION attributes are read-only.
+ ///
+ internal static AttributeWritability DetermineRfcAttributeWritability(
+ Rfc4512AttributeUsage usage, bool isNoUserModification)
+ {
+ if (isNoUserModification)
+ return AttributeWritability.ReadOnly;
+
+ if (usage != Rfc4512AttributeUsage.UserApplications)
+ return AttributeWritability.ReadOnly;
+
+ return AttributeWritability.Writable;
+ }
+
+ // -----------------------------------------------------------------------
+ // Tokeniser and token readers
+ // -----------------------------------------------------------------------
+
+ ///
+ /// Splits an RFC 4512 description string into tokens, handling parenthesised groups
+ /// and quoted strings as atomic units.
+ ///
+ private static List Tokenise(string definition)
+ {
+ var tokens = new List();
+ var i = 0;
+
+ while (i < definition.Length)
+ {
+ // Skip whitespace
+ if (char.IsWhiteSpace(definition[i]))
+ {
+ i++;
+ continue;
+ }
+
+ // Quoted string — collect everything between single quotes as one token
+ if (definition[i] == '\'')
+ {
+ var end = definition.IndexOf('\'', i + 1);
+ if (end == -1) break;
+ tokens.Add(definition[(i + 1)..end]);
+ i = end + 1;
+ continue;
+ }
+
+ // Parentheses are their own tokens
+ if (definition[i] == '(' || definition[i] == ')')
+ {
+ tokens.Add(definition[i].ToString());
+ i++;
+ continue;
+ }
+
+ // Dollar sign separator in attribute lists
+ if (definition[i] == '$')
+ {
+ tokens.Add("$");
+ i++;
+ continue;
+ }
+
+ // Unquoted word — collect until whitespace, paren, quote, or dollar
+ var start = i;
+ while (i < definition.Length && !char.IsWhiteSpace(definition[i]) &&
+ definition[i] != '(' && definition[i] != ')' &&
+ definition[i] != '\'' && definition[i] != '$')
+ {
+ i++;
+ }
+
+ if (i > start)
+ tokens.Add(definition[start..i]);
+ }
+
+ return tokens;
+ }
+
+ ///
+ /// Reads a NAME value which can be either a single quoted string or a parenthesised list.
+ /// Returns the first name when multiple names are given.
+ ///
+ private static string? ReadNameValue(List tokens, ref int i)
+ {
+ if (i + 1 >= tokens.Count)
+ return null;
+
+ // Check if next token is a parenthesised list: NAME ( 'name1' 'name2' )
+ if (tokens[i + 1] == "(")
+ {
+ i += 2; // skip "(" token
+ string? firstName = null;
+ while (i < tokens.Count && tokens[i] != ")")
+ {
+ if (tokens[i] != "$" && tokens[i] != "(")
+ firstName ??= tokens[i];
+ i++;
+ }
+ return firstName;
+ }
+
+ // Single name: NAME 'name'
+ i++;
+ return tokens[i];
+ }
+
+ ///
+ /// Reads the next quoted string token after a keyword like DESC.
+ /// The tokeniser has already stripped the quotes.
+ ///
+ private static string? ReadQuotedString(List tokens, ref int i)
+ {
+ if (i + 1 >= tokens.Count)
+ return null;
+
+ // DESC values may span multiple tokens if the tokeniser split on spaces within quotes.
+ // But our tokeniser preserves quoted strings as a single token, so just read the next one.
+ // However, the DESC value might contain spaces and be enclosed in a single pair of quotes
+ // that our tokeniser already handled. The token after DESC is the unquoted content.
+
+ // Actually — the tokeniser collects everything between ' and ' as one token.
+ // But DESC 'foo bar' means the content between quotes is "foo bar" — one token.
+ // However if the description contains an apostrophe... that's an edge case we handle
+ // by just reading the next token.
+ i++;
+ return tokens[i];
+ }
+
+ ///
+ /// Reads a single unquoted value after a keyword like SUP.
+ ///
+ private static string? ReadSingleValue(List tokens, ref int i)
+ {
+ if (i + 1 >= tokens.Count)
+ return null;
+
+ i++;
+ return tokens[i];
+ }
+
+ ///
+ /// Reads a SYNTAX OID value, stripping any length constraint suffix (e.g., {64}).
+ ///
+ private static string? ReadSyntaxOid(List tokens, ref int i)
+ {
+ if (i + 1 >= tokens.Count)
+ return null;
+
+ i++;
+ var oid = tokens[i];
+
+ // Strip length constraint: "1.3.6.1.4.1.1466.115.121.1.15{64}" → "1.3.6.1.4.1.1466.115.121.1.15"
+ var braceIndex = oid.IndexOf('{');
+ if (braceIndex > 0)
+ oid = oid[..braceIndex];
+
+ return oid;
+ }
+
+ ///
+ /// Reads a USAGE value (one of: userApplications, directoryOperation, distributedOperation, dSAOperation).
+ ///
+ private static Rfc4512AttributeUsage ReadUsageValue(List tokens, ref int i)
+ {
+ if (i + 1 >= tokens.Count)
+ return Rfc4512AttributeUsage.UserApplications;
+
+ i++;
+ return tokens[i] switch
+ {
+ "directoryOperation" => Rfc4512AttributeUsage.DirectoryOperation,
+ "distributedOperation" => Rfc4512AttributeUsage.DistributedOperation,
+ "dSAOperation" => Rfc4512AttributeUsage.DsaOperation,
+ _ => Rfc4512AttributeUsage.UserApplications
+ };
+ }
+
+ ///
+ /// Reads an OID list which can be either a single name or a parenthesised $-separated list.
+ /// Used for MUST and MAY attribute lists.
+ ///
+ private static List ReadOidList(List tokens, ref int i)
+ {
+ var result = new List();
+ if (i + 1 >= tokens.Count)
+ return result;
+
+ // Check if next token starts a parenthesised list
+ if (tokens[i + 1] == "(")
+ {
+ i += 2; // skip "("
+ while (i < tokens.Count && tokens[i] != ")")
+ {
+ if (tokens[i] != "$")
+ result.Add(tokens[i]);
+ i++;
+ }
+ }
+ else
+ {
+ // Single attribute name
+ i++;
+ result.Add(tokens[i]);
+ }
+
+ return result;
+ }
+}
+
+///
+/// Parsed representation of an RFC 4512 objectClass description.
+///
+internal class Rfc4512ObjectClassDescription
+{
+ public string? Name { get; set; }
+ public string? Description { get; set; }
+ public string? SuperiorName { get; set; }
+ public Rfc4512ObjectClassKind Kind { get; set; } = Rfc4512ObjectClassKind.Structural;
+ public List MustAttributes { get; set; } = new();
+ public List MayAttributes { get; set; } = new();
+}
+
+///
+/// Parsed representation of an RFC 4512 attributeType description.
+///
+internal class Rfc4512AttributeTypeDescription
+{
+ public string? Name { get; set; }
+ public string? Description { get; set; }
+ public string? SuperiorName { get; set; }
+ public string? SyntaxOid { get; set; }
+ public bool IsSingleValued { get; set; }
+ public bool IsNoUserModification { get; set; }
+ public Rfc4512AttributeUsage Usage { get; set; } = Rfc4512AttributeUsage.UserApplications;
+}
+
+///
+/// RFC 4512 object class kinds.
+///
+internal enum Rfc4512ObjectClassKind
+{
+ Abstract,
+ Structural,
+ Auxiliary
+}
+
+///
+/// RFC 4512 attribute usage types, determining whether an attribute is user-modifiable or operational.
+///
+internal enum Rfc4512AttributeUsage
+{
+ /// User data attribute — writable by clients.
+ UserApplications,
+ /// Operational attribute managed by the directory server.
+ DirectoryOperation,
+ /// Operational attribute shared across DSAs (e.g., subschemaSubentry).
+ DistributedOperation,
+ /// Operational attribute local to a single DSA (e.g., createTimestamp).
+ DsaOperation
+}
diff --git a/src/JIM.Data/Repositories/ISyncRepository.cs b/src/JIM.Data/Repositories/ISyncRepository.cs
index 6da30c675..10f08af93 100644
--- a/src/JIM.Data/Repositories/ISyncRepository.cs
+++ b/src/JIM.Data/Repositories/ISyncRepository.cs
@@ -206,6 +206,14 @@ public interface ISyncRepository
///
Task FixupCrossBatchChangeRecordReferenceIdsAsync(int connectedSystemId);
+ ///
+ /// Tactical fixup: populates ReferenceValueId on MetaverseObjectAttributeValues where the FK is
+ /// null but the referenced MVO exists. EF does not reliably infer the FK from the ReferenceValue
+ /// navigation when entities are managed via explicit state (Entry().State = Added/Modified).
+ /// This will be retired when MVO persistence is converted to direct SQL.
+ ///
+ Task FixupMvoReferenceValueIdsAsync(IReadOnlyList<(Guid MvoId, int AttributeId, Guid TargetMvoId)> fixups);
+
#endregion
#region Object Matching — Data Access
diff --git a/src/JIM.InMemoryData/SyncRepository.cs b/src/JIM.InMemoryData/SyncRepository.cs
index 29f2e6d3e..d9e71fc27 100644
--- a/src/JIM.InMemoryData/SyncRepository.cs
+++ b/src/JIM.InMemoryData/SyncRepository.cs
@@ -572,6 +572,12 @@ public Task FixupCrossBatchChangeRecordReferenceIdsAsync(int connectedSyste
return Task.FromResult(resolved);
}
+ public Task FixupMvoReferenceValueIdsAsync(IReadOnlyList<(Guid MvoId, int AttributeId, Guid TargetMvoId)> fixups)
+ {
+ // In-memory repo handles this via FixupMvoAttributeValues on read.
+ return Task.FromResult(0);
+ }
+
#endregion
#region Metaverse Object — Reads
@@ -1497,6 +1503,7 @@ private void FixupPendingExportAttributeChanges(PendingExport pe)
///
/// Fixes up MVO attribute value FK/nav prop mismatches.
/// Production code sets Attribute nav prop but not AttributeId; EF resolves on SaveChanges.
+ /// Similarly, ReferenceValue nav prop may be set without ReferenceValueId.
///
private static void FixupMvoAttributeValues(MetaverseObject mvo)
{
@@ -1504,6 +1511,9 @@ private static void FixupMvoAttributeValues(MetaverseObject mvo)
{
if (av.AttributeId == 0 && av.Attribute != null)
av.AttributeId = av.Attribute.Id;
+ if (av.ReferenceValue != null && av.ReferenceValue.Id != Guid.Empty
+ && (!av.ReferenceValueId.HasValue || av.ReferenceValueId == Guid.Empty))
+ av.ReferenceValueId = av.ReferenceValue.Id;
}
}
diff --git a/src/JIM.Models/Activities/Activity.cs b/src/JIM.Models/Activities/Activity.cs
index d914e95d7..e1d1c5cdd 100644
--- a/src/JIM.Models/Activities/Activity.cs
+++ b/src/JIM.Models/Activities/Activity.cs
@@ -56,6 +56,14 @@ public class Activity
public string? ErrorStackTrace { get; set; }
+ ///
+ /// Connector-level warning message describing a non-fatal operational note about the activity.
+ /// For example, when a delta import falls back to a full import because the watermark was unavailable.
+ /// This is displayed on the activity detail page and causes the activity to complete with warning status.
+ /// Unlike ErrorMessage, this does not indicate a failure — the activity still succeeded, just with caveats.
+ ///
+ public string? WarningMessage { get; set; }
+
///
/// When the activity is complete, a value for how long the activity took to complete should be stored here.
/// This may be a noticeably smaller value than the total activity time, as some activities take a while before
diff --git a/src/JIM.Models/Activities/ActivityRunProfileExecutionItemErrorType.cs b/src/JIM.Models/Activities/ActivityRunProfileExecutionItemErrorType.cs
index 8ee554aae..30d7e2afd 100644
--- a/src/JIM.Models/Activities/ActivityRunProfileExecutionItemErrorType.cs
+++ b/src/JIM.Models/Activities/ActivityRunProfileExecutionItemErrorType.cs
@@ -80,5 +80,22 @@ public enum ActivityRunProfileExecutionItemErrorType
/// An unexpected exception occurred during sync processing. The full exception message and
/// stack trace are captured. This indicates a bug in the processing logic rather than a data issue.
///
- UnhandledError
+ UnhandledError,
+
+ ///
+ /// During import attribute flow, a multi-valued source attribute was mapped to a single-valued
+ /// target attribute. The first value was used. This is informational — the sync succeeded,
+ /// but the administrator should verify the correct value was selected.
+ ///
+ MultiValuedAttributeTruncated,
+
+ ///
+ /// A delta import could not be performed because the directory's change tracking watermark
+ /// (e.g., OpenLDAP accesslog timestamp) was not available. The connector automatically
+ /// fell back to performing a full import instead, which correctly imported all objects and
+ /// established the watermark for future delta imports. This is informational — the import
+ /// succeeded, but was slower than expected. If this recurs, verify that the directory's
+ /// change tracking mechanism (e.g., accesslog overlay) is accessible to the bind account.
+ ///
+ DeltaImportFallbackToFullImport
}
diff --git a/src/JIM.Models/Core/MetaverseObjectChangeAttribute.cs b/src/JIM.Models/Core/MetaverseObjectChangeAttribute.cs
index 684911e90..ea26c4f7a 100644
--- a/src/JIM.Models/Core/MetaverseObjectChangeAttribute.cs
+++ b/src/JIM.Models/Core/MetaverseObjectChangeAttribute.cs
@@ -9,7 +9,23 @@ public class MetaverseObjectChangeAttribute
///
public MetaverseObjectChange MetaverseObjectChange { get; set; } = null!;
- public MetaverseAttribute Attribute { get; set; } = null!;
+ ///
+ /// The metaverse attribute definition. Nullable because the attribute may be deleted after
+ /// the change was recorded. When null, use and .
+ ///
+ public MetaverseAttribute? Attribute { get; set; }
+
+ ///
+ /// Snapshot of the attribute name at the time of the change.
+ /// Preserved even if the attribute definition is later deleted.
+ ///
+ public string AttributeName { get; set; } = null!;
+
+ ///
+ /// Snapshot of the attribute data type at the time of the change.
+ /// Preserved even if the attribute definition is later deleted.
+ ///
+ public AttributeDataType AttributeType { get; set; }
///
/// A list of what values were added to or removed from this attribute.
diff --git a/src/JIM.Models/Staging/ConnectedSystemEnums.cs b/src/JIM.Models/Staging/ConnectedSystemEnums.cs
index 5dc353c55..f52bae01b 100644
--- a/src/JIM.Models/Staging/ConnectedSystemEnums.cs
+++ b/src/JIM.Models/Staging/ConnectedSystemEnums.cs
@@ -168,5 +168,13 @@ public enum ConnectedSystemExportErrorType
/// This typically occurs when expression-based ID attributes evaluate with null or empty
/// input values, producing malformed identifiers.
///
- InvalidGeneratedExternalId
+ InvalidGeneratedExternalId,
+
+ ///
+ /// A constraint violation occurred when managing a placeholder member on a group.
+ /// This typically means the directory has referential integrity enabled and the placeholder DN
+ /// does not reference an existing entry. The administrator should update the 'Group Placeholder
+ /// Member DN' connector setting to point to a valid entry in the directory.
+ ///
+ PlaceholderMemberConstraintViolation
}
diff --git a/src/JIM.Models/Staging/ConnectedSystemImportResult.cs b/src/JIM.Models/Staging/ConnectedSystemImportResult.cs
index a5c464904..dcbec0c4d 100644
--- a/src/JIM.Models/Staging/ConnectedSystemImportResult.cs
+++ b/src/JIM.Models/Staging/ConnectedSystemImportResult.cs
@@ -1,4 +1,6 @@
-namespace JIM.Models.Staging;
+using JIM.Models.Activities;
+
+namespace JIM.Models.Staging;
public class ConnectedSystemImportResult
{
@@ -21,4 +23,17 @@ public class ConnectedSystemImportResult
/// JIM will pass this data to Connectors on each synchronisation run.
///
public string? PersistedConnectorData { get; set; }
+
+ ///
+ /// Optional warning message from the connector. When set, the import will complete with a warning
+ /// status and the message will be recorded as an RPEI. Use this to communicate non-fatal operational
+ /// issues to the administrator (e.g., a delta import that fell back to a full import).
+ ///
+ public string? WarningMessage { get; set; }
+
+ ///
+ /// Optional error type classification for the warning. When is set,
+ /// this categorises the warning for filtering and integration test assertions.
+ ///
+ public ActivityRunProfileExecutionItemErrorType? WarningErrorType { get; set; }
}
\ No newline at end of file
diff --git a/src/JIM.Models/Staging/ConnectedSystemObjectAttributeValue.cs b/src/JIM.Models/Staging/ConnectedSystemObjectAttributeValue.cs
index f20263ca1..ec7455ab6 100644
--- a/src/JIM.Models/Staging/ConnectedSystemObjectAttributeValue.cs
+++ b/src/JIM.Models/Staging/ConnectedSystemObjectAttributeValue.cs
@@ -5,6 +5,14 @@ namespace JIM.Models.Staging;
public class ConnectedSystemObjectAttributeValue
{
+ ///
+ /// The MetaverseObjectId of the referenced CSO (the CSO pointed to by ReferenceValueId).
+ /// Populated via direct SQL in the repository to avoid the deep EF Include chain
+ /// (AttributeValues → ReferenceValue → MetaverseObject) that fails at scale.
+ ///
+ [NotMapped]
+ public Guid? ResolvedReferenceMetaverseObjectId { get; set; }
+
[Key]
[DatabaseGenerated(DatabaseGeneratedOption.Identity)]
public Guid Id { get; set; }
diff --git a/src/JIM.Models/Staging/ConnectedSystemObjectChangeAttribute.cs b/src/JIM.Models/Staging/ConnectedSystemObjectChangeAttribute.cs
index 826da8bd9..a63644b0a 100644
--- a/src/JIM.Models/Staging/ConnectedSystemObjectChangeAttribute.cs
+++ b/src/JIM.Models/Staging/ConnectedSystemObjectChangeAttribute.cs
@@ -1,4 +1,6 @@
-namespace JIM.Models.Staging;
+using JIM.Models.Core;
+
+namespace JIM.Models.Staging;
public class ConnectedSystemObjectChangeAttribute
{
@@ -11,9 +13,22 @@ public class ConnectedSystemObjectChangeAttribute
public ConnectedSystemObjectChange ConnectedSystemChange { get; set; } = null!;
///
- /// The connected system attribute these value changes relates to.
+ /// The connected system attribute definition. Nullable because the attribute may be deleted after
+ /// the change was recorded. When null, use and .
+ ///
+ public ConnectedSystemObjectTypeAttribute? Attribute { get; set; }
+
+ ///
+ /// Snapshot of the attribute name at the time of the change.
+ /// Preserved even if the attribute definition is later deleted.
+ ///
+ public string AttributeName { get; set; } = null!;
+
+ ///
+ /// Snapshot of the attribute data type at the time of the change.
+ /// Preserved even if the attribute definition is later deleted.
///
- public ConnectedSystemObjectTypeAttribute Attribute { get; set; } = null!;
+ public AttributeDataType AttributeType { get; set; }
///
/// A list of what values were added to or removed from this attribute.
@@ -22,6 +37,6 @@ public class ConnectedSystemObjectChangeAttribute
public override string ToString()
{
- return Attribute.Name;
+ return AttributeName;
}
}
\ No newline at end of file
diff --git a/src/JIM.Models/Sync/AttributeFlowWarning.cs b/src/JIM.Models/Sync/AttributeFlowWarning.cs
new file mode 100644
index 000000000..f0d6b8af9
--- /dev/null
+++ b/src/JIM.Models/Sync/AttributeFlowWarning.cs
@@ -0,0 +1,29 @@
+namespace JIM.Models.Sync;
+
+///
+/// Records a warning generated during inbound attribute flow when a multi-valued
+/// source attribute was mapped to a single-valued target attribute. The first value
+/// was selected and all others were discarded.
+///
+public class AttributeFlowWarning
+{
+ ///
+ /// The name of the source connected system attribute.
+ ///
+ public required string SourceAttributeName { get; set; }
+
+ ///
+ /// The name of the target metaverse attribute.
+ ///
+ public required string TargetAttributeName { get; set; }
+
+ ///
+ /// The total number of values present on the source attribute.
+ ///
+ public int ValueCount { get; set; }
+
+ ///
+ /// A string representation of the value that was selected (the first value).
+ ///
+ public required string SelectedValue { get; set; }
+}
diff --git a/src/JIM.PostgresData/JimDbContext.cs b/src/JIM.PostgresData/JimDbContext.cs
index f8ef6b7fc..f2209eb2a 100644
--- a/src/JIM.PostgresData/JimDbContext.cs
+++ b/src/JIM.PostgresData/JimDbContext.cs
@@ -220,6 +220,14 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
.WithOne(av => av.ConnectedSystemObjectChangeAttribute)
.OnDelete(DeleteBehavior.Cascade); // let the db delete all dependent ConnectedSystemObjectChangeAttributeValue objects when the parent is deleted.
+ // When a connected system attribute definition is deleted, preserve the change history record
+ // by setting the FK to null. The AttributeName and AttributeType sibling properties retain
+ // the attribute metadata even after the definition is removed.
+ modelBuilder.Entity()
+ .HasOne(ca => ca.Attribute)
+ .WithMany()
+ .OnDelete(DeleteBehavior.SetNull);
+
// When a CSO is deleted, set the ReferenceValueId to null in any change attribute values that reference it.
// This prevents FK violations when deleting CSOs that are referenced in historical change records.
modelBuilder.Entity()
@@ -244,6 +252,14 @@ protected override void OnModelCreating(ModelBuilder modelBuilder)
.HasMany(mvo => mvo.Changes)
.WithOne(mvoc => mvoc.MetaverseObject);
+ // When a metaverse attribute definition is deleted, preserve the change history record
+ // by setting the FK to null. The AttributeName and AttributeType sibling properties retain
+ // the attribute metadata even after the definition is removed.
+ modelBuilder.Entity()
+ .HasOne(ca => ca.Attribute)
+ .WithMany()
+ .OnDelete(DeleteBehavior.SetNull);
+
modelBuilder.Entity()
.HasOne(moav => moav.MetaverseObject)
.WithMany(mo => mo.AttributeValues);
diff --git a/src/JIM.PostgresData/Migrations/20260327085706_PreventAttributeChangeCascadeDelete.Designer.cs b/src/JIM.PostgresData/Migrations/20260327085706_PreventAttributeChangeCascadeDelete.Designer.cs
new file mode 100644
index 000000000..9bedfeada
--- /dev/null
+++ b/src/JIM.PostgresData/Migrations/20260327085706_PreventAttributeChangeCascadeDelete.Designer.cs
@@ -0,0 +1,4316 @@
+//
+using System;
+using System.Collections.Generic;
+using JIM.PostgresData;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
+using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata;
+
+#nullable disable
+
+namespace JIM.PostgresData.Migrations
+{
+ [DbContext(typeof(JimDbContext))]
+ [Migration("20260327085706_PreventAttributeChangeCascadeDelete")]
+ partial class PreventAttributeChangeCascadeDelete
+ {
+ ///
+ protected override void BuildTargetModel(ModelBuilder modelBuilder)
+ {
+#pragma warning disable 612, 618
+ modelBuilder
+ .HasAnnotation("ProductVersion", "9.0.14")
+ .HasAnnotation("Relational:MaxIdentifierLength", 63);
+
+ NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder);
+
+ modelBuilder.Entity("ApiKeyRole", b =>
+ {
+ b.Property("ApiKeyId")
+ .HasColumnType("uuid");
+
+ b.Property("RolesId")
+ .HasColumnType("integer");
+
+ b.HasKey("ApiKeyId", "RolesId");
+
+ b.HasIndex("RolesId");
+
+ b.ToTable("ApiKeyRole");
+ });
+
+ modelBuilder.Entity("ExampleDataTemplateAttributeMetaverseObjectType", b =>
+ {
+ b.Property("ExampleDataTemplateAttributesId")
+ .HasColumnType("integer");
+
+ b.Property("ReferenceMetaverseObjectTypesId")
+ .HasColumnType("integer");
+
+ b.HasKey("ExampleDataTemplateAttributesId", "ReferenceMetaverseObjectTypesId");
+
+ b.HasIndex("ReferenceMetaverseObjectTypesId");
+
+ b.ToTable("ExampleDataTemplateAttributeMetaverseObjectType");
+ });
+
+ modelBuilder.Entity("JIM.Models.Activities.Activity", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("ConnectedSystemId")
+ .HasColumnType("integer");
+
+ b.Property("ConnectedSystemRunProfileId")
+ .HasColumnType("integer");
+
+ b.Property("ConnectedSystemRunType")
+ .HasColumnType("integer");
+
+ b.Property("Created")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("DeletedActivityCount")
+ .HasColumnType("integer");
+
+ b.Property("DeletedCsoChangeCount")
+ .HasColumnType("integer");
+
+ b.Property("DeletedMvoChangeCount")
+ .HasColumnType("integer");
+
+ b.Property("DeletedRecordsFromDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("DeletedRecordsToDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("ErrorMessage")
+ .HasColumnType("text");
+
+ b.Property("ErrorStackTrace")
+ .HasColumnType("text");
+
+ b.Property("ExampleDataTemplateId")
+ .HasColumnType("integer");
+
+ b.Property("Executed")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("ExecutionTime")
+ .HasColumnType("interval");
+
+ b.Property("InitiatedById")
+ .HasColumnType("uuid");
+
+ b.Property("InitiatedByName")
+ .HasColumnType("text");
+
+ b.Property("InitiatedByType")
+ .HasColumnType("integer");
+
+ b.Property("Message")
+ .HasColumnType("text");
+
+ b.Property("MetaverseObjectId")
+ .HasColumnType("uuid");
+
+ b.Property("ObjectsProcessed")
+ .HasColumnType("integer");
+
+ b.Property("ObjectsToProcess")
+ .HasColumnType("integer");
+
+ b.Property("ParentActivityId")
+ .HasColumnType("uuid");
+
+ b.Property("PendingExportsConfirmed")
+ .HasColumnType("integer");
+
+ b.Property("ScheduleExecutionId")
+ .HasColumnType("uuid");
+
+ b.Property("ScheduleStepIndex")
+ .HasColumnType("integer");
+
+ b.Property("Status")
+ .HasColumnType("integer");
+
+ b.Property("SyncRuleId")
+ .HasColumnType("integer");
+
+ b.Property("TargetContext")
+ .HasColumnType("text");
+
+ b.Property("TargetName")
+ .HasColumnType("text");
+
+ b.Property("TargetOperationType")
+ .HasColumnType("integer");
+
+ b.Property("TargetType")
+ .HasColumnType("integer");
+
+ b.Property("TotalActivityTime")
+ .HasColumnType("interval");
+
+ b.Property("TotalAdded")
+ .HasColumnType("integer");
+
+ b.Property("TotalAttributeFlows")
+ .HasColumnType("integer");
+
+ b.Property("TotalCreated")
+ .HasColumnType("integer");
+
+ b.Property("TotalDeleted")
+ .HasColumnType("integer");
+
+ b.Property("TotalDeprovisioned")
+ .HasColumnType("integer");
+
+ b.Property("TotalDisconnected")
+ .HasColumnType("integer");
+
+ b.Property("TotalDisconnectedOutOfScope")
+ .HasColumnType("integer");
+
+ b.Property("TotalDriftCorrections")
+ .HasColumnType("integer");
+
+ b.Property("TotalErrors")
+ .HasColumnType("integer");
+
+ b.Property("TotalExported")
+ .HasColumnType("integer");
+
+ b.Property("TotalJoined")
+ .HasColumnType("integer");
+
+ b.Property("TotalOutOfScopeRetainJoin")
+ .HasColumnType("integer");
+
+ b.Property("TotalPendingExports")
+ .HasColumnType("integer");
+
+ b.Property("TotalProjected")
+ .HasColumnType("integer");
+
+ b.Property("TotalProvisioned")
+ .HasColumnType("integer");
+
+ b.Property("TotalUpdated")
+ .HasColumnType("integer");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ConnectedSystemId");
+
+ b.HasIndex("ConnectedSystemRunProfileId");
+
+ b.HasIndex("Created")
+ .IsDescending()
+ .HasDatabaseName("IX_Activities_Created");
+
+ b.HasIndex("SyncRuleId");
+
+ b.ToTable("Activities");
+ });
+
+ modelBuilder.Entity("JIM.Models.Activities.ActivityRunProfileExecutionItem", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("ActivityId")
+ .HasColumnType("uuid");
+
+ b.Property("AttributeFlowCount")
+ .HasColumnType("integer");
+
+ b.Property("ConnectedSystemObjectId")
+ .HasColumnType("uuid");
+
+ b.Property("DisplayNameSnapshot")
+ .HasColumnType("text");
+
+ b.Property("ErrorMessage")
+ .HasColumnType("text");
+
+ b.Property("ErrorStackTrace")
+ .HasColumnType("text");
+
+ b.Property("ErrorType")
+ .HasColumnType("integer");
+
+ b.Property("ExternalIdSnapshot")
+ .HasColumnType("text");
+
+ b.Property("NoChangeReason")
+ .HasColumnType("integer");
+
+ b.Property("ObjectChangeType")
+ .HasColumnType("integer");
+
+ b.Property("ObjectTypeSnapshot")
+ .HasColumnType("text");
+
+ b.Property("OutcomeSummary")
+ .HasColumnType("text");
+
+ b.Property("PendingExportId")
+ .HasColumnType("uuid");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ActivityId");
+
+ b.HasIndex("ConnectedSystemObjectId");
+
+ b.ToTable("ActivityRunProfileExecutionItems");
+ });
+
+ modelBuilder.Entity("JIM.Models.Activities.ActivityRunProfileExecutionItemSyncOutcome", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("ActivityRunProfileExecutionItemId")
+ .HasColumnType("uuid");
+
+ b.Property("ConnectedSystemObjectChangeId")
+ .HasColumnType("uuid");
+
+ b.Property("DetailCount")
+ .HasColumnType("integer");
+
+ b.Property("DetailMessage")
+ .HasColumnType("text");
+
+ b.Property("Ordinal")
+ .HasColumnType("integer");
+
+ b.Property("OutcomeType")
+ .HasColumnType("integer");
+
+ b.Property("ParentSyncOutcomeId")
+ .HasColumnType("uuid");
+
+ b.Property("TargetEntityDescription")
+ .HasColumnType("text");
+
+ b.Property("TargetEntityId")
+ .HasColumnType("uuid");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ActivityRunProfileExecutionItemId")
+ .HasDatabaseName("IX_ActivityRunProfileExecutionItemSyncOutcomes_ActivityRunProfileExecutionItemId");
+
+ b.HasIndex("ConnectedSystemObjectChangeId");
+
+ b.HasIndex("ParentSyncOutcomeId");
+
+ b.HasIndex("ActivityRunProfileExecutionItemId", "OutcomeType")
+ .HasDatabaseName("IX_ActivityRunProfileExecutionItemSyncOutcomes_RpeiId_OutcomeType");
+
+ b.ToTable("ActivityRunProfileExecutionItemSyncOutcomes");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.MetaverseAttribute", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("integer");
+
+ NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id"));
+
+ b.Property("AttributePlurality")
+ .HasColumnType("integer");
+
+ b.Property("BuiltIn")
+ .HasColumnType("boolean");
+
+ b.Property("Created")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("CreatedById")
+ .HasColumnType("uuid");
+
+ b.Property("CreatedByName")
+ .HasColumnType("text");
+
+ b.Property("CreatedByType")
+ .HasColumnType("integer");
+
+ b.Property("LastUpdated")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("LastUpdatedById")
+ .HasColumnType("uuid");
+
+ b.Property("LastUpdatedByName")
+ .HasColumnType("text");
+
+ b.Property("LastUpdatedByType")
+ .HasColumnType("integer");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("RenderingHint")
+ .HasColumnType("integer");
+
+ b.Property("Type")
+ .HasColumnType("integer");
+
+ b.HasKey("Id");
+
+ b.HasIndex("Name");
+
+ b.ToTable("MetaverseAttributes");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.MetaverseObject", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("Created")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("DeletionInitiatedById")
+ .HasColumnType("uuid");
+
+ b.Property("DeletionInitiatedByName")
+ .HasColumnType("text");
+
+ b.Property("DeletionInitiatedByType")
+ .HasColumnType("integer");
+
+ b.Property("LastConnectorDisconnectedDate")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("LastUpdated")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("Origin")
+ .HasColumnType("integer");
+
+ b.Property("Status")
+ .HasColumnType("integer");
+
+ b.Property("TypeId")
+ .HasColumnType("integer");
+
+ b.Property("xmin")
+ .IsConcurrencyToken()
+ .ValueGeneratedOnAddOrUpdate()
+ .HasColumnType("xid")
+ .HasColumnName("xmin");
+
+ b.HasKey("Id");
+
+ b.HasIndex("TypeId");
+
+ b.HasIndex("Origin", "TypeId", "LastConnectorDisconnectedDate")
+ .HasDatabaseName("IX_MetaverseObjects_Origin_Type_LastDisconnected");
+
+ b.ToTable("MetaverseObjects");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.MetaverseObjectAttributeValue", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("AttributeId")
+ .HasColumnType("integer");
+
+ b.Property("BoolValue")
+ .HasColumnType("boolean");
+
+ b.Property("ByteValue")
+ .HasColumnType("bytea");
+
+ b.Property("ContributedBySystemId")
+ .HasColumnType("integer");
+
+ b.Property("DateTimeValue")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("GuidValue")
+ .HasColumnType("uuid");
+
+ b.Property("IntValue")
+ .HasColumnType("integer");
+
+ b.Property("LongValue")
+ .HasColumnType("bigint");
+
+ b.Property("MetaverseObjectId")
+ .HasColumnType("uuid");
+
+ b.Property("ReferenceValueId")
+ .HasColumnType("uuid");
+
+ b.Property("StringValue")
+ .HasColumnType("text");
+
+ b.Property("UnresolvedReferenceValueId")
+ .HasColumnType("uuid");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ContributedBySystemId");
+
+ b.HasIndex("DateTimeValue");
+
+ b.HasIndex("GuidValue");
+
+ b.HasIndex("IntValue");
+
+ b.HasIndex("LongValue");
+
+ b.HasIndex("MetaverseObjectId");
+
+ b.HasIndex("ReferenceValueId");
+
+ b.HasIndex("StringValue");
+
+ b.HasIndex("UnresolvedReferenceValueId");
+
+ b.HasIndex("AttributeId", "StringValue")
+ .HasDatabaseName("IX_MetaverseObjectAttributeValues_AttributeId_StringValue");
+
+ b.ToTable("MetaverseObjectAttributeValues");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.MetaverseObjectChange", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("ActivityRunProfileExecutionItemId")
+ .HasColumnType("uuid");
+
+ b.Property("ChangeInitiatorType")
+ .HasColumnType("integer");
+
+ b.Property("ChangeTime")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("ChangeType")
+ .HasColumnType("integer");
+
+ b.Property("DeletedMetaverseObjectId")
+ .HasColumnType("uuid");
+
+ b.Property("DeletedObjectDisplayName")
+ .HasColumnType("text");
+
+ b.Property("DeletedObjectTypeId")
+ .HasColumnType("integer");
+
+ b.Property("InitiatedById")
+ .HasColumnType("uuid");
+
+ b.Property("InitiatedByName")
+ .HasColumnType("text");
+
+ b.Property("InitiatedByType")
+ .HasColumnType("integer");
+
+ b.Property("MetaverseObjectId")
+ .HasColumnType("uuid");
+
+ b.Property("SyncRuleId")
+ .HasColumnType("integer");
+
+ b.Property("SyncRuleName")
+ .HasColumnType("text");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ActivityRunProfileExecutionItemId")
+ .IsUnique();
+
+ b.HasIndex("DeletedObjectTypeId");
+
+ b.HasIndex("MetaverseObjectId");
+
+ b.HasIndex("SyncRuleId");
+
+ b.ToTable("MetaverseObjectChanges");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.MetaverseObjectChangeAttribute", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("AttributeId")
+ .HasColumnType("integer");
+
+ b.Property("AttributeName")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("AttributeType")
+ .HasColumnType("integer");
+
+ b.Property("MetaverseObjectChangeId")
+ .HasColumnType("uuid");
+
+ b.HasKey("Id");
+
+ b.HasIndex("AttributeId");
+
+ b.HasIndex("MetaverseObjectChangeId");
+
+ b.ToTable("MetaverseObjectChangeAttributes");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.MetaverseObjectChangeAttributeValue", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("BoolValue")
+ .HasColumnType("boolean");
+
+ b.Property("ByteValueLength")
+ .HasColumnType("integer");
+
+ b.Property("DateTimeValue")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("GuidValue")
+ .HasColumnType("uuid");
+
+ b.Property("IntValue")
+ .HasColumnType("integer");
+
+ b.Property("MetaverseObjectChangeAttributeId")
+ .HasColumnType("uuid");
+
+ b.Property("ReferenceValueId")
+ .HasColumnType("uuid");
+
+ b.Property("StringValue")
+ .HasColumnType("text");
+
+ b.Property("ValueChangeType")
+ .HasColumnType("integer");
+
+ b.HasKey("Id");
+
+ b.HasIndex("MetaverseObjectChangeAttributeId");
+
+ b.HasIndex("ReferenceValueId");
+
+ b.ToTable("MetaverseObjectChangeAttributeValues");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.MetaverseObjectType", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("integer");
+
+ NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id"));
+
+ b.Property("BuiltIn")
+ .HasColumnType("boolean");
+
+ b.Property("Created")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("CreatedById")
+ .HasColumnType("uuid");
+
+ b.Property("CreatedByName")
+ .HasColumnType("text");
+
+ b.Property("CreatedByType")
+ .HasColumnType("integer");
+
+ b.Property("DeletionGracePeriod")
+ .HasColumnType("interval");
+
+ b.Property("DeletionRule")
+ .HasColumnType("integer");
+
+ b.PrimitiveCollection>("DeletionTriggerConnectedSystemIds")
+ .IsRequired()
+ .HasColumnType("integer[]");
+
+ b.Property("Icon")
+ .HasColumnType("text");
+
+ b.Property("LastUpdated")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("LastUpdatedById")
+ .HasColumnType("uuid");
+
+ b.Property("LastUpdatedByName")
+ .HasColumnType("text");
+
+ b.Property("LastUpdatedByType")
+ .HasColumnType("integer");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("PluralName")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.HasKey("Id");
+
+ b.HasIndex("Name");
+
+ b.HasIndex("Name", "DeletionRule")
+ .HasDatabaseName("IX_MetaverseObjectTypes_Name_DeletionRule");
+
+ b.ToTable("MetaverseObjectTypes");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.ServiceSetting", b =>
+ {
+ b.Property("Key")
+ .HasMaxLength(100)
+ .HasColumnType("character varying(100)");
+
+ b.Property("Category")
+ .HasColumnType("integer");
+
+ b.Property("Created")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("CreatedById")
+ .HasColumnType("uuid");
+
+ b.Property("CreatedByName")
+ .HasColumnType("text");
+
+ b.Property("CreatedByType")
+ .HasColumnType("integer");
+
+ b.Property("DefaultValue")
+ .HasMaxLength(2000)
+ .HasColumnType("character varying(2000)");
+
+ b.Property("Description")
+ .HasMaxLength(1000)
+ .HasColumnType("character varying(1000)");
+
+ b.Property("DisplayName")
+ .IsRequired()
+ .HasMaxLength(200)
+ .HasColumnType("character varying(200)");
+
+ b.Property("EnumTypeName")
+ .HasMaxLength(500)
+ .HasColumnType("character varying(500)");
+
+ b.Property("IsReadOnly")
+ .HasColumnType("boolean");
+
+ b.Property("LastUpdated")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("LastUpdatedById")
+ .HasColumnType("uuid");
+
+ b.Property("LastUpdatedByName")
+ .HasColumnType("text");
+
+ b.Property("LastUpdatedByType")
+ .HasColumnType("integer");
+
+ b.Property("Value")
+ .HasMaxLength(2000)
+ .HasColumnType("character varying(2000)");
+
+ b.Property("ValueType")
+ .HasColumnType("integer");
+
+ b.HasKey("Key");
+
+ b.ToTable("ServiceSettingItems");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.ServiceSettings", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("integer");
+
+ NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id"));
+
+ b.Property("Created")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("HistoryRetentionPeriod")
+ .HasColumnType("interval");
+
+ b.Property("IsServiceInMaintenanceMode")
+ .HasColumnType("boolean");
+
+ b.Property("SSOAuthority")
+ .HasColumnType("text");
+
+ b.Property("SSOClientId")
+ .HasColumnType("text");
+
+ b.Property("SSOEnableLogOut")
+ .HasColumnType("boolean");
+
+ b.Property("SSOSecret")
+ .HasColumnType("text");
+
+ b.Property("SSOUniqueIdentifierClaimType")
+ .HasColumnType("text");
+
+ b.Property("SSOUniqueIdentifierMetaverseAttributeId")
+ .HasColumnType("integer");
+
+ b.HasKey("Id");
+
+ b.HasIndex("SSOUniqueIdentifierMetaverseAttributeId");
+
+ b.ToTable("ServiceSettings");
+ });
+
+ modelBuilder.Entity("JIM.Models.Core.TrustedCertificate", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("uuid");
+
+ b.Property("CertificateData")
+ .HasColumnType("bytea");
+
+ b.Property("Created")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("CreatedById")
+ .HasColumnType("uuid");
+
+ b.Property("CreatedByName")
+ .HasColumnType("text");
+
+ b.Property("CreatedByType")
+ .HasColumnType("integer");
+
+ b.Property("FilePath")
+ .HasColumnType("text");
+
+ b.Property("IsEnabled")
+ .HasColumnType("boolean");
+
+ b.Property("Issuer")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("LastUpdated")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("LastUpdatedById")
+ .HasColumnType("uuid");
+
+ b.Property("LastUpdatedByName")
+ .HasColumnType("text");
+
+ b.Property("LastUpdatedByType")
+ .HasColumnType("integer");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("Notes")
+ .HasColumnType("text");
+
+ b.Property("SerialNumber")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("SourceType")
+ .HasColumnType("integer");
+
+ b.Property("Subject")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("Thumbprint")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("ValidFrom")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("ValidTo")
+ .HasColumnType("timestamp with time zone");
+
+ b.HasKey("Id");
+
+ b.HasIndex("Thumbprint")
+ .IsUnique();
+
+ b.ToTable("TrustedCertificates");
+ });
+
+ modelBuilder.Entity("JIM.Models.ExampleData.ExampleDataObjectType", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("integer");
+
+ NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id"));
+
+ b.Property("ExampleDataTemplateId")
+ .HasColumnType("integer");
+
+ b.Property("MetaverseObjectTypeId")
+ .HasColumnType("integer");
+
+ b.Property("ObjectsToCreate")
+ .HasColumnType("integer");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ExampleDataTemplateId");
+
+ b.HasIndex("MetaverseObjectTypeId");
+
+ b.ToTable("ExampleDataObjectTypes");
+ });
+
+ modelBuilder.Entity("JIM.Models.ExampleData.ExampleDataSet", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("integer");
+
+ NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id"));
+
+ b.Property("BuiltIn")
+ .HasColumnType("boolean");
+
+ b.Property("Created")
+ .HasColumnType("timestamp with time zone");
+
+ b.Property("Culture")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.Property("Name")
+ .IsRequired()
+ .HasColumnType("text");
+
+ b.HasKey("Id");
+
+ b.ToTable("ExampleDataSets");
+ });
+
+ modelBuilder.Entity("JIM.Models.ExampleData.ExampleDataSetInstance", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("integer");
+
+ NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id"));
+
+ b.Property("ExampleDataSetId")
+ .HasColumnType("integer");
+
+ b.Property("ExampleDataTemplateAttributeId")
+ .HasColumnType("integer");
+
+ b.Property("Order")
+ .HasColumnType("integer");
+
+ b.HasKey("Id");
+
+ b.HasIndex("ExampleDataSetId");
+
+ b.HasIndex("ExampleDataTemplateAttributeId");
+
+ b.ToTable("ExampleDataSetInstances");
+ });
+
+ modelBuilder.Entity("JIM.Models.ExampleData.ExampleDataSetValue", b =>
+ {
+ b.Property("Id")
+ .ValueGeneratedOnAdd()
+ .HasColumnType("integer");
+
+ NpgsqlPropertyBuilderExtensions.UseIdentityByDefaultColumn(b.Property("Id"));
+
+ b.Property