diff --git a/Robust.Client/ClientIoC.cs b/Robust.Client/ClientIoC.cs index 3df84845048..0a4879990d6 100644 --- a/Robust.Client/ClientIoC.cs +++ b/Robust.Client/ClientIoC.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics.Metrics; using Robust.Client.Audio; using Robust.Client.Audio.Midi; using Robust.Client.Configuration; @@ -37,6 +38,7 @@ using Robust.Shared.Configuration; using Robust.Shared.Console; using Robust.Shared.ContentPack; +using Robust.Shared.DataMetrics; using Robust.Shared.GameObjects; using Robust.Shared.IoC; using Robust.Shared.Localization; @@ -114,6 +116,8 @@ public static void RegisterIoC(GameController.DisplayMode mode, IDependencyColle deps.Register(); deps.Register(); deps.Register(); + deps.Register(); + deps.Register(); switch (mode) { diff --git a/Robust.Client/GameController/GameController.cs b/Robust.Client/GameController/GameController.cs index 14f5770d13e..bbf265d6a56 100644 --- a/Robust.Client/GameController/GameController.cs +++ b/Robust.Client/GameController/GameController.cs @@ -30,6 +30,7 @@ using Robust.Shared.Audio; using Robust.Shared.Configuration; using Robust.Shared.ContentPack; +using Robust.Shared.DataMetrics; using Robust.Shared.Exceptions; using Robust.Shared.IoC; using Robust.Shared.Localization; @@ -102,6 +103,7 @@ internal sealed partial class GameController : IGameControllerInternal [Dependency] private readonly LoadingScreenManager _loadscr = default!; [Dependency] private readonly ITransferManager _transfer = default!; [Dependency] private readonly ClientTransferTestManager _transferTest = default!; + [Dependency] private readonly IMeterFactoryInternal _meterFactory = default!; private IWebViewManagerHook? _webViewHook; @@ -426,6 +428,8 @@ internal bool StartupSystemSplash( _configurationManager.OverrideConVars(_commandLineArgs.CVars); } + _meterFactory.Initialize(); + ProfileOptSetup.Setup(_configurationManager); _prof.Initialize(); diff --git a/Robust.Server.Testing/RobustServerSimulation.cs b/Robust.Server.Testing/RobustServerSimulation.cs index 66eb92d861d..789cffd1575 100644 --- a/Robust.Server.Testing/RobustServerSimulation.cs +++ b/Robust.Server.Testing/RobustServerSimulation.cs @@ -1,3 +1,4 @@ +using System.Diagnostics.Metrics; using System.Reflection; using JetBrains.Annotations; using Moq; @@ -22,6 +23,7 @@ using Robust.Shared.Console; using Robust.Shared.Containers; using Robust.Shared.ContentPack; +using Robust.Shared.DataMetrics; using Robust.Shared.Exceptions; using Robust.Shared.GameObjects; using Robust.Shared.IoC; @@ -264,6 +266,7 @@ public ISimulation InitializeInstance() container.Register(); // Needed for grid fixture debugging. container.Register(); + container.Register(); container.Register(); // I just wanted to load pvs system diff --git a/Robust.Server/DataMetrics/MetricsManager.UpdateMetrics.cs b/Robust.Server/DataMetrics/MetricsManager.UpdateMetrics.cs index d91ffa1b895..f37bfef8f00 100644 --- a/Robust.Server/DataMetrics/MetricsManager.UpdateMetrics.cs +++ b/Robust.Server/DataMetrics/MetricsManager.UpdateMetrics.cs @@ -23,7 +23,7 @@ internal sealed partial class MetricsManager private void InitializeUpdateMetrics() { - _cfg.OnValueChanged( + Cfg.OnValueChanged( CVars.MetricsUpdateInterval, seconds => { diff --git a/Robust.Server/DataMetrics/MetricsManager.cs b/Robust.Server/DataMetrics/MetricsManager.cs index 88c19ec36b6..2940dfd74fe 100644 --- a/Robust.Server/DataMetrics/MetricsManager.cs +++ b/Robust.Server/DataMetrics/MetricsManager.cs @@ -10,6 +10,7 @@ using Robust.Shared; using Robust.Shared.Asynchronous; using Robust.Shared.Configuration; +using Robust.Shared.DataMetrics; using Robust.Shared.GameObjects; using Robust.Shared.IoC; using Robust.Shared.Log; @@ -44,9 +45,8 @@ public interface IMetricsManager event Action UpdateMetrics; } -internal sealed partial class MetricsManager : IMetricsManagerInternal, IDisposable +internal sealed partial class MetricsManager : MeterFactory, IMetricsManagerInternal { - [Dependency] private readonly IConfigurationManager _cfg = default!; [Dependency] private readonly IEntitySystemManager _entitySystemManager = default!; [Dependency] private readonly ILogManager _logManager = default!; [Dependency] private readonly ITaskManager _taskManager = default!; @@ -57,8 +57,10 @@ internal sealed partial class MetricsManager : IMetricsManagerInternal, IDisposa private IDisposable? _runtimeCollector; private ISawmill _sawmill = default!; - public void Initialize() + public override void Initialize() { + base.Initialize(); + _sawmill = _logManager.GetSawmill("metrics"); _initialized = true; @@ -82,7 +84,7 @@ public void Initialize() void ValueChanged(CVarDef cVar) where T : notnull { - _cfg.OnValueChanged(cVar, _ => Reload()); + Cfg.OnValueChanged(cVar, _ => Reload()); } InitializeUpdateMetrics(); @@ -102,9 +104,9 @@ private async Task Stop() _runtimeCollector = null; } - async void IDisposable.Dispose() + protected override async void Dispose() { - DisposeMeters(); + base.Dispose(); await Stop(); @@ -120,7 +122,7 @@ private async void Reload() await Stop(); - var enabled = _cfg.GetCVar(CVars.MetricsEnabled); + var enabled = Cfg.GetCVar(CVars.MetricsEnabled); _entitySystemManager.MetricsEnabled = enabled; if (!enabled) @@ -128,8 +130,8 @@ private async void Reload() return; } - var host = _cfg.GetCVar(CVars.MetricsHost); - var port = _cfg.GetCVar(CVars.MetricsPort); + var host = Cfg.GetCVar(CVars.MetricsHost); + var port = Cfg.GetCVar(CVars.MetricsPort); _sawmill.Info("Prometheus metrics enabled, host: {1} port: {0}", port, host); var sawmill = Logger.GetSawmill("metrics.server"); @@ -141,7 +143,7 @@ private async void Reload() beforeCollect: BeforeCollectCallback); _metricServer.Start(); - if (_cfg.GetCVar(CVars.MetricsRuntime)) + if (Cfg.GetCVar(CVars.MetricsRuntime)) { _sawmill.Debug("Enabling runtime metrics"); _runtimeCollector = BuildRuntimeStats().StartCollecting(); @@ -160,7 +162,7 @@ private DotNetRuntimeStatsBuilder.Builder BuildRuntimeStats() if (CapLevel(CVars.MetricsRuntimeContention) is { } contention) { - var rate = _cfg.GetCVar(CVars.MetricsRuntimeContentionSampleRate); + var rate = Cfg.GetCVar(CVars.MetricsRuntimeContentionSampleRate); builder.WithContentionStats(contention, (SampleEvery)rate); } @@ -174,7 +176,7 @@ private DotNetRuntimeStatsBuilder.Builder BuildRuntimeStats() if (CapLevel(CVars.MetricsRuntimeJit) is { } jit) { - var rate = _cfg.GetCVar(CVars.MetricsRuntimeJitSampleRate); + var rate = Cfg.GetCVar(CVars.MetricsRuntimeJitSampleRate); builder.WithJitStats(jit, (SampleEvery) rate); } @@ -190,7 +192,7 @@ private DotNetRuntimeStatsBuilder.Builder BuildRuntimeStats() CaptureLevel? CapLevel(CVarDef cvar) { - var val = _cfg.GetCVar(cvar); + var val = Cfg.GetCVar(cvar); if (val != "") return Enum.Parse(val); @@ -200,7 +202,7 @@ private DotNetRuntimeStatsBuilder.Builder BuildRuntimeStats() // 🪣 double[] Buckets(CVarDef cvar, double divide=1) { - return _cfg.GetCVar(cvar) + return Cfg.GetCVar(cvar) .Split(',') .Select(x => double.Parse(x, CultureInfo.InvariantCulture) / divide) .ToArray(); diff --git a/Robust.Shared/CVars.cs b/Robust.Shared/CVars.cs index ad0c8a4ee41..556d4f804b5 100644 --- a/Robust.Shared/CVars.cs +++ b/Robust.Shared/CVars.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics.Metrics; using System.Threading; using Lidgren.Network; using Robust.Shared.Audio; @@ -540,6 +541,15 @@ protected CVars() public static readonly CVarDef MetricsUpdateInterval = CVarDef.Create("metrics.update_interval", 0f, CVar.SERVERONLY); + /// + /// If set, an "instance" tag will be added to all metrics created via . + /// + /// + /// This is set by integration tests to distinguish instances. + /// + public static readonly CVarDef MetricsInstanceName = + CVarDef.Create("metrics.instance_name", ""); + /// /// Enable detailed runtime metrics. Empty to disable. /// diff --git a/Robust.Server/DataMetrics/MetricsManager.Factory.cs b/Robust.Shared/DataMetrics/MeterFactory.cs similarity index 66% rename from Robust.Server/DataMetrics/MetricsManager.Factory.cs rename to Robust.Shared/DataMetrics/MeterFactory.cs index f9543db929f..c195732ed7f 100644 --- a/Robust.Server/DataMetrics/MetricsManager.Factory.cs +++ b/Robust.Shared/DataMetrics/MeterFactory.cs @@ -2,14 +2,34 @@ using System.Collections.Generic; using System.Diagnostics.Metrics; using System.Linq; +using System.Threading; +using Robust.Shared.Configuration; +using Robust.Shared.IoC; using Robust.Shared.Utility; -namespace Robust.Server.DataMetrics; +namespace Robust.Shared.DataMetrics; -internal sealed partial class MetricsManager : IMeterFactory +internal interface IMeterFactoryInternal : IMeterFactory { + void Initialize(); +} + +[Virtual] +internal class MeterFactory : IMeterFactoryInternal +{ + [Dependency] protected readonly IConfigurationManager Cfg = null!; + private readonly Dictionary> _meterCache = new(); - private readonly object _meterCacheLock = new(); + private readonly Lock _meterCacheLock = new(); + + private string? _instanceName; + + public virtual void Initialize() + { + _instanceName = Cfg.GetCVar(CVars.MetricsInstanceName); + if (string.IsNullOrEmpty(_instanceName)) + _instanceName = null; + } Meter IMeterFactory.Create(MeterOptions options) { @@ -21,9 +41,21 @@ Meter IMeterFactory.Create(MeterOptions options) if (LockedFindCachedMeter(options) is { } cached) return cached.Meter; - var meter = new Meter(options.Name, options.Version, options.Tags, this); + var tags = options.Tags; + if (_instanceName != null) + { + tags = + [ + ..options.Tags ?? [], + new KeyValuePair("instance", _instanceName) + ]; + } + + // ReSharper disable once PossibleMultipleEnumeration + var meter = new Meter(options.Name, options.Version, tags, this); var meterList = _meterCache.GetOrNew(options.Name); - meterList.Add(new CachedMeter(options.Version, TagsToDict(options.Tags), meter)); + // ReSharper disable once PossibleMultipleEnumeration + meterList.Add(new CachedMeter(options.Version, TagsToDict(tags), meter)); return meter; } } @@ -66,7 +98,12 @@ private static bool TagsMatch(Dictionary a, Dictionary Entities.Count); + } +} diff --git a/Robust.Shared/GameObjects/EntityManager.cs b/Robust.Shared/GameObjects/EntityManager.cs index 4c8e1d5e84a..5da6a3f5bb4 100644 --- a/Robust.Shared/GameObjects/EntityManager.cs +++ b/Robust.Shared/GameObjects/EntityManager.cs @@ -142,6 +142,8 @@ public virtual void Initialize() if (Initialized) throw new InvalidOperationException("Initialize() called multiple times"); + InitMetrics(); + EventBusInternal = new EntityEventBus(this, _reflection); InitializeComponents(); diff --git a/Robust.UnitTesting/Pool/PoolManager.cs b/Robust.UnitTesting/Pool/PoolManager.cs index df2ab432315..cac7dee27cb 100644 --- a/Robust.UnitTesting/Pool/PoolManager.cs +++ b/Robust.UnitTesting/Pool/PoolManager.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.Metrics; using System.IO; using System.Linq; using System.Reflection; @@ -15,6 +16,8 @@ namespace Robust.UnitTesting.Pool; public abstract class BasePoolManager { + internal static readonly Meter MetricsMeter = new("Robust.UnitTesting.PoolManager"); + internal abstract void Return(ITestPair pair); public abstract Assembly[] ClientAssemblies { get; } public abstract Assembly[] ServerAssemblies { get; } @@ -51,6 +54,16 @@ public abstract class BasePoolManager public override Assembly[] ClientAssemblies => _clientAssemblies; public override Assembly[] ServerAssemblies => _serverAssemblies; + protected PoolManager() + { + MetricsMeter.CreateObservableUpDownCounter( + "pair_count", + MeasurePairCount, + null, + null, + tags: [new KeyValuePair("type", typeof(TPair).FullName)]); + } + /// /// Initialize the pool manager. Override this to configure what assemblies should get loaded. /// @@ -164,7 +177,7 @@ public async Task GetPair( { await testOut.WriteLineAsync( $"{nameof(GetPair)}: Creating pair, because settings of pool settings"); - pair = await CreateServerClientPair(settings, testOut); + pair = await CreateServerClientPair(settings, testOut, "MustBeNew", currentTestName); } else { @@ -192,7 +205,7 @@ await testOut.WriteLineAsync( else { await testOut.WriteLineAsync($"{nameof(GetPair)}: Creating a new pair, no suitable pair found in pool"); - pair = await CreateServerClientPair(settings, testOut); + pair = await CreateServerClientPair(settings, testOut, "NoneInPool", currentTestName); } } } @@ -211,6 +224,11 @@ await testOut.WriteLineAsync( await testOut.WriteLineAsync($"{nameof(GetPair)}: Retrieving pair {pair.Id} from pool took {watch.Elapsed.TotalMilliseconds} ms"); + PoolManagerEvents.Log.PairRetrieved( + typeof(TPair).FullName!, + pair.Id, + currentTestName); + pair.ValidateSettings(settings); pair.ClearModifiedCvars(); pair.Settings = settings; @@ -289,16 +307,22 @@ private void DieIfPoolFailure() } } - private async Task CreateServerClientPair(PairSettings settings, TextWriter testOut) + private async Task CreateServerClientPair( + PairSettings settings, + TextWriter testOut, + string reason, + string testName) { try { var id = Interlocked.Increment(ref _nextPairId); var pair = new TPair(); + PoolManagerEvents.Log.PairCreated(typeof(TPair).FullName!, id, reason, testName); await pair.Init(id, this, settings, testOut); pair.Use(); await pair.RunTicksSync(5); await pair.SyncTicks(targetDelta: 1); + PoolManagerEvents.Log.PairFinishedInit(typeof(TPair).FullName!, id); return pair; } catch (Exception ex) @@ -330,4 +354,23 @@ private void DiscoverTestPrototypes(Assembly assembly) } } } + + private IEnumerable> MeasurePairCount() + { + var inUse = 0; + var notInUse = 0; + lock (_pairLock) + { + foreach (var useBool in Pairs.Values) + { + if (useBool) + inUse += 1; + else + notInUse += 1; + } + } + + yield return new Measurement(inUse, new KeyValuePair("in_use", "true")); + yield return new Measurement(notInUse, new KeyValuePair("in_use", "false")); + } } diff --git a/Robust.UnitTesting/Pool/PoolManagerEvents.cs b/Robust.UnitTesting/Pool/PoolManagerEvents.cs new file mode 100644 index 00000000000..7b5c789c255 --- /dev/null +++ b/Robust.UnitTesting/Pool/PoolManagerEvents.cs @@ -0,0 +1,68 @@ +using System.Diagnostics.Tracing; + +namespace Robust.UnitTesting.Pool; + +[EventSource(Name = "Robust.UnitTesting.PoolManagerEvents")] +internal sealed class PoolManagerEvents : EventSource +{ + public static readonly PoolManagerEvents Log = new(); + + [Event(1)] + public void PairCreated(string type, int id, string reason, string testName) => WriteEvent(1, type, id, reason, testName); + + [Event(2)] + public void PairFinishedInit(string type, int id) => WriteEvent(2, type, id); + + [Event(3)] + public void PairDestroyed(string type, int id) => WriteEvent(3, type, id); + + [Event(4)] + public void PairCleanReturned(string type, int id) => WriteEvent(4, type, id); + + [Event(5)] + public void PairDirtyReturned(string type, int id) => WriteEvent(5, type, id); + + [Event(6)] + public void PairRetrieved(string type, int id, string testName) => WriteEvent(6, type, id, testName); + + [NonEvent] + private unsafe void WriteEvent(int eventId, string arg1, int arg2, string arg3) + { + fixed (char* arg1Ptr = arg1) + fixed (char* arg3Ptr = arg3) + { + var dataDesc = stackalloc EventData[3]; + + dataDesc[0].DataPointer = (nint)arg1Ptr; + dataDesc[0].Size = (arg1.Length + 1) * 2; + dataDesc[1].DataPointer = (nint)(&arg2); + dataDesc[1].Size = 4; + dataDesc[2].DataPointer = (nint)arg3Ptr; + dataDesc[2].Size = (arg3.Length + 1) * 2; + + WriteEventCore(eventId, 3, dataDesc); + } + } + + [NonEvent] + private unsafe void WriteEvent(int eventId, string arg1, int arg2, string arg3, string arg4) + { + fixed (char* arg1Ptr = arg1) + fixed (char* arg3Ptr = arg3) + fixed (char* arg4Ptr = arg4) + { + var dataDesc = stackalloc EventData[4]; + + dataDesc[0].DataPointer = (nint)arg1Ptr; + dataDesc[0].Size = (arg1.Length + 1) * 2; + dataDesc[1].DataPointer = (nint)(&arg2); + dataDesc[1].Size = 4; + dataDesc[2].DataPointer = (nint)arg3Ptr; + dataDesc[2].Size = (arg3.Length + 1) * 2; + dataDesc[3].DataPointer = (nint)arg4Ptr; + dataDesc[3].Size = (arg4.Length + 1) * 2; + + WriteEventCore(eventId, 4, dataDesc); + } + } +} diff --git a/Robust.UnitTesting/Pool/TestPair.Recycle.cs b/Robust.UnitTesting/Pool/TestPair.Recycle.cs index 0d43685af09..a62efbb15b0 100644 --- a/Robust.UnitTesting/Pool/TestPair.Recycle.cs +++ b/Robust.UnitTesting/Pool/TestPair.Recycle.cs @@ -88,6 +88,8 @@ public async ValueTask CleanReturnAsync() if (State != PairState.InUse) throw new Exception($"{nameof(CleanReturnAsync)}: Unexpected state. Pair: {Id}. State: {State}."); + PoolManagerEvents.Log.PairCleanReturned(GetType().FullName!, Id); + await TestOut.WriteLineAsync($"{nameof(CleanReturnAsync)}: Return of pair {Id} started"); State = PairState.CleanDisposed; await OnCleanDispose(); @@ -104,6 +106,8 @@ public async ValueTask DisposeAsync() case PairState.Ready: break; case PairState.InUse: + PoolManagerEvents.Log.PairDirtyReturned(GetType().FullName!, Id); + await TestOut.WriteLineAsync($"{nameof(DisposeAsync)}: Dirty return of pair {Id} started"); await OnDirtyDispose(); Manager.Return(this); diff --git a/Robust.UnitTesting/Pool/TestPair.cs b/Robust.UnitTesting/Pool/TestPair.cs index 0ccd0e95b1e..2f3705c1940 100644 --- a/Robust.UnitTesting/Pool/TestPair.cs +++ b/Robust.UnitTesting/Pool/TestPair.cs @@ -107,6 +107,8 @@ protected virtual Task Initialize() public void Kill() { + PoolManagerEvents.Log.PairDestroyed(GetType().FullName!, Id); + State = PairState.Dead; ServerLogHandler.ShuttingDown = true; ClientLogHandler.ShuttingDown = true; diff --git a/Robust.UnitTesting/RobustIntegrationTest.TestPair.cs b/Robust.UnitTesting/RobustIntegrationTest.TestPair.cs index a2bb7f552e1..98f32776eb9 100644 --- a/Robust.UnitTesting/RobustIntegrationTest.TestPair.cs +++ b/Robust.UnitTesting/RobustIntegrationTest.TestPair.cs @@ -1,4 +1,5 @@ using System.Threading.Tasks; +using Robust.Shared; using Robust.Shared.Log; using Robust.UnitTesting.Pool; @@ -50,6 +51,8 @@ protected virtual ClientIntegrationOptions ClientOptions() options.CVarOverrides[cvar] = value; } + options.CVarOverrides[CVars.MetricsInstanceName.Name] = $"{GetType().FullName}_client_{Id}"; + return options; } @@ -72,6 +75,8 @@ protected virtual ServerIntegrationOptions ServerOptions() options.CVarOverrides[cvar] = value; } + options.CVarOverrides[CVars.MetricsInstanceName.Name] = $"{GetType().FullName}_server_{Id}"; + return options; } }