diff --git a/DevOps.Util.DotNet/DotNetQueryUtil.cs b/DevOps.Util.DotNet/DotNetQueryUtil.cs index 66cc972..b66cc0d 100644 --- a/DevOps.Util.DotNet/DotNetQueryUtil.cs +++ b/DevOps.Util.DotNet/DotNetQueryUtil.cs @@ -655,12 +655,12 @@ async Task GetJobMachineInfoAsync() foreach (var mapping in document.AllNodes.OfType()) { - if (!TryGetScalarValue(mapping, "job", out var jobName)) + if (!mapping.TryGetScalarValue("job", out var jobName)) { continue; } - if (TryGetScalarValue(mapping, "displayName", out var displayNameValue)) + if (mapping.TryGetScalarValue("displayName", out var displayNameValue)) { jobName = displayNameValue; } @@ -671,20 +671,20 @@ async Task GetJobMachineInfoAsync() } string? container = null; - if (TryGetNode(mapping, "container", out var containerNode) && - TryGetScalarValue(containerNode, "alias", out var alias)) + if (mapping.TryGetNode("container", out var containerNode) && + containerNode.TryGetScalarValue("alias", out var alias)) { container = alias; } string? queue = null; - if (TryGetNode(mapping, "pool", out var poolNode)) + if (mapping.TryGetNode("pool", out var poolNode)) { - if (TryGetScalarValue(poolNode, "vmImage", out var vmName)) + if (poolNode.TryGetScalarValue("vmImage", out var vmName)) { queue = vmName; } - else if (TryGetScalarValue(poolNode, "queue", out var queueName)) + else if (poolNode.TryGetScalarValue("queue", out var queueName)) { queue = queueName; } @@ -704,32 +704,6 @@ async Task GetJobMachineInfoAsync() isHelixSubmission: false)); } } - - bool TryGetNode(YamlMappingNode node, string name, out T childNode) - where T : YamlNode - { - if (node.Children.TryGetValue(name, out var n) && - n is T t) - { - childNode = t; - return true; - } - - childNode = null!; - return false; - } - - bool TryGetScalarValue(YamlMappingNode node, string name, out string value) - { - if (TryGetNode(node, name, out var scalarNode)) - { - value = scalarNode.Value!; - return true; - } - - value = null!; - return false; - } } async Task GetHelixMachineInfoAsync() diff --git a/DevOps.Util.DotNet/Extensions.cs b/DevOps.Util.DotNet/Extensions.cs index 45f0821..661a787 100644 --- a/DevOps.Util.DotNet/Extensions.cs +++ b/DevOps.Util.DotNet/Extensions.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Runtime.CompilerServices; using System.Threading.Tasks; +using YamlDotNet.RepresentationModel; namespace DevOps.Util.DotNet { @@ -202,5 +203,35 @@ public static async Task> GetHelixMa }; #endregion + + #region Yaml + + public static bool TryGetNode(this YamlMappingNode node, string name, out T childNode) + where T : YamlNode + { + if (node.Children.TryGetValue(name, out var n) && + n is T t) + { + childNode = t; + return true; + } + + childNode = null!; + return false; + } + + public static bool TryGetScalarValue(this YamlMappingNode node, string name, out string value) + { + if (TryGetNode(node, name, out var scalarNode)) + { + value = scalarNode.Value!; + return true; + } + + value = null!; + return false; + } + + #endregion } } diff --git a/scratch/Program.cs b/scratch/Program.cs index 55a4e2e..786bc22 100644 --- a/scratch/Program.cs +++ b/scratch/Program.cs @@ -24,10 +24,13 @@ using System.Linq; using System.Linq.Expressions; using System.Net.Http; +using System.Security.Principal; using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using YamlDotNet.Core; +using YamlDotNet.RepresentationModel; // [assembly: Microsoft.Extensions.Configuration.UserSecrets.UserSecretsId("67c4a872-5dd7-422a-acad-fdbe907ace33")] @@ -167,10 +170,143 @@ internal static IConfiguration CreateConfiguration(bool useProduction = false) internal async Task Scratch() { - await DeleteOldBuilds(useProduction: true); + await GetHelixWaitTimes(); // await FindMatchingSdkMissingBuilds(); // await FindLogMacBuildAsync(); } + + internal async Task GetHelixWaitTimes() + { + var project = "public"; + var definition = 686; + var count = 0; + var map = new Dictionary>(); + + await foreach (var build in DevOpsServer.EnumerateBuildsAsync(project, definitions: new[] { definition }, statusFilter: BuildStatus.Completed)) + { + var buildInfo = build.GetBuildInfo(); + try + { + var timeline = await DevOpsServer.GetTimelineAsync(project, buildInfo.Number); + if (timeline is null) + { + continue; + } + + foreach (var (jobName, logId) in GetHelixLogIds(timeline)) + { + if (await GetHelixWaitTimeAsync(logId) is not { } time) + { + continue; + } + + if (!map.TryGetValue(jobName, out var list)) + { + list = new(); + map[jobName] = list; + } + + list.Add(time); + } + + if (++count == 100) + { + break; + } + + } + catch (Exception ex) + { + Console.WriteLine($"Error processing build: {buildInfo.BuildUri}"); + Console.WriteLine(ex.Message); + } + + async Task GetHelixWaitTimeAsync(int logId) + { + try + { + var content = await DevOpsServer.GetBuildLogAsync(project, buildInfo.Number, logId); + using var reader = new StringReader(content); + DateTime? start = null; ; + while (reader.ReadLine() is { } line) + { + var parts = line.Split(new[] { ' ' }, count: 2); + var message = parts[1].Trim(); + if (start is null && message.StartsWith("Starting Azure Pipelines Test")) + { + start = DateTime.Parse(parts[0]); + } + else if (message.StartsWith("Stopping Azure Pipelines Test")) + { + var end = DateTime.Parse(parts[0]); + return end - start; + } + } + + return null; + } + catch + { + return null; + } + } + + IEnumerable<(string, int)> GetHelixLogIds(Timeline timeline) + { + var tree = TimelineTree.Create(timeline); + if (definition == 15) + { + var helixNodes = tree + .JobNodes + .Where(x => x.Name.StartsWith("Test_")) + .ToList(); + foreach (var jobNode in helixNodes) + { + if (jobNode.TimelineRecord?.Log?.Id is not { } logId) + { + continue; + } + + yield return (jobNode.Name, logId); + } + } + else if (definition == 686) + { + foreach (var record in timeline.Records) + { + if (record.Name != "Send to Helix" || + record.Log?.Id is not { } logId || + !tree.TryGetJob(record, out var jobNode)) + { + continue; + } + + yield return (jobNode.Name, logId); + } + } + else + { + throw new Exception("Unknown build definition"); + } + } + } + + Console.WriteLine("|Job Name|Average Wasted Time (minutes)| Total Wasted Time (minutes)|"); + Console.WriteLine("|---|---|---|"); + foreach (var pair in map) + { + var average = pair.Value.Average(x => x.TotalSeconds) / 60; + var total = pair.Value.Sum(x => x.TotalSeconds) / 60; + Console.WriteLine($"|{pair.Key}|{average:N2}|{total:N2}|"); + } + + var all = map + .Values + .SelectMany(x => x) + .Sum(x => x.TotalSeconds) / 60; + Console.WriteLine($"Total Wasted minutes {all:N2}"); + } + internal async Task FindMatchingSdkMissingBuilds() { var count = 0;