diff --git a/DevOps.Functions/Functions.cs b/DevOps.Functions/Functions.cs index a8230e0..b9a13ce 100644 --- a/DevOps.Functions/Functions.cs +++ b/DevOps.Functions/Functions.cs @@ -33,10 +33,10 @@ public class Functions public TriageContextUtil TriageContextUtil { get; } public DevOpsServer Server { get; } public HelixServer HelixServer { get; } - public GitHubClientFactory GitHubClientFactory { get; } + public IGitHubClientFactory GitHubClientFactory { get; } public SiteLinkUtil SiteLinkUtil { get; } - public Functions(DevOpsServer server, TriageContext context, GitHubClientFactory gitHubClientFactory) + public Functions(DevOpsServer server, TriageContext context, IGitHubClientFactory gitHubClientFactory) { Server = server; Context = context; diff --git a/DevOps.Functions/Startup.cs b/DevOps.Functions/Startup.cs index 9c57e56..3ac3cde 100644 --- a/DevOps.Functions/Startup.cs +++ b/DevOps.Functions/Startup.cs @@ -31,11 +31,11 @@ public override void Configure(IFunctionsHostBuilder builder) new DevOpsServer( DotNetConstants.AzureOrganization, new AuthorizationToken(AuthorizationKind.PersonalAccessToken, azdoToken))); - builder.Services.AddScoped(_ => + builder.Services.AddScoped(_ => { var appId = int.Parse(config[DotNetConstants.ConfigurationGitHubAppId]); var appPrivateKey = config[DotNetConstants.ConfigurationGitHubAppPrivateKey]; - return new GitHubClientFactory(appId, appPrivateKey); + return new GitHubAppClientFactory(appId, appPrivateKey); }); } } diff --git a/DevOps.Status/DevOps.Status.csproj b/DevOps.Status/DevOps.Status.csproj index ad91533..fb25148 100644 --- a/DevOps.Status/DevOps.Status.csproj +++ b/DevOps.Status/DevOps.Status.csproj @@ -8,6 +8,10 @@ + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/DevOps.Status/Program.cs b/DevOps.Status/Program.cs index 2873c7e..1ca6d5e 100644 --- a/DevOps.Status/Program.cs +++ b/DevOps.Status/Program.cs @@ -32,9 +32,17 @@ public static IHostBuilder CreateHostBuilder(string[] args) => } else { - config.AddAzureKeyVault( - DotNetConstants.KeyVaultEndPoint, - new DefaultKeyVaultSecretManager()); + // We can disable using key vault with this env var. See ..\Documentation\DevWithoutKeyVault.md + if (Environment.GetEnvironmentVariable("USE_KEYVAULT") == "0") + { + Console.WriteLine("Disabling Azure KeyVault"); + } + else + { + config.AddAzureKeyVault( + DotNetConstants.KeyVaultEndPoint, + new DefaultKeyVaultSecretManager()); + } } }) .ConfigureWebHostDefaults(webBuilder => diff --git a/DevOps.Status/Startup.cs b/DevOps.Status/Startup.cs index 32c6726..78be23b 100644 --- a/DevOps.Status/Startup.cs +++ b/DevOps.Status/Startup.cs @@ -14,6 +14,7 @@ using Microsoft.AspNetCore.Authentication.Cookies; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.HttpsPolicy; using Microsoft.AspNetCore.Routing; using Microsoft.EntityFrameworkCore; @@ -55,13 +56,13 @@ public void ConfigureServices(IServiceCollection services) services.AddHttpContextAccessor(); services.AddScoped(); - services.AddScoped(_ => new GitHubClientFactory(Configuration)); + services.AddScoped(_ => GitHubClientFactory.Create(Configuration)); services.AddScoped(_ => new FunctionQueueUtil(Configuration[DotNetConstants.ConfigurationAzureBlobConnectionString])); services.AddDbContext(options => { - var connectionString = Configuration[DotNetConstants.ConfigurationSqlConnectionString]; + var connectionString = Configuration.GetNonNull(DotNetConstants.ConfigurationSqlConnectionString); #if DEBUG options.UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole())); options.EnableSensitiveDataLogging(); @@ -82,8 +83,14 @@ public void ConfigureServices(IServiceCollection services) }) .AddGitHub(options => { - options.ClientId = Configuration[DotNetConstants.ConfigurationGitHubClientId]; - options.ClientSecret = Configuration[DotNetConstants.ConfigurationGitHubClientSecret]; + // If we are going to impersonate the logged in user rather than use the GH app + // then we need some additional permissions to manage GH issues + if (Configuration[DotNetConstants.ConfigurationGitHubImpersonateUser] == "true") + { + options.Scope.Add("public_repo"); + } + options.ClientId = Configuration.GetNonNull(DotNetConstants.ConfigurationGitHubClientId); + options.ClientSecret = Configuration.GetNonNull(DotNetConstants.ConfigurationGitHubClientSecret); options.SaveTokens = true; options.ClaimActions.MapJsonKey(Constants.GitHubAvatarUrl, Constants.GitHubAvatarUrl); @@ -132,6 +139,21 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env) app.UseAuthentication(); app.UseAuthorization(); + // At the time the ClientFactory was created we hadn't done authentication yet. Now that we have + // associate the user's GH access token with the factory in case we need it to impersonate them. + // This is only necessary when the app is configured to use the logged in user's identity rather + // than the dedicated GH runfo app identity. + app.Use(async (context, next) => + { + if(context.User.Identity.IsAuthenticated) + { + string accessToken = await context.GetTokenAsync("access_token"); + IGitHubClientFactory gitHubClientFactory = context.RequestServices.GetService(); + gitHubClientFactory.SetUserOAuthToken(accessToken); + } + await next(); + }); + app.UseEndpoints(endpoints => { endpoints.MapRazorPages(); diff --git a/DevOps.Util.DotNet/DotNetConstants.cs b/DevOps.Util.DotNet/DotNetConstants.cs index f486541..1108b9e 100644 --- a/DevOps.Util.DotNet/DotNetConstants.cs +++ b/DevOps.Util.DotNet/DotNetConstants.cs @@ -16,6 +16,7 @@ public static class DotNetConstants public const string ConfigurationSqlConnectionString = "RunfoConnectionString"; public const string ConfigurationAzdoToken = "RunfoAzdoToken"; + public const string ConfigurationGitHubImpersonateUser = "GitHubImpersonateUser"; public const string ConfigurationGitHubAppId = "GitHubAppId"; public const string ConfigurationGitHubAppPrivateKey = "GitHubAppPrivateKey"; public const string ConfigurationGitHubClientId = "GitHubClientId"; diff --git a/DevOps.Util.DotNet/Extensions.cs b/DevOps.Util.DotNet/Extensions.cs index 661a787..d9bffe9 100644 --- a/DevOps.Util.DotNet/Extensions.cs +++ b/DevOps.Util.DotNet/Extensions.cs @@ -1,4 +1,5 @@ using Microsoft.DotNet.Helix.Client; +using Microsoft.Extensions.Configuration; using Octokit; using System; using System.Collections.Generic; @@ -233,5 +234,16 @@ public static bool TryGetScalarValue(this YamlMappingNode node, string name, out } #endregion + + #region IConfiguration + public static string GetNonNull(this IConfiguration config, string key) + { + if (config[key]==null) + { + throw new Exception($"No {key} configuration set"); + } + return config[key]; + } + #endregion } } diff --git a/DevOps.Util.DotNet/GitHubClientFactory.cs b/DevOps.Util.DotNet/GitHubClientFactory.cs index c922e50..d19e4f5 100644 --- a/DevOps.Util.DotNet/GitHubClientFactory.cs +++ b/DevOps.Util.DotNet/GitHubClientFactory.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.IO; using System.Net; +using System.Security.Claims; using System.Text; using System.Threading.Tasks; @@ -12,26 +13,42 @@ namespace DevOps.Util.DotNet { public interface IGitHubClientFactory { + void SetUserOAuthToken(string token); Task CreateForAppAsync(string owner, string repository); } - public sealed class GitHubClientFactory : IGitHubClientFactory + public sealed class OAuthAppClientFactory : IGitHubClientFactory { - public const string GitHubProductName = "runfo.azurewebsites.net"; + string _oauthToken; - private string AppPrivateKey { get; } - public int AppId { get; } + #region IGitHubClientFactory - public GitHubClientFactory(IConfiguration configuration) + public void SetUserOAuthToken(string token) { - AppId = int.Parse(configuration[DotNetConstants.ConfigurationGitHubAppId]); - AppPrivateKey = configuration[DotNetConstants.ConfigurationGitHubAppPrivateKey]; + _oauthToken = token; } - public GitHubClientFactory(int appId, string privateKey) + async Task IGitHubClientFactory.CreateForAppAsync(string owner, string repository) + { + if(_oauthToken == null) + { + throw new Exception("This action requires the user to be logged in via GitHub first"); + } + return GitHubClientFactory.CreateForToken(_oauthToken, AuthenticationType.Oauth); + } + + #endregion + } + + public sealed class GitHubAppClientFactory : IGitHubClientFactory + { + private string AppPrivateKey { get; } + public int AppId { get; } + + public GitHubAppClientFactory(int appId, string appPrivateKey) { AppId = appId; - AppPrivateKey = privateKey; + AppPrivateKey = appPrivateKey; } public async Task CreateForAppAsync(string owner, string repository) @@ -39,7 +56,7 @@ public async Task CreateForAppAsync(string owner, string repositor var gitHubClient = CreateForAppCore(); var installation = await gitHubClient.GitHubApps.GetRepositoryInstallationForCurrent(owner, repository).ConfigureAwait(false); var installationToken = await gitHubClient.GitHubApps.CreateInstallationToken(installation.Id); - return CreateForToken(installationToken.Token, AuthenticationType.Oauth); + return GitHubClientFactory.CreateForToken(installationToken.Token, AuthenticationType.Oauth); } private GitHubClient CreateForAppCore() @@ -52,28 +69,18 @@ private GitHubClient CreateForAppCore() new GitHubJwtFactoryOptions { AppIntegrationId = AppId, - ExpirationSeconds = 600 + ExpirationSeconds = 600 }); var token = generator.CreateEncodedJwtToken(); - return CreateForToken(token, AuthenticationType.Bearer); - } - - public static GitHubClient CreateAnonymous() => new GitHubClient(new ProductHeaderValue(GitHubProductName)); - - public static GitHubClient CreateForToken(string token, AuthenticationType authenticationType) - { - var productInformation = new ProductHeaderValue(GitHubProductName); - var client = new GitHubClient(productInformation) - { - Credentials = new Credentials(token, authenticationType) - }; - return client; + return GitHubClientFactory.CreateForToken(token, AuthenticationType.Bearer); } #region IGitHubClientFactory + public void SetUserOAuthToken(string token) { } + async Task IGitHubClientFactory.CreateForAppAsync(string owner, string repository) => await CreateForAppAsync(owner, repository).ConfigureAwait(false); @@ -91,4 +98,34 @@ internal PlainStringPrivateKeySource(string key) public TextReader GetPrivateKeyReader() => new StringReader(_key); } } + + public static class GitHubClientFactory + { + public const string GitHubProductName = "runfo.azurewebsites.net"; + + public static GitHubClient CreateAnonymous() => new GitHubClient(new ProductHeaderValue(GitHubProductName)); + + public static GitHubClient CreateForToken(string token, AuthenticationType authenticationType) + { + var productInformation = new ProductHeaderValue(GitHubProductName); + var client = new GitHubClient(productInformation) + { + Credentials = new Credentials(token, authenticationType) + }; + return client; + } + + public static IGitHubClientFactory Create(IConfiguration configuration) + { + if (configuration[DotNetConstants.ConfigurationGitHubImpersonateUser] == "true") + { + return new OAuthAppClientFactory(); + } + else + { + return new GitHubAppClientFactory(int.Parse(configuration.GetNonNull(DotNetConstants.ConfigurationGitHubAppId)), + configuration.GetNonNull(DotNetConstants.ConfigurationGitHubAppPrivateKey)); + } + } + } } diff --git a/Documentation/DevWithoutKeyVault.md b/Documentation/DevWithoutKeyVault.md new file mode 100644 index 0000000..ada21a7 --- /dev/null +++ b/Documentation/DevWithoutKeyVault.md @@ -0,0 +1,96 @@ +# Development without Azure KeyVault + +Several of the runfo apps store secrets in Azure Key Vault and the apps will load configuration from there by default. +However if your user account doesn't have access to the Azure Key Vault or for whatever other reason you don't want to use +Key Vault there is an alternative approach. This will require you to provide various tokens and connection strings +manually. At present only the DevOps.Status and scratch project are coded to support this but it wouldn't be hard to add it +to others. + +## 1. GitHub tokens + +Runfo uses OAuth to authenticate the current user to GitHub which requires creating an app registration. +Follow the [instructions from GitHub](https://docs.github.com/en/developers/apps/building-oauth-apps/creating-an-oauth-app) +to create one. The app name, app description and web page don't matter other than maybe needing to be unique. They will just +be displayed back to you later during login. The callback URL should point to the /signin-github page where you will host +the runfo web app. For example: + - Name: Runfo_Noah_Local_Dev + - URL: https://github.com/noahfalk/runfo + - Description: blank + - Callback URL: https://localhost:5001/signin-github + + After registering you will get a client id and client secret, save these for later. + + By default Runfo also uses a GitHub App (OAuth apps and GH apps are not the same thing) which it uses for managing + issues. If you don't have Azure Key Vault access you probably don't have access to the AppId and AppSecret tokens for the + GitHub app either. There is a GitHubImpersonateUser configuration option we'll set later which avoids using the GH app and instead + will impersonate your logged in GH identity when making changes to issues. + +## 2. Runfo Database + +Runfo uses a database to store tracking issues and caching data from AzDo and Helix. You will either need to obtain a connection +string for the live test/prod database or set up your own. To set one up I recommend installing the Developer edition of SQL +and include the Full Text Indexing optional feature. You can't use SQL Express or the LocalDB that comes with VS because these +editions do not support Full Text Search. + +## 3. AzDo Personal Access Token + +Get a PAT for your user account at https://dev.azure.com/dnceng. See the +[AzDo docs](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows) +for more details. Select Read access for the Work Items, Code, Build, Release, and Test Management scopes. (I guessed at these +permissions and they appear to work but it might be more than is needed). + +Save the PAT for later. + +## 4. Create User Secrets + +DotNet supports saving per-user secrets in a file called secrets.json that is separate from your project source so that +you don't accidentally check it in. You can create the file using Visual Studio's 'Manage User Secrets' or via command-line using +dotnet user-secrets. See [the docs](https://docs.microsoft.com/en-us/aspnet/core/security/app-secrets?view=aspnetcore-6.0&tabs=windows) +for more details. + +Edit the secrets.json file so it looks like the one below. Use the client id and client secret from step 1, whatever connection +string is appropriate for the database you set up in step 2, and RunfoAzdoToken is the token from step 3. + +``` +{ + "GitHubClientId": "11122233344455566677", + "GitHubClientSecret": "11112222333334444555666777888999aaabbbcc", + "RunfoAzdoToken": "abcdefghijklmnopqrstuvwxyz1234567890abcdefghijklmnop", + "RunfoConnectionString": "Server=localhost;Database=TriageContext;Trusted_Connection=True;", + "GitHubImpersonateUser": "true" +} +``` + +GitHubImpersonateUser isn't really a secret, but it was convenient to set it here because these are per-user settings that won't +get checked in. + +## 5. Set the environment variable USE_KEYVAULT=0 + +Program.cs for the DevOps.Status and scratch projects will check for this env var and avoid using Azure Key Vault in the configuration +setup. The config values that normally would have come from key vault will come from the secrets.json instead. + +## 6. Create the Runfo TriageContext database if needed + +If you installed a new SQL server instance in step 2 then we need to initialize Runfo's database. Navigate to the scratch +source folder. + +If needed install the ef command line tools: + +``` +dotnet tool install -g dotnet-ef +``` + +Once the tools are installed run this command to initialize the database schema: + +``` +dotnet ef database update +``` + +Then run the scratch project with argument "populateDb". This will take a long time as scratch queries AzDo and Helix +caching lots of data into the new runfo database. You can quit part way through if you don't care about getting +complete data. + +## 7. Start developing + +You should now be able to run the runfo apps and have things mostly function. Use dotnet run or launch from VS with F5. +If you use the debugger make sure USE_KEYVAULT=0 is included in your launch profile. diff --git a/scratch/Program.cs b/scratch/Program.cs index f548d39..3b51639 100644 --- a/scratch/Program.cs +++ b/scratch/Program.cs @@ -40,7 +40,17 @@ public class Program public static async Task Main(string[] args) { var scratchUtil = new ScratchUtil(); - await scratchUtil.Scratch(); + if(args.Length > 0) + { + if (args[0].Equals("populateDb", StringComparison.OrdinalIgnoreCase)) + { + await scratchUtil.PopulateDb(); + } + } + else + { + await scratchUtil.Scratch(); + } } // This entry point exists so that `dotnet ef database` and `migrations` has an @@ -85,6 +95,7 @@ public FakeGitHubClientFactory(GitHubClient gitHubClient) GitHubClient = gitHubClient; } + public void SetUserOAuthToken(string token) { } public Task CreateForAppAsync(string owner, string repository) => Task.FromResult(GitHubClient); } @@ -111,11 +122,11 @@ public ScratchUtil() public void Reset(string organization = DefaultOrganization, bool useProduction = false) { var configuration = CreateConfiguration(useProduction); - var azureToken = configuration[DotNetConstants.ConfigurationAzdoToken]; + var azureToken = configuration.GetNonNull(DotNetConstants.ConfigurationAzdoToken); DevOpsServer = new DevOpsServer(organization, new AuthorizationToken(AuthorizationKind.PersonalAccessToken, azureToken)); var builder = new DbContextOptionsBuilder(); - var connectionString = configuration[DotNetConstants.ConfigurationSqlConnectionString]; + var connectionString = configuration.GetNonNull(DotNetConstants.ConfigurationSqlConnectionString); var message = connectionString.Contains("triage-scratch-dev") ? "Using sql developer" : "Using sql production"; @@ -148,12 +159,14 @@ internal static IConfiguration CreateConfiguration(bool useProduction = false) { var keyVault = useProduction ? DotNetConstants.KeyVaultEndPointProduction : DotNetConstants.KeyVaultEndPointTest; var config = new ConfigurationBuilder() - .AddUserSecrets() - .AddAzureKeyVault( - keyVault, - new DefaultKeyVaultSecretManager()) - .Build(); - return config; + .AddUserSecrets(); + + // We can disable using key vault with this env var. See ..\Documentation\DevWithoutKeyVault.md + if (Environment.GetEnvironmentVariable("USE_KEYVAULT") != "0") + { + config = config.AddAzureKeyVault(keyVault, new DefaultKeyVaultSecretManager()); + } + return config.Build(); } internal static ILogger CreateLogger() => LoggerFactory.Create(builder => builder.AddConsole()).CreateLogger("Scratch");