Enable bypassing Azure Key Vault

DevOps.Functions/Functions.cs

@@ -33,10 +33,10 @@ public class Functions
         public TriageContextUtil TriageContextUtil { get; }
         public DevOpsServer Server { get; }
         public HelixServer HelixServer { get; }
-        public GitHubClientFactory GitHubClientFactory { get; }
+        public IGitHubClientFactory GitHubClientFactory { get; }
         public SiteLinkUtil SiteLinkUtil { get; }
 
-        public Functions(DevOpsServer server, TriageContext context, GitHubClientFactory gitHubClientFactory)
+        public Functions(DevOpsServer server, TriageContext context, IGitHubClientFactory gitHubClientFactory)
         {
             Server = server;
             Context = context;

DevOps.Functions/Startup.cs

@@ -31,11 +31,11 @@ public override void Configure(IFunctionsHostBuilder builder)
                 new DevOpsServer(
                     DotNetConstants.AzureOrganization,
                     new AuthorizationToken(AuthorizationKind.PersonalAccessToken, azdoToken)));
-            builder.Services.AddScoped<GitHubClientFactory>(_ =>
+            builder.Services.AddScoped<IGitHubClientFactory>(_ =>
             {
                 var appId = int.Parse(config[DotNetConstants.ConfigurationGitHubAppId]);
                 var appPrivateKey = config[DotNetConstants.ConfigurationGitHubAppPrivateKey];
-                return new GitHubClientFactory(appId, appPrivateKey);
+                return new GitHubAppClientFactory(appId, appPrivateKey);
             });
         }
     }

DevOps.Status/DevOps.Status.csproj

@@ -8,6 +8,10 @@
     <PackageReference Include="AspNet.Security.OAuth.GitHub" Version="3.1.2" />
     <PackageReference Include="Azure.Identity" Version="1.4.0" />
     <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.2.0" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="3.1.3">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageReference>
     <PackageReference Include="Microsoft.Extensions.Configuration.AzureKeyVault" Version="3.1.17" />
     <PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="3.1.4" />
     <PackageReference Include="Westwind.AspNetCore.Markdown" Version="3.4.0" />

DevOps.Status/Program.cs

@@ -32,9 +32,17 @@ public static IHostBuilder CreateHostBuilder(string[] args) =>
                     }
                     else
                     {
-                        config.AddAzureKeyVault(
-                            DotNetConstants.KeyVaultEndPoint,
-                            new DefaultKeyVaultSecretManager());
+                        // We can disable using key vault with this env var. See ..\Documentation\DevWithoutKeyVault.md
+                        if (Environment.GetEnvironmentVariable("USE_KEYVAULT") == "0")
+                        {
+                            Console.WriteLine("Disabling Azure KeyVault");
+                        }
+                        else
+                        {
+                            config.AddAzureKeyVault(
+                                DotNetConstants.KeyVaultEndPoint,
+                                new DefaultKeyVaultSecretManager());
+                        }
                     }
                 })
                 .ConfigureWebHostDefaults(webBuilder =>

DevOps.Status/Startup.cs

@@ -14,6 +14,7 @@
 using Microsoft.AspNetCore.Authentication.Cookies;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.HttpsPolicy;
 using Microsoft.AspNetCore.Routing;
 using Microsoft.EntityFrameworkCore;
@@ -55,13 +56,13 @@ public void ConfigureServices(IServiceCollection services)
 
             services.AddHttpContextAccessor();
             services.AddScoped<DotNetQueryUtilFactory>();
-            services.AddScoped<IGitHubClientFactory>(_ => new GitHubClientFactory(Configuration));
+            services.AddScoped<IGitHubClientFactory>(_ => GitHubClientFactory.Create(Configuration));
 
             services.AddScoped(_ => new FunctionQueueUtil(Configuration[DotNetConstants.ConfigurationAzureBlobConnectionString]));
 
             services.AddDbContext<TriageContext>(options => 
             {
-                var connectionString = Configuration[DotNetConstants.ConfigurationSqlConnectionString];
+                var connectionString = Configuration.GetNonNull(DotNetConstants.ConfigurationSqlConnectionString);
 #if DEBUG
                 options.UseLoggerFactory(LoggerFactory.Create(builder => builder.AddConsole()));
                 options.EnableSensitiveDataLogging();
@@ -82,8 +83,14 @@ public void ConfigureServices(IServiceCollection services)
             })
             .AddGitHub(options =>
             {
-                options.ClientId = Configuration[DotNetConstants.ConfigurationGitHubClientId];
-                options.ClientSecret = Configuration[DotNetConstants.ConfigurationGitHubClientSecret];
+                // If we are going to impersonate the logged in user rather than use the GH app
+                // then we need some additional permissions to manage GH issues
+                if (Configuration[DotNetConstants.ConfigurationGitHubImpersonateUser] == "true")
+                {
+                    options.Scope.Add("public_repo");
+                }
+                options.ClientId = Configuration.GetNonNull(DotNetConstants.ConfigurationGitHubClientId);
+                options.ClientSecret = Configuration.GetNonNull(DotNetConstants.ConfigurationGitHubClientSecret);
                 options.SaveTokens = true;
 
                 options.ClaimActions.MapJsonKey(Constants.GitHubAvatarUrl, Constants.GitHubAvatarUrl);
@@ -132,6 +139,21 @@ public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
             app.UseAuthentication();
             app.UseAuthorization();
 
+            // At the time the ClientFactory was created we hadn't done authentication yet. Now that we have
+            // associate the user's GH access token with the factory in case we need it to impersonate them.
+            // This is only necessary when the app is configured to use the logged in user's identity rather
+            // than the dedicated GH runfo app identity.
+            app.Use(async (context, next) =>
+            {
+                if(context.User.Identity.IsAuthenticated)
+                {
+                    string accessToken = await context.GetTokenAsync("access_token");
+                    IGitHubClientFactory gitHubClientFactory = context.RequestServices.GetService<IGitHubClientFactory>();
+                    gitHubClientFactory.SetUserOAuthToken(accessToken);
+                }
+                await next();
+            });
+
             app.UseEndpoints(endpoints =>
             {
                 endpoints.MapRazorPages();

DevOps.Util.DotNet/DotNetConstants.cs

@@ -16,6 +16,7 @@ public static class DotNetConstants
 
         public const string ConfigurationSqlConnectionString = "RunfoConnectionString";
         public const string ConfigurationAzdoToken = "RunfoAzdoToken";
+        public const string ConfigurationGitHubImpersonateUser = "GitHubImpersonateUser";
         public const string ConfigurationGitHubAppId = "GitHubAppId";
         public const string ConfigurationGitHubAppPrivateKey = "GitHubAppPrivateKey";
         public const string ConfigurationGitHubClientId = "GitHubClientId";

DevOps.Util.DotNet/Extensions.cs

@@ -1,4 +1,5 @@
 using Microsoft.DotNet.Helix.Client;
+using Microsoft.Extensions.Configuration;
 using Octokit;
 using System;
 using System.Collections.Generic;
@@ -233,5 +234,16 @@ public static bool TryGetScalarValue(this YamlMappingNode node, string name, out
         }
 
         #endregion
+
+        #region IConfiguration
+        public static string GetNonNull(this IConfiguration config, string key)
+        {
+            if (config[key]==null)
+            {
+                throw new Exception($"No {key} configuration set");
+            }
+            return config[key];
+        }
+        #endregion
     }
 }

DevOps.Util.DotNet/GitHubClientFactory.cs

@@ -5,41 +5,58 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Net;
+using System.Security.Claims;
 using System.Text;
 using System.Threading.Tasks;
 
 namespace DevOps.Util.DotNet
 {
     public interface IGitHubClientFactory
     {
+        void SetUserOAuthToken(string token);
         Task<IGitHubClient> CreateForAppAsync(string owner, string repository);
     }
 
-    public sealed class GitHubClientFactory : IGitHubClientFactory
+    public sealed class OAuthAppClientFactory : IGitHubClientFactory
     {
-        public const string GitHubProductName = "runfo.azurewebsites.net";
+        string _oauthToken;
 
-        private string AppPrivateKey { get; }
-        public int AppId { get; }
+        #region IGitHubClientFactory
 
-        public GitHubClientFactory(IConfiguration configuration)
+        public void SetUserOAuthToken(string token)
         {
-            AppId = int.Parse(configuration[DotNetConstants.ConfigurationGitHubAppId]);
-            AppPrivateKey = configuration[DotNetConstants.ConfigurationGitHubAppPrivateKey];
+            _oauthToken = token;
         }
 
-        public GitHubClientFactory(int appId, string privateKey)
+        async Task<IGitHubClient> IGitHubClientFactory.CreateForAppAsync(string owner, string repository)
+        {
+            if(_oauthToken == null)
+            {
+                throw new Exception("This action requires the user to be logged in via GitHub first");
+            }
+            return GitHubClientFactory.CreateForToken(_oauthToken, AuthenticationType.Oauth);
+        }
+
+        #endregion
+    }
+
+    public sealed class GitHubAppClientFactory : IGitHubClientFactory
+    {
+        private string AppPrivateKey { get; }
+        public int AppId { get; }
+
+        public GitHubAppClientFactory(int appId, string appPrivateKey)
         {
             AppId = appId;
-            AppPrivateKey = privateKey;
+            AppPrivateKey = appPrivateKey;
         }
 
         public async Task<GitHubClient> CreateForAppAsync(string owner, string repository)
         {
             var gitHubClient = CreateForAppCore();
             var installation = await gitHubClient.GitHubApps.GetRepositoryInstallationForCurrent(owner, repository).ConfigureAwait(false);
             var installationToken = await gitHubClient.GitHubApps.CreateInstallationToken(installation.Id);
-            return CreateForToken(installationToken.Token, AuthenticationType.Oauth);
+            return GitHubClientFactory.CreateForToken(installationToken.Token, AuthenticationType.Oauth);
         }
 
         private GitHubClient CreateForAppCore()
@@ -52,28 +69,18 @@ private GitHubClient CreateForAppCore()
                 new GitHubJwtFactoryOptions
                 {
                     AppIntegrationId = AppId,
-                    ExpirationSeconds = 600                    
+                    ExpirationSeconds = 600
                 });
 
             var token = generator.CreateEncodedJwtToken();
 
-            return CreateForToken(token, AuthenticationType.Bearer);
-        }
-
-        public static GitHubClient CreateAnonymous() => new GitHubClient(new ProductHeaderValue(GitHubProductName));
-
-        public static GitHubClient CreateForToken(string token, AuthenticationType authenticationType)
-        {
-            var productInformation = new ProductHeaderValue(GitHubProductName);
-            var client = new GitHubClient(productInformation)
-            {
-                Credentials = new Credentials(token, authenticationType)
-            };
-            return client;
+            return GitHubClientFactory.CreateForToken(token, AuthenticationType.Bearer);
         }
 
         #region IGitHubClientFactory
 
+        public void SetUserOAuthToken(string token) { }
+
         async Task<IGitHubClient> IGitHubClientFactory.CreateForAppAsync(string owner, string repository) =>
             await CreateForAppAsync(owner, repository).ConfigureAwait(false);
 
@@ -91,4 +98,34 @@ internal PlainStringPrivateKeySource(string key)
             public TextReader GetPrivateKeyReader() => new StringReader(_key);
         }
     }
+
+    public static class GitHubClientFactory
+    {
+        public const string GitHubProductName = "runfo.azurewebsites.net";
+
+        public static GitHubClient CreateAnonymous() => new GitHubClient(new ProductHeaderValue(GitHubProductName));
+
+        public static GitHubClient CreateForToken(string token, AuthenticationType authenticationType)
+        {
+            var productInformation = new ProductHeaderValue(GitHubProductName);
+            var client = new GitHubClient(productInformation)
+            {
+                Credentials = new Credentials(token, authenticationType)
+            };
+            return client;
+        }
+
+        public static IGitHubClientFactory Create(IConfiguration configuration)
+        {
+            if (configuration[DotNetConstants.ConfigurationGitHubImpersonateUser] == "true")
+            {
+                return new OAuthAppClientFactory();
+            }
+            else
+            {
+                return new GitHubAppClientFactory(int.Parse(configuration.GetNonNull(DotNetConstants.ConfigurationGitHubAppId)),
+                    configuration.GetNonNull(DotNetConstants.ConfigurationGitHubAppPrivateKey));
+            }
+        }
+    }
 }

Documentation/DevWithoutKeyVault.md

@@ -0,0 +1,96 @@
+# Development without Azure KeyVault
+
+Several of the runfo apps store secrets in Azure Key Vault and the apps will load configuration from there by default.
+However if your user account doesn't have access to the Azure Key Vault or for whatever other reason you don't want to use
+Key Vault there is an alternative approach. This will require you to provide various tokens and connection strings
+manually. At present only the DevOps.Status and scratch project are coded to support this but it wouldn't be hard to add it
+to others. 
+
+## 1. GitHub tokens
+
+Runfo uses OAuth to authenticate the current user to GitHub which requires creating an app registration.
+Follow the [instructions from GitHub](https://docs.github.com/en/developers/apps/building-oauth-apps/creating-an-oauth-app)
+to create one. The app name, app description and web page don't matter other than maybe needing to be unique. They will just
+be displayed back to you later during login. The callback URL should point to the /signin-github page where you will host
+the runfo web app. For example:
+ - Name: Runfo_Noah_Local_Dev
+ - URL: https://github.com/noahfalk/runfo
+ - Description: blank
+ - Callback URL: https://localhost:5001/signin-github
+
+ After registering you will get a client id and client secret, save these for later.
+
+ By default Runfo also uses a GitHub App (OAuth apps and GH apps are not the same thing) which it uses for managing
+ issues. If you don't have Azure Key Vault access you probably don't have access to the AppId and AppSecret tokens for the
+ GitHub app either. There is a GitHubImpersonateUser configuration option we'll set later which avoids using the GH app and instead
+ will impersonate your logged in GH identity when making changes to issues.
+
+## 2. Runfo Database
+
+Runfo uses a database to store tracking issues and caching data from AzDo and Helix. You will either need to obtain a connection
+string for the live test/prod database or set up your own. To set one up I recommend installing the Developer edition of SQL 
+and include the Full Text Indexing optional feature. You can't use SQL Express or the LocalDB that comes with VS because these
+editions do not support Full Text Search.
+
+## 3. AzDo Personal Access Token
+
+Get a PAT for your user account at https://dev.azure.com/dnceng. See the
+[AzDo docs](https://docs.microsoft.com/en-us/azure/devops/organizations/accounts/use-personal-access-tokens-to-authenticate?view=azure-devops&tabs=Windows)
+for more details. Select Read access for the Work Items, Code, Build, Release, and Test Management scopes. (I guessed at these
+permissions and they appear to work but it might be more than is needed).
+
+Save the PAT for later.
+
+## 4. Create User Secrets
+
+DotNet supports saving per-user secrets in a file called secrets.json that is separate from your project source so that
+you don't accidentally check it in. You can create the file using Visual Studio's 'Manage User Secrets' or via command-line using
+dotnet user-secrets. See [the docs](https://docs.microsoft.com/en-us/aspnet/core/security/app-secrets?view=aspnetcore-6.0&tabs=windows)
+for more details.
+
+Edit the secrets.json file so it looks like the one below. Use the client id and client secret from step 1, whatever connection
+string is appropriate for the database you set up in step 2, and RunfoAzdoToken is the token from step 3.
+
+```
+{
+  "GitHubClientId": "11122233344455566677",
+  "GitHubClientSecret": "11112222333334444555666777888999aaabbbcc",
+  "RunfoAzdoToken": "abcdefghijklmnopqrstuvwxyz1234567890abcdefghijklmnop",
+  "RunfoConnectionString": "Server=localhost;Database=TriageContext;Trusted_Connection=True;",
+  "GitHubImpersonateUser": "true"
+}
+```
+
+GitHubImpersonateUser isn't really a secret, but it was convenient to set it here because these are per-user settings that won't
+get checked in.
+
+## 5. Set the environment variable USE_KEYVAULT=0
+
+Program.cs for the DevOps.Status and scratch projects will check for this env var and avoid using Azure Key Vault in the configuration
+setup. The config values that normally would have come from key vault will come from the secrets.json instead.
+
+## 6. Create the Runfo TriageContext database if needed
+
+If you installed a new SQL server instance in step 2 then we need to initialize Runfo's database. Navigate to the scratch
+source folder.
+
+If needed install the ef command line tools:
+
+```
+dotnet tool install -g dotnet-ef
+```
+
+Once the tools are installed run this command to initialize the database schema:
+
+```
+dotnet ef database update
+```
+
+Then run the scratch project with argument "populateDb". This will take a long time as scratch queries AzDo and Helix
+caching lots of data into the new runfo database. You can quit part way through if you don't care about getting
+complete data.
+
+## 7. Start developing
+
+You should now be able to run the runfo apps and have things mostly function. Use dotnet run or launch from VS with F5.
+If you use the debugger make sure USE_KEYVAULT=0 is included in your launch profile.