New Initiative: Bar21

.github/ISSUE_TEMPLATE/new_initiative.yml

@@ -48,28 +48,33 @@ body:
     attributes:
       label: Initiative Category
       options:
-        - Agriculture
-        - Children
+        - Agriculture/Main
+        - Children/Main
         - Defense/Evacuation
         - Defense/Shelters
         - Defense/Weapons
-        - Donations
-        - FamilyHosting
+        - Donations/Agriculture
+        - Donations/Main
+        - FamilyHosting/Main
         - GeneralAssistance
-        - Initiatives
-        - Medical
-        - MemorialRemembrance
-        - MentalSupport
+        - Initiatives/Main
+        - Medical/Main
+        - MemorialRemembrance/Main
+        - MentalSupport/Main
         - MissingPersons/Main
-        - Reserves
-        - SocialNetworks
-        - StrategicCommunication
-        - Supplies
+        - Reserves/Main
+        - StrategicCommunication/Funding
+        - StrategicCommunication/Info
+        - StrategicCommunication/Main
+        - StrategicCommunication/SocialNetworks
+        - StrategicCommunication/Tools
+        - Supplies/Donation
+        - Supplies/Main
         - TechInitiatives/Main
         - TechInitiatives/Support
         - TechInitiatives/Volunteers
-        - Transportation
-        - Volunteering
+        - Transportation/Main
+        - Volunteering/Main
       default: 0
     validations:
       required: true
@@ -151,7 +156,7 @@ body:
       placeholder: https://twitter.com/iron-widgets
 
   - type: input
-    id: logo
+    id: initiativeImage
     attributes:
       label: Initiative logo
       description: Link to an image of the initiative's logo (suitable for inclusion in an HTML `IMG` tag)

.github/workflows/initiative-auto-comment.yml → .github/workflows/initiative-auto-pr.yml

@@ -1,16 +1,17 @@
-name: Initiative Request - Auto Comment
+name: Initiative Request - Automatic PR
 
 on:
   issues:
     types: [opened, edited]
 
 jobs:
-  add-comment:
+  create-pr:
     if: contains(github.event.issue.labels.*.name, 'new-initiative-request')
     runs-on: ubuntu-latest
     permissions:
-      contents: read
+      contents: write
       issues: write
+      pull-requests: write
 
     steps:
       - name: Checkout code
@@ -27,14 +28,18 @@ jobs:
           pip install -r "$GITHUB_WORKSPACE/scripts/requirements.txt"
 
       - name: Run GPT script
+        uses: nick-fields/retry@v2
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           ISSUE_BODY: ${{ github.event.issue.body }}
-        run: python $GITHUB_WORKSPACE/scripts/gptGenerateJsonFromIssue.py
-
-      - name: Post comment
+        with:
+          timeout_minutes: 5
+          max_attempts: 3
+          command: python $GITHUB_WORKSPACE/scripts/gptGenerateJsonFromIssue.py
+
+      - name: Generate PR
         uses: actions/github-script@v6
         with:
           script: |
-            const script = require(process.env.GITHUB_WORKSPACE + '/scripts/commentUsingGptResponse.js')
+            const script = require(process.env.GITHUB_WORKSPACE + '/scripts/generatePullRequestForNewInitiative.js')
             script({github, context})

_data/links/Children/Main/links.json

@@ -55,6 +55,31 @@
       "description": "קוד: בריינפופ23",
       "url": "https://il.brainpop.com/",
       "website": "https://il.brainpop.com/"
+    },
+    {
+      "displayName": "Bar21",
+      "name": "Bar21",
+      "shortDescription": "FooBaz",
+      "description": "hello world",
+      "initiativeValidationDetails": "Source: Are you the developer? If not, how did you hear about it?\nLink to reputable source(s) promoting the initiative: [e.g. https://www.israelhayom.co.il/tech/tech-news/iron-swords]\nContact details for members of the initiative's team (so we can contact them directly for validation): [e.g. info@iron-widgets.com]",
+      "url": "N/A",
+      "website": "N/A",
+      "whatsapp": "N/A",
+      "telegram": "N/A",
+      "drive": "N/A",
+      "form": "N/A",
+      "docs": "N/A",
+      "discord": "N/A",
+      "facebook": "N/A",
+      "instagram": "N/A",
+      "tiktok": "N/A",
+      "twitter": "N/A",
+      "initiativeImage": "N/A",
+      "tags": [
+        "Bar21",
+        "FooBaz",
+        "hello world"
+      ]
     }
   ]
 }

scripts/generatePullRequestForNewInitiative.js

@@ -0,0 +1,152 @@
+module.exports = ({github, context}) => {
+
+  const fs = require('fs');
+  const cp = require('child_process');
+
+  function tryExtractJson(text, jsonStartMarker, jsonEndMarker) {
+    console.log(`Attempting to extract JSON with start marker "${jsonStartMarker}" and end marker "${jsonEndMarker}"` );
+
+    const indexOfJsonStart = text.indexOf(jsonStartMarker);
+    if (indexOfJsonStart === -1) {
+      console.log("Could not find JSON start marker");
+      return null;
+    }
+
+    const indexOfJsonEnd = text.indexOf(jsonEndMarker, indexOfJsonStart + 1);
+    if (indexOfJsonEnd === -1) {
+      console.log("Could not find JSON end marker");
+      return null;
+    }
+
+    return text.substring(indexOfJsonStart + jsonStartMarker.length, indexOfJsonEnd);
+  }
+
+  function createOrUpdatePullRequest(branch, name) {
+
+    const existingResponse = github.rest.pulls.list({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      head: branch,
+      base: 'main',
+    })
+    console.log("existing PRs: " + JSON.stringify(existingResponse));
+
+    existingPrs = existingResponse.data
+    if (existingPrs && existingPrs.length > 0) {
+      console.log("At least one PR exists for this branch");
+      existingPrs[0].existing = true;
+      return existingPrs[0];
+    }
+
+    const newResponse  = github.rest.pulls.create({
+      title: 'New Initiative: ' + name,
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      head: branch,
+      base: 'main',
+      body: "*IMPORTANT: Only merge after validating the initiative and double checking the generated JSON*"
+    });
+    console.log("Created new PR: " + JSON.stringify(newResponse));
+
+    return newResponse.data
+  }
+
+  function createComment(body) {
+    github.rest.issues.createComment({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      issue_number: context.issue.number,
+      body: body
+    });
+  }
+
+  function warnAndComment(warning, exception, json) {
+    console.warn(`${warning}: ${exception}`);
+    createComment(`WARNING: ${warning} (see GitHub Action logs for more details)\n` + "Automatic PR will NOT be generated:\n" + json);
+  }
+
+  function executeGitCommand(args) {
+    console.log(`Executing: git ${args}`);
+    cp.execFileSync("git", args);
+  }
+
+  const tempFolder = process.env.TEMP || "/tmp";
+  const gptResponse = fs.readFileSync(tempFolder + "/gpt-auto-comment.output", "utf8");
+
+  // https://stackoverflow.com/a/51602415/67824
+  const sanitizedGptResponse = gptResponse.replace(/[\u0000-\u001F\u007F-\u009F]/g, "");
+  console.log("Sanitized GPT response: " + sanitizedGptResponse);
+
+  const jsonStartMarker = "```json";
+  const jsonEndMarker = "```";
+  jsonString = tryExtractJson(sanitizedGptResponse, jsonStartMarker, jsonEndMarker) || tryExtractJson(sanitizedGptResponse, jsonEndMarker, jsonEndMarker);
+  if (jsonString == null) {
+    console.log("Could not find JSON markers in GPT output, assuming raw JSON");
+    jsonString = sanitizedGptResponse;
+  }
+  console.log("Extracted JSON: " + jsonString);
+
+  try {
+    json = JSON.parse(jsonString);
+  } 
+  catch (e) {
+    return warnAndComment("Could not process GPT response as JSON", e, jsonString);
+  }
+
+  humanReadableJson = "```json\n" + JSON.stringify(json, null, 2) + "\n```";
+
+  let categoryLinksJsonFile, categoryJsonString;
+  try {
+    categoryLinksJsonFile = `${process.env.GITHUB_WORKSPACE}/_data/links/${json.category}/links.json`;
+    console.log("resolved category links file: " + categoryLinksJsonFile);
+
+    categoryJsonString = fs.readFileSync(categoryLinksJsonFile, "utf8");
+    categoryJson = JSON.parse(categoryJsonString);
+  }
+  catch (e) {
+    return warnAndComment("Could not process category links JSON", e, humanReadableJson);
+  }
+
+  delete json.category //not in our schema, and worse - will interfere with the existing initiative detection below
+  console.log("Attempting to detect already existing initiative under this category");
+
+  const upperCategoryJsonString = categoryJsonString.toLocaleUpperCase("en-us");
+  for (const prop in json) {
+
+    const value = json[prop];
+    if (typeof value !== "string") {
+      continue;
+    }
+
+    const PropValueUpper = value.toLocaleUpperCase("en-us");
+    if (upperCategoryJsonString.indexOf(PropValueUpper) !== -1) {
+      return warnAndComment(`Initiative might already exist under this category, the value of property ${prop} is already present in the JSON: ${value}`, "suspected existing initiative", humanReadableJson);
+    }
+  }
+
+  categoryJson.links.push(json);
+  fs.writeFileSync(categoryLinksJsonFile, JSON.stringify(categoryJson, null, 2), "utf8");
+
+  const branch = `auto-pr-${context.issue.number}`;
+  try {
+    executeGitCommand(["config", "user.name", "github-actions"])
+    executeGitCommand(["config", "user.email", "41898282+github-actions[bot]@users.noreply.github.com"]) //https://github.com/orgs/community/discussions/26560
+    executeGitCommand(["checkout", "-b", branch])
+    executeGitCommand(["add", categoryLinksJsonFile])
+    executeGitCommand(["commit", "-m", json.name || "new initiative"])
+    executeGitCommand(["push", "origin", branch, "--force"])
+  }
+  catch (e) {
+    return warnAndComment("encountered error during git execution", e, humanReadableJson);
+  }
+
+  let pr;
+  try {
+    pr = createOrUpdatePullRequest(branch, json.name || "???");
+  }
+  catch (e) {
+    return warnAndComment("Could not create pull request", e, humanReadableJson);
+  }
+
+  createComment((pr.existing ? "Updated" : "Created") + " PR: " + pr.url);
+}

scripts/gptGenerateJsonFromIssue.py

@@ -110,7 +110,7 @@ def ask_gpt_for_initiative_json(self, issue_body: str) -> str:
 if __name__ == "__main__":
     gpt_response = GPTAssistant().ask_gpt_for_initiative_json(os.environ["ISSUE_BODY"])
 
-    respose_output_file = os.environ["GITHUB_WORKSPACE"] + "/gpt-auto-comment.output"
+    respose_output_file = os.getenv("TEMP", "/tmp") + "/gpt-auto-comment.output"
     logging.info(f"Writing GPT Response to {respose_output_file}:\n{gpt_response}")
 
     with open(respose_output_file, "w", encoding="utf-8") as output:

scripts/test_generatePullRequestForNewInitiative.js

@@ -0,0 +1,33 @@
+const github = {
+    rest: {
+        issues: {
+            createComment: (cmt) => { 
+                console.log("posting comment:")
+                console.log(cmt)
+            }
+        },
+        pulls: {
+            create: (pr) => {
+                console.log("creating PR:")
+                console.log(pr)
+                return {
+                    url: "https://example.com"
+                }
+            }
+        }
+    }
+}    
+
+const context = {
+    repo: {
+        owner: "ohadschn",
+        repo: "ohadschn/ConnectPortal"
+    },
+    issue: {
+        number: 42
+    }
+}
+
+const script = require('./generatePullRequestForNewInitiative.js')
+
+script({github, context})