Merge pull request #80 from Jalen-Stephens/54-feature-expand-analyze-to-correctly-identify-screen-shots

feat(c2pa): flag screenshots and surface in analyze responses

Author: ikeschmack

citations.md

@@ -1,3 +1,53 @@
+### **Commit / Ticket Reference**
+- **Commit:** feat(c2pa): flag screenshots and surface in analyze responses
+- **Ticket:** none (Feature #54)
+- **Date:** 2025-12-02
+- **Team Member:** Jalen Stephens
+
+---
+
+### **AI Tool Information**
+- **Tool Used:** OpenAI ChatGPT (GPT-5) via Codex CLI
+- **Access Method:** Local Codex CLI session (sandboxed; no paid API calls)
+- **Configuration:** Default model settings
+- **Cost:** $0 (course-provided access)
+
+---
+
+### **Purpose of AI Assistance**
+Added C2PA-driven screenshot detection and plumbing so screenshots are flagged as suspicious and exposed to clients. Work included heuristics based on claim generator and capture_type, new metadata fields, feature-vector update, score bump for screenshots, response DTO extensions, and regression tests.
+
+---
+
+### **Prompts / Interaction Summary**
+- “Implement Feature #54: Expand Analyze to correctly identify screen shots”
+- “Plumb screenshot flag into analyze pipeline and response DTO”
+- “Add JSON parsing and service tests for screenshot detection”
+- “Fix Checkstyle line length in AnalyzeServiceTest”
+
+---
+
+### **Resulting Artifacts**
+- `src/main/java/dev/coms4156/project/metadetect/c2pa/C2paToolInvoker.java`
+- `src/main/java/dev/coms4156/project/metadetect/service/FeatureExtractor.java`
+- `src/main/java/dev/coms4156/project/metadetect/service/AnalyzeService.java`
+- `src/main/java/dev/coms4156/project/metadetect/dto/Dtos.java`
+- `src/main/resources/model.json`
+- Tests: `src/test/java/dev/coms4156/project/metadetect/c2pa/C2paToolInvokerJsonParsingTest.java`, `src/test/java/dev/coms4156/project/metadetect/service/AnalyzeServiceTest.java`, `src/test/java/dev/coms4156/project/metadetect/service/FeatureExtractorTest.java`, `src/test/java/dev/coms4156/project/metadetect/controller/AnalyzeControllerTest.java`, `src/test/java/dev/coms4156/project/metadetect/dto/DtosTest.java`
+
+---
+
+### **Verification**
+- `./mvnw -q test` (passes; expected warnings from mocked c2patool branches)
+- `./mvnw -q -DskipTests compile` to regenerate main classes after changes
+
+---
+
+### **Attribution Statement**
+> Portions of this work were generated with assistance from OpenAI ChatGPT (GPT-5) on 2025-12-02. All AI-generated content was reviewed and finalized by the development team.
+
+---
+
 ### **Commit / Ticket Reference**
 - **Commit:** feat(logging): add request logging filter and console logback config
 - **Ticket:** none

src/main/java/dev/coms4156/project/metadetect/c2pa/C2paToolInvoker.java

@@ -38,6 +38,25 @@ public class C2paToolInvoker {
       "runway"
   );
 
+  /**
+   * Keywords used to conservatively infer that an asset is a screenshot.
+   * This list intentionally focuses on obvious OS/browser capture tools.
+   */
+  private static final List<String> SCREENSHOT_KEYWORDS = Arrays.asList(
+      "screenshot",
+      "screen shot",
+      "screen capture",
+      "screen-capture",
+      "screen grab",
+      "screen-grab",
+      "snipping tool",
+      "snip & sketch",
+      "snip and sketch",
+      "windows snip",
+      "macos screenshot",
+      "screencapture"
+  );
+
   private final String c2paToolPath;
 
   /**
@@ -191,6 +210,9 @@ private C2paMetadata parseMetadataFromJson(String json) {
       int hasManifest = 0;
       String claimGenerator = null;
       int claimGeneratorIsAi = 0;
+      int c2paIsScreenshot = 0;
+      String c2paScreenshotReason = null;
+      JsonNode claimNode = null;
 
       // Count manifests
       JsonNode manifestsNode = root.get("manifests");
@@ -220,7 +242,7 @@ private C2paMetadata parseMetadataFromJson(String json) {
         }
 
         if (activeManifestNode != null && !activeManifestNode.isMissingNode()) {
-          JsonNode claimNode = activeManifestNode.get("claim");
+          claimNode = activeManifestNode.get("claim");
           if (claimNode == null || claimNode.isMissingNode()) {
             claimNode = activeManifestNode;
           }
@@ -248,11 +270,38 @@ && isAiClaimGenerator(claimGenerator)) {
         claimGeneratorIsAi = 0;
       }
 
+      // Heuristic: detect screenshot tools via claim generator or capture_type.
+      if (claimGenerator != null && !claimGenerator.isBlank()) {
+        String gen = claimGenerator.toLowerCase();
+        for (String kw : SCREENSHOT_KEYWORDS) {
+          if (gen.contains(kw)) {
+            c2paIsScreenshot = 1;
+            c2paScreenshotReason = "claim_generator contains \"" + kw + "\"";
+            break;
+          }
+        }
+      }
+      if (c2paIsScreenshot == 0 && claimNode != null) {
+        JsonNode captureType = claimNode.get("capture_type");
+        if (captureType == null) {
+          captureType = claimNode.get("captureType");
+        }
+        String captureVal = captureType != null && !captureType.isNull()
+            ? captureType.asText(null)
+            : null;
+        if (captureVal != null && captureVal.equalsIgnoreCase("screenshot")) {
+          c2paIsScreenshot = 1;
+          c2paScreenshotReason = "capture_type indicates screenshot";
+        }
+      }
+
       return new C2paMetadata(
           hasManifest,
           manifestCount,
           claimGenerator,
           claimGeneratorIsAi,
+          c2paIsScreenshot,
+          c2paScreenshotReason,
           /*c2paErrorFlag*/ 0,
           /*c2paErrorMessage*/ null
       );
@@ -298,6 +347,8 @@ private static boolean isAiClaimGenerator(String generator) {
       "c2paManifestCount",
       "c2paClaimGenerator",
       "c2paClaimGeneratorIsAi",
+      "c2paIsScreenshot",
+      "c2paScreenshotReason",
       "c2paErrorFlag",
       "c2paErrorMessage"
   })
@@ -307,6 +358,8 @@ public static final class C2paMetadata {
     private final int c2paManifestCount;
     private final String c2paClaimGenerator;
     private final int c2paClaimGeneratorIsAi;
+    private final int c2paIsScreenshot;
+    private final String c2paScreenshotReason;
     private final int c2paErrorFlag;
     private final String c2paErrorMessage;
 
@@ -317,6 +370,8 @@ public static final class C2paMetadata {
      * @param c2paManifestCount number of manifests detected in the file
      * @param c2paClaimGenerator claim generator string from the active manifest
      * @param c2paClaimGeneratorIsAi integer flag: 1 if AI generator detected, 0 if not
+     * @param c2paIsScreenshot integer flag: 1 if screenshot heuristics matched, 0 otherwise
+     * @param c2paScreenshotReason optional note describing which heuristic fired
      * @param c2paErrorFlag integer flag: 1 if error occurred, 0 if no error
      * @param c2paErrorMessage error message if errorFlag=1, null otherwise
      */
@@ -325,13 +380,17 @@ public C2paMetadata(
         int c2paManifestCount,
         String c2paClaimGenerator,
         int c2paClaimGeneratorIsAi,
+        int c2paIsScreenshot,
+        String c2paScreenshotReason,
         int c2paErrorFlag,
         String c2paErrorMessage) {
 
       this.c2paHasManifest = c2paHasManifest;
       this.c2paManifestCount = c2paManifestCount;
       this.c2paClaimGenerator = c2paClaimGenerator;
       this.c2paClaimGeneratorIsAi = c2paClaimGeneratorIsAi;
+      this.c2paIsScreenshot = c2paIsScreenshot;
+      this.c2paScreenshotReason = c2paScreenshotReason;
       this.c2paErrorFlag = c2paErrorFlag;
       this.c2paErrorMessage = c2paErrorMessage;
     }
@@ -343,6 +402,8 @@ public static C2paMetadata noManifest() {
           /*c2paManifestCount*/ 0,
           /*c2paClaimGenerator*/ null,
           /*c2paClaimGeneratorIsAi*/ 0,
+          /*c2paIsScreenshot*/ 0,
+          /*c2paScreenshotReason*/ null,
           /*c2paErrorFlag*/ 0,
           /*c2paErrorMessage*/ null
       );
@@ -355,6 +416,8 @@ public static C2paMetadata error(String message) {
           /*c2paManifestCount*/ 0,
           /*c2paClaimGenerator*/ null,
           /*c2paClaimGeneratorIsAi*/ 0,
+          /*c2paIsScreenshot*/ 0,
+          /*c2paScreenshotReason*/ null,
           /*c2paErrorFlag*/ 1,
           /*c2paErrorMessage*/ message
       );
@@ -376,6 +439,14 @@ public int getc2paClaimGeneratorIsAi() {
       return c2paClaimGeneratorIsAi;
     }
 
+    public int getc2paIsScreenshot() {
+      return c2paIsScreenshot;
+    }
+
+    public String getc2paScreenshotReason() {
+      return c2paScreenshotReason;
+    }
+
     public int getc2paErrorFlag() {
       return c2paErrorFlag;
     }

src/main/java/dev/coms4156/project/metadetect/dto/Dtos.java

@@ -69,7 +69,9 @@ public record AnalyzeConfidenceResponse(
       String status,
       Double confidenceScore,
       boolean c2paUsed,
-      String modelVersion
+      String modelVersion,
+      boolean isScreenshot,
+      String screenshotReason
   ) { }
 
   /**

src/main/java/dev/coms4156/project/metadetect/service/AnalyzeService.java

@@ -48,6 +48,7 @@ public class AnalyzeService {
   private final UserService userService;
   private final LogisticRegressionService logisticRegressionService;
   private final Clock clock;
+  private static final double SCREENSHOT_SUSPICION_BONUS = 0.05;
 
   // Lightweight mapper for error JSON assembly.
   private final ObjectMapper objectMapper = new ObjectMapper();
@@ -174,12 +175,16 @@ public Dtos.AnalyzeConfidenceResponse getConfidence(UUID analysisId) {
     var currentUser = userService.getCurrentUserIdOrThrow();
     imageService.getById(currentUser, report.getImageId());
 
+    ScreenshotInfo screenshotInfo = deriveScreenshotInfo(report.getDetails());
+
     return new Dtos.AnalyzeConfidenceResponse(
       report.getId().toString(),
       report.getStatus().name(),
       report.getConfidence(),
       deriveC2paUsed(report.getDetails()),
-      logisticRegressionService.getModelVersion()
+      logisticRegressionService.getModelVersion(),
+      screenshotInfo.isScreenshot(),
+      screenshotInfo.reason()
     );
   }
 
@@ -228,12 +233,16 @@ private void runExtractionAndFinalize(UUID analysisId, String storagePath) {
           tempFile.getAbsolutePath(),
           meta
       );
+      double adjustedConfidence = applyScreenshotAdjustment(
+          inference.confidenceScore(),
+          meta
+      );
 
       // 4) Serialize metadata and mark COMPLETED with a confidence score
       String json = objectMapper.writeValueAsString(meta);
 
       // The details field now stores the C2PA metadata schema, not raw manifest JSON.
-      markCompleted(analysisId, json, inference.confidenceScore());
+      markCompleted(analysisId, json, adjustedConfidence);
 
     } catch (IOException ioe) {
       // IO-level failures (download, JSON serialization) are genuine failures.
@@ -335,6 +344,34 @@ private boolean deriveC2paUsed(String detailsJson) {
     }
   }
 
+  private ScreenshotInfo deriveScreenshotInfo(String detailsJson) {
+    if (!StringUtils.hasText(detailsJson)) {
+      return new ScreenshotInfo(false, null);
+    }
+    try {
+      JsonNode node = objectMapper.readTree(detailsJson);
+      boolean isScreenshot = node.path("c2paIsScreenshot").asInt(0) == 1;
+      String reason = node.path("c2paScreenshotReason").asText(null);
+      if (isScreenshot && (reason == null || reason.isBlank())) {
+        String generator = node.path("c2paClaimGenerator").asText(null);
+        if (StringUtils.hasText(generator)) {
+          reason = "Inferred from claim generator: " + generator;
+        }
+      }
+      return new ScreenshotInfo(isScreenshot, reason);
+    } catch (Exception e) {
+      return new ScreenshotInfo(false, null);
+    }
+  }
+
+  private double applyScreenshotAdjustment(double baseScore, C2paToolInvoker.C2paMetadata meta) {
+    if (meta != null && meta.getc2paIsScreenshot() == 1) {
+      double bumped = baseScore + SCREENSHOT_SUSPICION_BONUS;
+      return bumped > 1.0 ? 1.0 : bumped;
+    }
+    return baseScore;
+  }
+
   /** Truncates a string to a maximum length, null-safe. */
   private static String truncate(String s, int max) {
     if (s == null) {
@@ -370,6 +407,8 @@ static AnalysisReport pending(UUID imageId, Instant createdAt) {
     }
   }
 
+  private record ScreenshotInfo(boolean isScreenshot, String reason) { }
+
   /**
    * Handles a generic failure during analysis by marking the analysis as FAILED
    * and storing the error message in the details field.

src/main/java/dev/coms4156/project/metadetect/service/FeatureExtractor.java

@@ -84,6 +84,7 @@ public double[] extractAllFeatures(String imagePath, C2paMetadata c2pa) {
         c2pa.getc2paHasManifest(),
         c2pa.getc2paManifestCount(),
         c2pa.getc2paClaimGeneratorIsAi(),
+        c2pa.getc2paIsScreenshot(),
         c2pa.getc2paErrorFlag()
     };
   }
@@ -100,6 +101,7 @@ private double[] generateEmpty(C2paMetadata c2pa) {
         c2pa.getc2paHasManifest(),
         c2pa.getc2paManifestCount(),
         c2pa.getc2paClaimGeneratorIsAi(),
+        c2pa.getc2paIsScreenshot(),
         c2pa.getc2paErrorFlag()
     };
   }

src/main/resources/model.json

@@ -12,7 +12,8 @@
         0.0,
         0.0,
         0.0,
+        0.35,
         0.0
     ],
     "bias": -13.851754449616177
-}
+}