diff --git a/src/workflows/moderation.ts b/src/workflows/moderation.ts index 9dbdb38..8266b80 100644 --- a/src/workflows/moderation.ts +++ b/src/workflows/moderation.ts @@ -110,16 +110,15 @@ const DEFAULT_THRESHOLDS = { const DEFAULT_PROVIDER = "openai"; const HIVE_ENDPOINT = "https://api.thehive.ai/api/v2/task/sync"; -const HIVE_SEXUAL_CATEGORIES = [ +export const HIVE_SEXUAL_CATEGORIES = [ "general_nsfw", - "general_suggestive", "yes_sexual_activity", - "sex_toys", - "nudity_female", - "nudity_male", + "yes_sex_toy", + "yes_female_nudity", + "yes_male_nudity", ]; -const HIVE_VIOLENCE_CATEGORIES = [ +export const HIVE_VIOLENCE_CATEGORIES = [ "gun_in_hand", "gun_not_in_hand", "knife_in_hand", @@ -128,10 +127,8 @@ const HIVE_VIOLENCE_CATEGORIES = [ "hanging", "noose", "human_corpse", - "emaciated_body", - "self_harm", - "animal_abuse", - "fights", + "yes_emaciated_body", + "yes_self_harm", "garm_death_injury_or_military_conflict", ]; @@ -325,6 +322,12 @@ function getHiveCategoryScores( const scoreMap = Object.fromEntries( classes.map(c => [c.class, c.score]), ); + const missingCategories = categoryNames.filter(category => !(category in scoreMap)); + if (missingCategories.length > 0) { + console.warn( + `Hive response missing expected categories: ${missingCategories.join(", ")}`, + ); + } const scores = categoryNames.map(category => scoreMap[category] || 0); return Math.max(...scores, 0); } diff --git a/tests/unit/moderation.test.ts b/tests/unit/moderation.test.ts new file mode 100644 index 0000000..a576c73 --- /dev/null +++ b/tests/unit/moderation.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; + +import { HIVE_SEXUAL_CATEGORIES, HIVE_VIOLENCE_CATEGORIES } from "../../src/workflows/moderation"; + +describe("hive moderation categories", () => { + it("the HIVE_SEXUAL_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => { + expect(HIVE_SEXUAL_CATEGORIES).toEqual([ + "general_nsfw", + "yes_sexual_activity", + "yes_sex_toy", + "yes_female_nudity", + "yes_male_nudity", + ]); + }); + + it("the HIVE_VIOLENCE_CATEGORIES has not changed — If you change these, remember to check that these are accurate categories in Hive!", () => { + expect(HIVE_VIOLENCE_CATEGORIES).toEqual([ + "gun_in_hand", + "gun_not_in_hand", + "knife_in_hand", + "very_bloody", + "other_blood", + "hanging", + "noose", + "human_corpse", + "yes_emaciated_body", + "yes_self_harm", + "garm_death_injury_or_military_conflict", + ]); + }); +});