diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..82c7fd1a --- /dev/null +++ b/.env.example @@ -0,0 +1,20 @@ +# Environment variables for Awesomebits +# Copy to .env for local development (gitignored) +# +# The README documents .env as the standard location. +# For extra safety, dotenv-rails also loads .env.local +# (which takes precedence) — use that for personal API keys. + +# AWS (S3 uploads) +# AWS_ACCESS_KEY_ID=XXX +# AWS_SECRET_ACCESS_KEY=YYY +# AWS_BUCKET=your-bucket-name + +# Database (defaults work for local dev) +# DB_NAME=awesomefoundation_development +# POSTGRES_HOST=localhost +# POSTGRES_USER= +# POSTGRES_PASSWORD= + +# Spam detection +# SPAM_REGEXP= diff --git a/.env.local.example b/.env.local.example new file mode 100644 index 00000000..1a368f89 --- /dev/null +++ b/.env.local.example @@ -0,0 +1,9 @@ +# Personal API keys — takes precedence over .env +# Copy to .env.local and fill in your keys (gitignored) +# +# dotenv-rails loads .env.local after .env, so values here +# override anything in .env. Use this for secrets you don't +# want to accidentally commit. + +# Signal Score (LLM scoring via Anthropic Batch API) +# ANTHROPIC_API_KEY=sk-ant-... diff --git a/.gitignore b/.gitignore index f60b08fa..39349260 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ .ruby-gemset .bundle .env +.env.local .envrc +.scratch/ +context/memory/.basic-memory/ db/*.sqlite3 log/*.log tmp/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..bcefbc0d --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,113 @@ +# CLAUDE.md — Instructions for Claude Code Sessions + +This file is read automatically at the start of every Claude Code session in this workspace. + +## Project + +**Awesomebits** — the web application for the Awesome Foundation, a global micro-granting community that gives $1,000 grants to "awesome" projects. This is a fork (`divideby0/awesomebits`) of the upstream repo (`awesomefoundation/awesomebits`). + +We're building **Signal Score** — an AI-assisted application pre-screening system that helps trustees prioritize grant applications by surfacing quality signals. See GitHub issue #590. + +## Project Structure + +``` +. +├── app/ # Rails application +│ ├── extras/ # SpamChecker, SpamClassifier +│ ├── models/ # ActiveRecord models (Project, Chapter, etc.) +│ └── ... +├── context/ +│ └── memory/ # basic-memory knowledge base for Signal Score +├── scripts/ +│ └── signal-score/ # Signal Score tooling (Ruby) +├── .scratch/ # Local data, never committed +│ └── data/ # Parquet, CSV, DuckDB files +├── CLAUDE.md # This file +└── ... +``` + +## Tech Stack + +- **Ruby on Rails** — existing app (Ruby 3.3.6, Rails 7.2.3) +- **PostgreSQL** — production database +- **Ruby** — Signal Score scripts (same toolchain, future Rails integration) +- **DuckDB** — local analytical queries on historical grant data + +## Secrets & Configuration + +- **`.env.local`** — personal API keys and secrets (gitignored). Never use `.env` for secrets. +- `dotenv-rails` is in the Gemfile and loads `.env.local` automatically. +- Required keys: + ``` + ANTHROPIC_API_KEY=sk-ant-... + ``` + +## Git & Commits + +### Commit Lint +- **Header max 50 characters** — this is strict, plan for it +- **Body lines max 72 characters** +- **Format:** `type(scope): description` +- **Types:** feat, fix, refactor, chore, docs, test, perf, ci +- **Valid scopes:** signal-score, spam, data, scripts, context, infra, docs, deps, repo +- **References required:** Include `Refs: #591` or `Closes: #N` in the body +- **Subject must be lowercase** — no sentence-case, start-case, pascal-case, or upper-case + +### Branch Convention +- Feature branches: `feat/-` (e.g. `feat/0591-signal-score-scripts`) +- Always branch from `master` (upstream default) + +### Git Remotes +- `origin` → `awesomefoundation/awesomebits` (upstream, read-only for us) +- `fork` → `divideby0/awesomebits` (our fork, push here) +- Push to `fork`, PR against `origin` + +## Data + +### Historical Grant Data +Historical application data is exported from the Awesome Foundation production database and stored in a privileged Google Drive folder. Contact @divideby0 for access. + +### Local Data Setup +Data files live in `.scratch/data/` (gitignored). Expected files: +- `projects.csv` → `projects.parquet` → loaded into `awesomebits.duckdb` +- `chapters.csv` → `chapters.parquet` → loaded into `awesomebits.duckdb` +- Additional: `comments.csv`, sample sets, validation scores + +### DuckDB +```bash +duckdb .scratch/data/awesomebits.duckdb +``` + +Use for analytical queries on historical data. Never connect to production. + +## Signal Score Architecture + +### Scoring Pipeline +1. Application text → LLM batch API (Anthropic) +2. Structured JSON output with Trust Equation dimensions +3. Composite score 0.0–1.0 + feature breakdown + flags + +### Trust Equation +`T = (Credibility + Reliability + Intimacy) / (1 + Self-Interest)` + +### Key Files +- `scripts/signal-score/score_grants.rb` — batch scoring via Anthropic API +- `scripts/signal-score/import_data.rb` — CSV → Parquet → DuckDB +- `context/memory/` — research notes, analysis, pattern discovery + +## Context: Existing Spam Detection + +Two existing systems (bot detection only, not content analysis): +- `app/extras/spam_checker.rb` — blocklist + identical fields +- `app/extras/spam_classifier.rb` — behavioral JS metadata analysis + +Signal Score is complementary — it analyzes content, not form behavior. + +## Notifying Evie + +When you finish a task, get stuck, or need feedback: +```bash +openclaw agent --agent main --message "[CC: ] " --timeout 30 +``` + +Keep messages concise — they'll be read on a phone. diff --git a/Gemfile b/Gemfile index 7f74aacb..457d410c 100644 --- a/Gemfile +++ b/Gemfile @@ -78,6 +78,7 @@ group :test do gem 'shoulda-matchers', '~> 3' gem 'timecop' gem 'turnip' + gem 'webmock' end group :staging, :production do diff --git a/Gemfile.lock b/Gemfile.lock index 3f32a009..4b81c516 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -152,6 +152,9 @@ GEM concurrent-ruby (1.3.4) connection_pool (2.5.0) content_disposition (1.0.0) + crack (1.0.1) + bigdecimal + rexml crass (1.0.6) csv (3.3.3) cucumber-gherkin (28.0.0) @@ -200,6 +203,7 @@ GEM activerecord (>= 4.0.0) globalid (1.2.1) activesupport (>= 6.1) + hashdiff (1.2.1) high_voltage (4.0.0) honeypot-captcha (1.0.1) htmlentities (4.3.4) @@ -475,6 +479,10 @@ GEM activemodel (>= 6.0.0) bindex (>= 0.4.0) railties (>= 6.0.0) + webmock (3.26.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) webrick (1.9.1) websocket (1.2.11) websocket-driver (0.8.0) @@ -560,6 +568,7 @@ DEPENDENCIES tus-server (~> 2.3) uppy-s3_multipart (~> 1.2) web-console + webmock will_paginate (~> 3.3.1) xmlrpc diff --git a/README.md b/README.md index e8609a36..27ba14de 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,44 @@ AWS_SECRET_ACCESS_KEY=YYY AWS_BUCKET=your-bucket-name ``` +### Signal Score (AI-Assisted Pre-Screening) + +Signal Score uses the Anthropic API to pre-screen grant applications, helping trustees +prioritize their review. Scores are advisory only and never auto-hide applications. + +**Setup:** + +1. Get an Anthropic API key from [console.anthropic.com](https://console.anthropic.com) +2. Add it to `.env.local` (create this file if it doesn't exist — it's gitignored): + +```shell +ANTHROPIC_API_KEY=sk-ant-... +``` + +3. Enable scoring for a chapter by inserting a config row: + +```sql +INSERT INTO signal_score_configs (chapter_id, scoring_config, prompt_overrides, category_config, enabled, created_at, updated_at) +VALUES (NULL, '{"model": "claude-haiku-4-5-20251001"}'::jsonb, '{}'::jsonb, '{}'::jsonb, true, NOW(), NOW()); +``` + +Set `chapter_id` to NULL for a global default, or a specific chapter ID for per-chapter config. + +**How it works:** +- New applications are scored asynchronously via `ScoreGrantJob` (SuckerPunch) +- Each score costs ~$0.01 via the Anthropic Haiku model (~2 seconds per app) +- Results are stored in `projects.metadata['signal_score']` as JSONB +- Trustees can filter by score threshold and sort by score on the dashboard +- Score badges are hidden by default — trustees opt in via "Show scores" checkbox +- The scoring rubric uses a Trust Equation framework: `T = (C + R + I) / (1 + S)` + +**Dashboard controls:** +- **Show scores** — toggle score badges on individual applications +- **Sort** — Latest, Earliest, Score highest, Score lowest, Random (stable hash) +- **Filter** — All apps, Hide low signal (default), Borderline+, Solid+, Strong only + +**Cost at scale:** ~$0.01/app. A chapter receiving 50 apps/month = ~$0.50/month. + Subdomains ---------- diff --git a/app/assets/javascripts/application-score-filter.js b/app/assets/javascripts/application-score-filter.js new file mode 100644 index 00000000..69fff156 --- /dev/null +++ b/app/assets/javascripts/application-score-filter.js @@ -0,0 +1,46 @@ +/* Score filter dropdowns — each .score-filter-wrapper is independent */ + +$('a.score-selection').on('click', function(event) { + event.preventDefault(); + const $wrapper = $(this).closest('.score-filter-wrapper'); + const $selector = $wrapper.find('.score-selector'); + const isOpen = $selector.hasClass('expanded'); + + // Close all dropdowns first + $('.score-selector').removeClass('expanded'); + $('a.score-selection').removeClass('expanded'); + + if (!isOpen) { + $selector.addClass('expanded'); + $(this).addClass('expanded'); + } +}); + +$(document).click(function(e) { + if (!$(e.target).closest('.score-filter-wrapper').length) { + $('.score-selector').removeClass('expanded'); + $('a.score-selection').removeClass('expanded'); + } +}); + +/* Show/hide score badges — client-side toggle with localStorage persistence */ +(function() { + var STORAGE_KEY = 'awesomebits-show-scores'; + var $checkbox = $('#show-scores'); + + // Restore saved state on page load + if (localStorage.getItem(STORAGE_KEY) === 'true') { + $checkbox.prop('checked', true); + $('.signal-score').show(); + } + + $checkbox.on('change', function() { + var checked = $(this).is(':checked'); + localStorage.setItem(STORAGE_KEY, checked); + if (checked) { + $('.signal-score').show(); + } else { + $('.signal-score').hide(); + } + }); +})(); diff --git a/app/assets/stylesheets/_signal-score.scss b/app/assets/stylesheets/_signal-score.scss new file mode 100644 index 00000000..7cc1beee --- /dev/null +++ b/app/assets/stylesheets/_signal-score.scss @@ -0,0 +1,195 @@ +// Signal Score — AI-assisted grant pre-screening +// Matches existing awesomebits design language + +// Score filter dropdowns — floated like the other filter controls +.score-filter-wrapper { + float: left; + margin-right: 10px; + margin-top: 20px; + position: relative; + + a.score-selection { + background: rgb(230,230,230); + border: 1px solid rgb(190,190,190); + border-radius: 3px; + display: block; + height: 21px; + padding: 8px 34px 0 10px; + text-decoration: none; + position: relative; + + &:hover { + background: rgb(220,220,220); + } + + span { + color: rgb(55,55,55); + font: normal 12px $sans-serif; + text-shadow: 0 1px rgba(255,255,255, 0.8); + } + + span.arrow { + border-top: 5px solid rgb(150,150,150); + border-right: 5px solid transparent; + border-bottom: 5px solid transparent; + border-left: 5px solid transparent; + position: absolute; + top: 13px; + right: 10px; + } + } + + ol.score-selector { + background: rgb(255,255,255); + border: 1px solid rgb(150,150,150); + border-bottom-left-radius: 4px; + border-bottom-right-radius: 4px; + box-shadow: 0 3px 5px 0 rgba(0,0,0, 0.2); + display: none; + position: absolute; + z-index: 10; + margin-top: -1px; + margin-bottom: 0; + min-width: 100%; + padding: 0; + + li { + border-bottom: 1px solid rgb(200,200,200); + list-style: none; + margin: 0 5px; + padding: 3px 0; + + &:last-child { border: none; } + + a { + color: rgb(55,55,55); + display: block; + padding: 5px 8px; + font: normal 12px/16px $sans-serif; + text-decoration: none; + white-space: nowrap; + + &:hover { + background: rgb(240,240,240); + border-radius: 3px; + } + } + } + } + + ol.score-selector.expanded { + display: block; + } + + a.score-selection.expanded { + border-bottom-left-radius: 0; + border-bottom-right-radius: 0; + } +} + +// Score badge + +.signal-score { + margin: 0; + padding: 8px 10px; + border-left: 4px solid rgb(200,200,200); + font: normal 12px/16px $sans-serif; + + &--strong { border-left-color: #2ecc71; } + &--solid { border-left-color: #f39c12; } + &--borderline { border-left-color: #e67e22; } + &--weak { border-left-color: #e74c3c; } + + &__header { + display: flex; + align-items: baseline; + flex-wrap: wrap; + gap: 6px; + } + + &__value { + font: bold 16px/18px $sans-serif; + + .signal-score--strong & { color: #2ecc71; } + .signal-score--solid & { color: #f39c12; } + .signal-score--borderline & { color: #e67e22; } + .signal-score--weak & { color: #e74c3c; } + } + + &__label { + font: 500 13px/18px $sans-serif; + color: $base-font-color; + } + + &__category { + background: rgb(230,230,230); + border: 1px solid rgb(200,200,200); + border-radius: 3px; + padding: 1px 6px; + font: normal 11px/14px $sans-serif; + color: rgb(100,100,100); + } + + &__tag { + background: #e8f0f8; + border: 1px solid #c8d8e8; + border-radius: 3px; + padding: 1px 5px; + font: normal 10px/13px $sans-serif; + color: #4a7aaa; + } + + &__reason { + margin: 4px 0 0 0; + padding: 0; + font: italic 12px/16px $serif; + color: rgb(120,120,120); + } + + &__details { + margin-top: 4px; + + summary { + cursor: pointer; + font: normal 11px/14px $sans-serif; + color: rgb(160,160,160); + + &:hover { color: $pink; } + } + } + + &__features { + margin-top: 4px; + display: grid; + grid-template-columns: repeat(auto-fill, minmax(140px, 1fr)); + gap: 3px; + } + + &__feat { + display: flex; + justify-content: space-between; + padding: 2px 4px; + background: rgb(248,248,248); + border-radius: 2px; + } + + &__feat-name { + font: normal 11px/14px $sans-serif; + color: rgb(120,120,120); + text-transform: capitalize; + } + + &__feat-val { + font: 500 11px/14px $sans-serif; + } + + &__feat--good { color: #2ecc71; } + &__feat--bad { color: #e67e22; } + + &__ai-warning { + margin: 4px 0 0 0; + padding: 0; + font: normal 11px/14px $sans-serif; + color: #e74c3c; + } +} diff --git a/app/assets/stylesheets/application.css.scss b/app/assets/stylesheets/application.css.scss index 042988d8..3b33980a 100644 --- a/app/assets/stylesheets/application.css.scss +++ b/app/assets/stylesheets/application.css.scss @@ -37,5 +37,6 @@ @import 'pagination'; @import 'magnific-popup'; +@import 'signal-score'; @import 'owl.carousel.min'; @import 'owl.theme.default.min'; diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index b647cf89..ecfc5096 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -27,6 +27,28 @@ def index project_filter.funded end + # Signal Score filtering — only applied when explicitly requested via score_min param + @signal_score_min = params.key?(:score_min) ? params[:score_min].to_f : 0 + @sort_mode = params[:sort] if %w[score_desc score_asc earliest latest random].include?(params[:sort]) + + if @signal_score_min > 0 + project_filter.signal_score_above(@signal_score_min) + end + + case @sort_mode + when "score_desc" + project_filter.sort_by_signal_score(:desc) + when "score_asc" + project_filter.sort_by_signal_score(:asc) + when "earliest" + project_filter.sort_by_date(:asc) + when "latest" + project_filter.sort_by_date(:desc) + when "random" + project_filter.sort_by_random + # else: default sort (created_at DESC) applied by ProjectFilter#initialize + end + @q = params[:q].to_s.strip unless @q.blank? diff --git a/app/extras/project_filter.rb b/app/extras/project_filter.rb index 31aee442..b0f05895 100644 --- a/app/extras/project_filter.rb +++ b/app/extras/project_filter.rb @@ -29,6 +29,33 @@ def funded self end + def signal_score_above(threshold) + @projects = @projects.where("(metadata->'signal_score'->>'composite_score')::float >= ?", threshold) + self + end + + SORT_DIRECTIONS = { asc: "ASC", desc: "DESC" }.freeze + + def sort_by_signal_score(direction = :desc) + sql_dir = SORT_DIRECTIONS.fetch(direction, "DESC") + @projects = @projects.reorder( + Arel.sql("COALESCE((metadata->'signal_score'->>'composite_score')::float, -1) #{sql_dir}") + ) + self + end + + def sort_by_date(direction = :desc) + @projects = @projects.reorder(created_at: direction) + self + end + + def sort_by_random + # Stable random: hash the project ID so the order is consistent per page load + # but looks random to the reviewer (no bias from submission order or score) + @projects = @projects.reorder(Arel.sql("md5(projects.id::text)")) + self + end + def not_pending_moderation @projects = @projects.left_joins(:project_moderation).where(project_moderations: {id: nil}).or(@projects.merge(ProjectModeration.approved)) self diff --git a/app/extras/signal_scorer.rb b/app/extras/signal_scorer.rb new file mode 100644 index 00000000..638778b0 --- /dev/null +++ b/app/extras/signal_scorer.rb @@ -0,0 +1,268 @@ +# frozen_string_literal: true + +require "net/http" +require "json" +require "uri" + +# SignalScorer — AI-assisted grant application pre-screening. +# +# Two-layer pipeline: +# 1. PreScorer: deterministic text features (zero cost, local) +# 2. LLM Scorer: Anthropic API with Trust Equation rubric +# +# Usage: +# scorer = SignalScorer.new(project) +# result = scorer.score! # full pipeline: pre-score + LLM +# scorer.composite_score # 0.0-1.0 +# scorer.reason # one-sentence explanation +# scorer.primary_category # "public-art" +# scorer.tags # ["mentorship", "emerging-artists"] +# +# The result is persisted to project.metadata["signal_score"]. +# +class SignalScorer + ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages" + DEFAULT_MODEL = "claude-haiku-4-5-20251001" + MAX_TOKENS = 1024 + + attr_reader :project, :score_result + + class ScoringError < StandardError; end + + def initialize(project, api_key: nil) + @project = project + @api_key = api_key || ENV["ANTHROPIC_API_KEY"] + @config = SignalScoreConfig.resolve(project.chapter) rescue {} + end + + # Full scoring pipeline: LLM score + persist to metadata. + def score! + raise ScoringError, "No Anthropic API key configured" unless @api_key.present? + + # Build the prompt + application_text = format_application + system_prompt = build_system_prompt + + # Call Anthropic Messages API with tool_use for structured output + response = call_anthropic( + system: system_prompt, + messages: [{ "role" => "user", "content" => "Score this grant application:\n\n#{application_text}" }], + tools: [score_tool], + tool_choice: { "type" => "tool", "name" => "score_application" } + ) + + # Extract tool result + tool_block = response.dig("content")&.find { |c| c["type"] == "tool_use" } + raise ScoringError, "No tool_use block in response" unless tool_block + + @score_result = tool_block["input"] + + # Persist to metadata + persist_score! + + @score_result + end + + def composite_score + score_result&.dig("composite_score") + end + + def reason + score_result&.dig("reason") + end + + def flags + score_result&.dig("flags") || [] + end + + def features + score_result&.dig("features") || {} + end + + def primary_category + (project.metadata.dig("signal_score", "primary_category") rescue nil) || + score_result&.dig("primary_category") + end + + def tags + (project.metadata.dig("signal_score", "tags") rescue nil) || + score_result&.dig("tags") || [] + end + + # Score color for UI badge + def score_color + s = composite_score + return "gray" unless s + case s + when 0.7..1.0 then "green" + when 0.5...0.7 then "yellow" + when 0.3...0.5 then "orange" + else "red" + end + end + + # Human-readable score label + def score_label + s = composite_score + return "Unscored" unless s + case s + when 0.7..1.0 then "Strong" + when 0.5...0.7 then "Solid" + when 0.3...0.5 then "Borderline" + when 0.1...0.3 then "Weak" + else "Low Signal" + end + end + + private + + def format_application + parts = [] + parts << "Title: #{project.title}" if project.title.present? + parts << "About Me: #{project.about_me}" if project.about_me.present? + parts << "About Project: #{project.about_project}" if project.about_project.present? + parts << "Use for Money: #{project.use_for_money}" if project.use_for_money.present? + parts << "URL: #{project.url}" if project.url.present? + + # Include extra answers if present + (1..3).each do |i| + q = project.send("extra_question_#{i}") rescue nil + a = project.send("extra_answer_#{i}") rescue nil + if q.present? && a.present? + parts << "#{q}: #{a}" + end + end + + parts.join("\n\n") + end + + def build_system_prompt + # Use PromptBuilder if available (scripts), otherwise inline + if defined?(PromptBuilder) + builder = PromptBuilder.new(chapter: project.chapter&.name) + builder.system_prompt + else + default_system_prompt + end + end + + def score_tool + if defined?(PromptBuilder) + PromptBuilder::SCORE_TOOL + else + { + "name" => "score_application", + "description" => "Score a grant application and extract structured features", + "input_schema" => { + "type" => "object", + "required" => %w[composite_score reason flags features], + "properties" => { + "composite_score" => { "type" => "number", "description" => "Overall score 0.0-1.0" }, + "reason" => { "type" => "string", "description" => "One-sentence explanation" }, + "flags" => { "type" => "array", "items" => { "type" => "string" } }, + "features" => { + "type" => "object", + "required" => %w[credibility reliability intimacy self_interest specificity creativity budget_alignment catalytic_potential community_benefit personal_voice ai_spam_likelihood ai_writing_likelihood], + "properties" => %w[credibility reliability intimacy self_interest specificity creativity budget_alignment catalytic_potential community_benefit personal_voice ai_spam_likelihood ai_writing_likelihood].each_with_object({}) { |k, h| h[k] = { "type" => "number" } }, + }, + }, + }, + } + end + end + + def call_anthropic(system:, messages:, tools:, tool_choice:) + uri = URI(ANTHROPIC_API_URL) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + http.read_timeout = 30 + + model = @config.dig("scoring_config", "model") || DEFAULT_MODEL + + body = { + "model" => model, + "max_tokens" => MAX_TOKENS, + "system" => system, + "messages" => messages, + "tools" => tools, + "tool_choice" => tool_choice, + } + + request = Net::HTTP::Post.new(uri.path) + request["Content-Type"] = "application/json" + request["x-api-key"] = @api_key + request["anthropic-version"] = "2023-06-01" + request.body = body.to_json + + response = http.request(request) + + unless response.code.to_i == 200 + # Log only status code and error type — don't leak response body which may contain user content + error_type = begin + JSON.parse(response.body).dig("error", "type") + rescue + "unknown" + end + raise ScoringError, "Anthropic API error #{response.code} (#{error_type})" + end + + JSON.parse(response.body) + end + + def persist_score! + return unless @score_result + + score_data = @score_result.merge( + "scored_at" => Time.current.iso8601, + "model" => @config.dig("scoring_config", "model") || DEFAULT_MODEL, + "variant" => "live" + ) + + # Atomic JSONB merge — avoids lost updates from concurrent metadata writes. + # Uses || operator to merge into existing metadata rather than replacing it. + Project.where(id: project.id).update_all( + ["metadata = COALESCE(metadata, '{}'::jsonb) || jsonb_build_object('signal_score', ?::jsonb), updated_at = ?", + score_data.to_json, Time.current] + ) + + project.reload + end + + def default_system_prompt + <<~PROMPT + You are an expert grant application screener for the Awesome Foundation. + + The Awesome Foundation is a global network of volunteer "micro-trustees" who each chip in to award $1,000 grants for awesome projects. No strings attached — the money goes to creative, community-benefiting, unique ideas. + + Score each application using the score_application tool. Extract structured features to help trustees prioritize their review. + + ## Scoring Rubric (composite_score: 0.0 to 1.0) + - 0.0-0.1: Clear spam, gibberish, test submissions + - 0.1-0.3: Real but very weak — business pitches, personal fundraising, vague ideas + - 0.3-0.5: Borderline — decent concept but missing details, unclear community benefit + - 0.5-0.7: Solid — clear project, community benefit, actionable plan + - 0.7-0.9: Strong — creative, specific, well-articulated, exactly what AF funds + - 0.9-1.0: Exceptional — innovative, clearly impactful, inspiring + + ## Feature Dimensions (Trust Equation: T = (C + R + I) / (1 + S)) + Numerator (higher = better): + - credibility: Clear budget, realistic plan, relevant expertise (0-1) + - reliability: Track record, prior work, organizational backing (0-1) + - intimacy: Connection to community, local ties, authentic voice (0-1) + Denominator (higher = worse): + - self_interest: Money primarily benefits applicant? (0-1) + Additional: + - specificity, creativity, budget_alignment, catalytic_potential, community_benefit, personal_voice (all 0-1) + - ai_spam_likelihood: Mass-generated? (0-1) + - ai_writing_likelihood: AI writing patterns? INFORMATIONAL ONLY (0-1) + + ## Flags: spam, ai_spam, duplicate, incomplete, wrong_location, business_pitch, personal_fundraising, low_effort + + ## Key Principles + - AF values creativity, community impact, and fun + - $1,000 is small — projects should be scoped appropriately + - "Too weird for traditional funders" = MORE awesome, not less + - ~28% of applications are typically review-worthy + PROMPT + end +end diff --git a/app/jobs/score_grant_job.rb b/app/jobs/score_grant_job.rb new file mode 100644 index 00000000..f804dc82 --- /dev/null +++ b/app/jobs/score_grant_job.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require "sucker_punch" + +# Scores a grant application asynchronously via the Signal Score pipeline. +# +# Enqueued after a new project is created. Uses the Anthropic Messages API +# to generate structured scores with the Trust Equation rubric. +# +# Usage: +# ScoreGrantJob.perform_async(project_id) +# ScoreGrantJob.new.perform(project_id) # synchronous, for testing +# +class ScoreGrantJob + include SuckerPunch::Job + + MAX_RETRIES = 3 + + def perform(project_id) + ActiveRecord::Base.connection_pool.with_connection do + # Atomic claim: only proceed if no score exists yet. + # This prevents double-scoring when multiple workers pick up the same job. + claimed = Project.where(id: project_id) + .where("metadata IS NULL OR metadata->'signal_score' IS NULL") + .update_all("metadata = COALESCE(metadata, '{}'::jsonb) || '{\"signal_score\": {\"status\": \"scoring\"}}'::jsonb") + return unless claimed > 0 + + project = Project.find(project_id) + + # Skip if scoring is disabled for this chapter + config = SignalScoreConfig.resolve(project.chapter) rescue {} + unless config["enabled"] + release_claim!(project_id) + return + end + + score_with_retries!(project, project_id) + rescue => e + Rails.logger.error("[SignalScore] Unexpected error scoring project ##{project_id}: #{e.class}: #{e.message}") + release_claim!(project_id) + end + end + + private + + def score_with_retries!(project, project_id) + attempts = 0 + + begin + attempts += 1 + scorer = SignalScorer.new(project) + scorer.score! + + Rails.logger.info( + "[SignalScore] Project ##{project_id} scored: " \ + "#{scorer.composite_score} (#{scorer.score_label})" + ) + rescue SignalScorer::ScoringError => e + Rails.logger.error("[SignalScore] Failed to score project ##{project_id}: #{e.message}") + + if attempts < MAX_RETRIES + sleep_time = 2**attempts # 2s, 4s + Rails.logger.info("[SignalScore] Retrying project ##{project_id} in #{sleep_time}s (attempt #{attempts + 1}/#{MAX_RETRIES})") + sleep(sleep_time) + retry + end + + Rails.logger.error("[SignalScore] Gave up on project ##{project_id} after #{MAX_RETRIES} attempts") + release_claim!(project_id) + end + end + + def release_claim!(project_id) + Project.where(id: project_id) + .where("metadata->'signal_score'->>'status' = 'scoring'") + .update_all("metadata = metadata - 'signal_score'") + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 1704e14d..a8298344 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -36,6 +36,7 @@ class Project < ApplicationRecord before_save :delete_photos after_create :analyze_for_spam + after_create :enqueue_signal_score # For dependency injection cattr_accessor :mailer @@ -272,6 +273,10 @@ def analyze_for_spam end end + def enqueue_signal_score + ScoreGrantJob.perform_async(id) if ENV["ANTHROPIC_API_KEY"].present? + end + # before save def ensure_funded_description if winner? diff --git a/app/models/signal_score_config.rb b/app/models/signal_score_config.rb new file mode 100644 index 00000000..305a4799 --- /dev/null +++ b/app/models/signal_score_config.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +# Per-chapter configuration for the Signal Score pipeline. +# +# A row with chapter_id = NULL is the global default. +# Chapter-specific rows override any fields they set (deep merge). +# +# JSONB columns: +# scoring_config: model, temperature, few_shot_count, score_threshold, grant_amount, currency +# prompt_overrides: system_prompt, rubric_addendum, few_shot_project_ids +# category_config: weights, disabled, custom +# +class SignalScoreConfig < ApplicationRecord + belongs_to :chapter, optional: true + + validates :chapter_id, uniqueness: { allow_nil: true } + validate :only_one_global_default + + scope :global_default, -> { where(chapter_id: nil) } + scope :enabled, -> { where(enabled: true) } + + # Resolve config for a chapter: deep-merge chapter overrides onto global default. + def self.resolve(chapter = nil) + global = global_default.first + return {} unless global + + base = { + "scoring_config" => global.scoring_config, + "prompt_overrides" => global.prompt_overrides, + "category_config" => global.category_config, + "enabled" => global.enabled, + } + + if chapter + override = find_by(chapter: chapter) + if override + base = deep_merge(base, { + "scoring_config" => override.scoring_config, + "prompt_overrides" => override.prompt_overrides, + "category_config" => override.category_config, + "enabled" => override.enabled, + }) + end + end + + base + end + + private + + def only_one_global_default + if chapter_id.nil? && self.class.global_default.where.not(id: id).exists? + errors.add(:chapter_id, "global default already exists") + end + end + + def self.deep_merge(base, overlay) + base.merge(overlay) do |_key, old_val, new_val| + if old_val.is_a?(Hash) && new_val.is_a?(Hash) + deep_merge(old_val, new_val) + elsif new_val.nil? || (new_val.is_a?(Hash) && new_val.empty?) + old_val + else + new_val + end + end + end +end diff --git a/app/views/projects/_project.html.erb b/app/views/projects/_project.html.erb index 3cf52221..d6885914 100644 --- a/app/views/projects/_project.html.erb +++ b/app/views/projects/_project.html.erb @@ -1,6 +1,6 @@
<%= "winner" if project.winner? %>" id="project<%= project.id %>" data-id="<%= project.id %>" data-controller="jump" data-jump-scroll-value="<%= local_assigns[:scroll] == true %>"> <% if !project.hidden? || @display_project_even_if_hidden %> - <%= render "projects/project_details", project: project, moderation: local_assigns[:moderation] %> + <%= render "projects/project_details", project: project, moderation: local_assigns[:moderation], full_view: local_assigns[:full_view] %> <% else %> <%= render "projects/project_hidden", project: project, moderation: local_assigns[:moderation] %> <% end %> diff --git a/app/views/projects/_project_details.html.erb b/app/views/projects/_project_details.html.erb index 3e226dd8..425352f4 100644 --- a/app/views/projects/_project_details.html.erb +++ b/app/views/projects/_project_details.html.erb @@ -26,6 +26,11 @@ + <% signal_score = project.metadata&.dig("signal_score") %> + <% if signal_score && signal_score["composite_score"] %> + <%= render "projects/signal_score_badge", signal_score: signal_score, always_show: local_assigns[:full_view] %> + <% end %> +
<%= project.created_at %> (<%= project.chapter.display_name %>) diff --git a/app/views/projects/_signal_score_badge.html.erb b/app/views/projects/_signal_score_badge.html.erb new file mode 100644 index 00000000..cd5144ff --- /dev/null +++ b/app/views/projects/_signal_score_badge.html.erb @@ -0,0 +1,60 @@ +<% + score = signal_score["composite_score"].to_f + score_pct = (score * 100).round + color_class = case score + when 0.7..1.0 then "signal-score--strong" + when 0.5...0.7 then "signal-score--solid" + when 0.3...0.5 then "signal-score--borderline" + else "signal-score--weak" + end + label = case score + when 0.7..1.0 then "Strong" + when 0.5...0.7 then "Solid" + when 0.3...0.5 then "Borderline" + when 0.1...0.3 then "Weak" + else "Low Signal" + end + category = signal_score["primary_category"]&.gsub(/[-_]/, " ")&.titleize + reason = signal_score["reason"] + tags = signal_score["tags"] || [] + features = signal_score["features"] || {} + always_show = local_assigns[:always_show] +%> + +
+
+ <%= score_pct %> + <%= label %> + <% if category.present? %> + <%= category %> + <% end %> + <% tags.first(3).each do |tag| %> + <%= tag %> + <% end %> +
+ + <% if reason.present? %> +

<%= reason %>

+ <% end %> + +
+ Scoring factors +
+ <% %w[credibility reliability intimacy self_interest specificity creativity budget_alignment catalytic_potential community_benefit personal_voice].each do |feat| %> + <% val = features[feat].to_f %> + <% feat_class = feat == "self_interest" ? (val > 0.5 ? "signal-score__feat--bad" : "signal-score__feat--good") : (val >= 0.5 ? "signal-score__feat--good" : "signal-score__feat--bad") %> +
+ <%= feat.gsub("_", " ") %> + <%= "%.1f" % val %> +
+ <% end %> +
+ <% ai_spam = features["ai_spam_likelihood"].to_f %> + <% ai_write = features["ai_writing_likelihood"].to_f %> + <% if ai_spam > 0.3 || ai_write > 0.5 %> +

+ ⚠️ AI signals: spam=<%= "%.1f" % ai_spam %>, writing=<%= "%.1f" % ai_write %> +

+ <% end %> +
+
diff --git a/app/views/projects/index.html.erb b/app/views/projects/index.html.erb index 6a38340c..d8c6c6f9 100644 --- a/app/views/projects/index.html.erb +++ b/app/views/projects/index.html.erb @@ -29,6 +29,8 @@
+ <% if @sort_mode.present? %><% end %> + <% if @signal_score_min.present? && @signal_score_min > 0 %><% end %>
type="checkbox"> @@ -39,6 +41,37 @@ <%= label_tag "funded", t(".funded-filter") %>
+
+ + +
+ +
+ <% sort_labels = { nil => "Latest", "score_desc" => "Score, highest", "score_asc" => "Score, lowest", "latest" => "Latest", "earliest" => "Earliest", "random" => "Random" } %> + + <%= sort_labels[@sort_mode] || "Default order" %> + + +
    + <% sort_labels.each do |val, label| %> +
  1. <%= link_to label, url_for(request.query_parameters.merge(sort: val).compact_blank) %>
    2. + <% end %> +
+
+ +
+ <% score_labels_map = { 0 => "All apps", 0.15 => "Hide low signal", 0.3 => "Borderline+ (≥0.3)", 0.5 => "Solid+ (≥0.5)", 0.7 => "Strong only (≥0.7)" } %> + + <%= score_labels_map[@signal_score_min] || "All apps" %> + + +
    + <% score_labels_map.each do |val, label| %> +
  1. <%= link_to label, url_for(request.query_parameters.merge(score_min: val)) %>
    2. + <% end %> +
+
+
diff --git a/config/signal_score/categories.json b/config/signal_score/categories.json new file mode 100644 index 00000000..e8c510b9 --- /dev/null +++ b/config/signal_score/categories.json @@ -0,0 +1,102 @@ +[ + { + "slug": "public-art", + "name": "Public Art", + "description": "Murals, installations, sculptures, street art, public creative works" + }, + { + "slug": "community-garden", + "name": "Community Garden", + "description": "Gardens, urban farming, composting, food growing, green spaces" + }, + { + "slug": "youth-education", + "name": "Youth & Education", + "description": "After-school programs, tutoring, mentoring, youth camps, student support" + }, + { + "slug": "music", + "name": "Music", + "description": "Instruments, bands, concerts, recording, music education, community music" + }, + { + "slug": "food-community", + "name": "Food & Community", + "description": "Cooking, community meals, food security, kitchen programs, nutrition" + }, + { + "slug": "film-media", + "name": "Film & Media", + "description": "Film, documentary, podcast, video, animation, media production" + }, + { + "slug": "literary", + "name": "Literary", + "description": "Books, libraries, reading programs, zines, publishing, writing workshops" + }, + { + "slug": "dance", + "name": "Dance", + "description": "Dance, ballet, movement, choreography, dance education" + }, + { + "slug": "theater", + "name": "Theater", + "description": "Theater, improv, puppets, drama, comedy, stage performance" + }, + { + "slug": "environment", + "name": "Environment", + "description": "Conservation, recycling, sustainability, ecology, tree planting, clean energy" + }, + { + "slug": "sports-recreation", + "name": "Sports & Recreation", + "description": "Bikes, skating, sports leagues, outdoor recreation, fitness, play" + }, + { + "slug": "festival-event", + "name": "Festival & Event", + "description": "Festivals, block parties, parades, celebrations, community gatherings" + }, + { + "slug": "technology-stem", + "name": "Technology & STEM", + "description": "Robotics, coding, maker spaces, digital literacy, science education" + }, + { + "slug": "women-girls", + "name": "Women & Girls", + "description": "Women's empowerment, girls in STEM, gender equity, women's health" + }, + { + "slug": "health-wellness", + "name": "Health & Wellness", + "description": "Mental health, disability support, therapy, wellness, medical access" + }, + { + "slug": "children-families", + "name": "Children & Families", + "description": "Childcare, family support, parenting resources, family activities" + }, + { + "slug": "civic-social-justice", + "name": "Civic & Social Justice", + "description": "Activism, voting, rights, advocacy, community organizing" + }, + { + "slug": "cultural-heritage", + "name": "Cultural Heritage", + "description": "Indigenous culture, preservation, history, museums, ancestral arts" + }, + { + "slug": "immigrant-refugee", + "name": "Immigrant & Refugee", + "description": "Immigrant services, ESL, refugee support, newcomer programs" + }, + { + "slug": "housing-homelessness", + "name": "Housing & Homelessness", + "description": "Shelter, housing, homelessness services, housing advocacy" + } +] diff --git a/context/memory/README.md b/context/memory/README.md new file mode 100644 index 00000000..f8933253 --- /dev/null +++ b/context/memory/README.md @@ -0,0 +1,13 @@ +# Signal Score — Knowledge Base + +This directory is a [basic-memory](https://github.com/basicmachines-co/basic-memory) project containing research notes, analysis reports, and pattern discovery for the Signal Score feature. + +## Structure + +``` +context/memory/ +├── README.md # This file +├── research/ # Pattern discovery, validation results +├── decisions/ # Architecture and design decisions +└── reference/ # AF handbook excerpts, scoring rubrics +``` diff --git a/context/memory/decisions/signal-score-architecture.md b/context/memory/decisions/signal-score-architecture.md new file mode 100644 index 00000000..186c82e0 --- /dev/null +++ b/context/memory/decisions/signal-score-architecture.md @@ -0,0 +1,33 @@ +# Signal Score Architecture Decisions + +## LLM-as-Judge over Embeddings +**Decision:** Use LLM direct evaluation, not embedding similarity. +**Rationale:** Embeddings add infrastructure for a problem that doesn't need it. LLMs can read and reason about full application text. Cost: ~$0.02/app with Haiku 4.5. + +## Trust Equation as Framework +**Decision:** Score using `T = (C + R + I) / (1 + S)` mapped to AF's own evaluation criteria. +**Rationale:** The Trusted Advisor framework maps directly to the Awesome Foundation's New Chapter Handbook criteria. Validated empirically with zero false negatives on 181 labeled apps. + +## Content Analysis, Not Bot Detection +**Decision:** Signal Score analyzes *what* was written. Existing SpamChecker/SpamClassifier handle *how* the form was filled out. +**Rationale:** Clean separation of concerns. No overlap. Signal Score runs async after submission. + +## Batch API for Scoring +**Decision:** Use Anthropic Batch API (not real-time). +**Rationale:** 50% cost reduction, no rate limit pressure, results within 24h. Good fit since scoring doesn't need to be instant. + +## Pattern Discovery Before Rubric +**Decision:** Use LLM as qualitative researcher first, then build rubric from discovered patterns. +**Rationale:** Avoids encoding assumptions. Patterns emerged from data, not preconceptions. + +## "Signal Score" Naming +**Decision:** Renamed from "Auto-Awesome Score" to "Signal Score." +**Rationale:** "Auto-Awesome" implies bypassing human review. Signal Score communicates that it surfaces signals for trustees to interpret. + +## Data Location +**Decision:** Historical data in `.scratch/data/` (gitignored). Source CSVs from privileged Google Drive folder. +**Rationale:** Data contains PII (applicant names, emails, project descriptions). Must not be committed to a public fork. + +## Secrets in .env.local +**Decision:** Use `.env.local` for personal API keys, not `.env`. +**Rationale:** `.env` could be checked in by accident. `.env.local` is the Rails/dotenv convention for personal secrets. `dotenv-rails` already in the Gemfile loads it automatically. diff --git a/context/memory/research/pattern-discovery-findings.md b/context/memory/research/pattern-discovery-findings.md new file mode 100644 index 00000000..7b34af89 --- /dev/null +++ b/context/memory/research/pattern-discovery-findings.md @@ -0,0 +1,37 @@ +# Pattern Discovery Findings + +## Method + +3 independent batches of 40 labeled Chicago applications analyzed via Claude Sonnet for qualitative patterns. Cross-batch synthesis to identify stable signals. + +## Stable Signals (consistent across all 3 batches) + +1. **Money destination is #1 discriminator** — Materials/supplies = funded. Self-payment/living expenses = hidden. If >50% budget goes to applicant → almost always hidden. + +2. **Local specificity signals authenticity** — Funded apps mention specific streets, organizations, aldermen. Hidden apps are geographically vague. + +3. **Writing quality barely matters** — Quirky, unpolished but authentic applications outperform polished corporate ones. + +4. **Scale fit is critical** — Projects perfectly sized for $1K succeed. "Drop in the bucket" requests get filtered. + +5. **Community connector density** — Funded apps average 3-5 named local partnerships. Hidden apps have 0-1. + +6. **Professional credentials hurt** — Academic degrees, awards, institutional affiliations correlate with HIDDEN status. + +## Additional Signals from Video Archive Analysis (39 videos, 4 analysis groups) + +### 4/4 Consensus (universal across all groups) +- **Catalytic Potential** — Does $1K unlock something bigger? +- **Creativity/Surprise** — Core AF identity, not a nice-to-have + +### 3/4 Consensus +- **Funding Orphan Score** — Too weird/small for traditional funders = more awesome +- **Agency Framing** — "I will build" vs. "please help us" + +### 2/4 Consensus +- **Boring Detection** — Not spam, just generic and uninspired +- **Validation Hunger** — First-time applicants where AF's belief matters most +- **Community Creation** — Builds new connections, not just serves existing + +## Calibration +Multiple chapters independently report ~28% of applications are review-worthy regardless of volume. diff --git a/context/memory/research/trust-equation-framework.md b/context/memory/research/trust-equation-framework.md new file mode 100644 index 00000000..e8251827 --- /dev/null +++ b/context/memory/research/trust-equation-framework.md @@ -0,0 +1,54 @@ +# Trust Equation Framework + +## Overview + +Signal Score uses Maister's Trust Equation as its scoring backbone: + +``` +T = (Credibility + Reliability + Intimacy) / (1 + Self-Interest) +``` + +Each dimension maps to observable signals in grant applications. + +## Dimension Mapping + +### Credibility (numerator) +- Clear budget with specific line items +- Realistic plan with timeline +- Relevant expertise or demonstrated competence +- Named vendors, partners, suppliers + +### Reliability (numerator) +- Track record: prior events, projects, grants +- Organizational backing or institutional support +- Evidence of follow-through (photos, links to past work) +- Repeat applicants with history + +### Intimacy (numerator) +- Connection to cause/community +- Local specificity: named neighborhoods, streets, aldermen +- Personal anecdotes and authentic voice +- Sub-city precision (ward, zip, not just city name) + +### Self-Interest (denominator — higher = worse) +- Money primarily benefits applicant (living expenses, tuition, salary) +- Business startup where applicant is primary beneficiary +- Self-payment > 50% of budget → almost always hidden +- Professional credential emphasis (awards, degrees) + +## Validation Results (Chicago Chapter, n=181) + +| Metric | Value | +|--------|-------| +| Funded avg score | 0.717 | +| Hidden avg score | 0.277 | +| Funded > 0.7 | 88% | +| Hidden < 0.3 | 65% | +| False negatives (funded < 0.3) | **0** | + +## Key Insight + +The Trust Equation maps directly to AF's own grantee evaluation criteria from the New Chapter Handbook: +1. Likelihood of short-timespan completion → Reliability + Specificity +2. Perceived ability of applicant → Credibility +3. Ability of grant to make a difference → Budget Alignment diff --git a/db/migrate/20260222_create_signal_score_configs.rb b/db/migrate/20260222_create_signal_score_configs.rb new file mode 100644 index 00000000..24d58562 --- /dev/null +++ b/db/migrate/20260222_create_signal_score_configs.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class CreateSignalScoreConfigs < ActiveRecord::Migration[7.0] + def change + create_table :signal_score_configs do |t| + t.references :chapter, null: true, foreign_key: true, index: { unique: true } + t.jsonb :scoring_config, null: false, default: {} + t.jsonb :prompt_overrides, null: false, default: {} + t.jsonb :category_config, null: false, default: {} + t.boolean :enabled, default: false, null: false + t.timestamps + end + + # Partial index for quick lookup of the global default (chapter_id IS NULL) + add_index :signal_score_configs, :enabled, + where: "chapter_id IS NULL", + name: "index_signal_score_configs_on_global_default" + end +end diff --git a/db/migrate/20260223_add_signal_score_index.rb b/db/migrate/20260223_add_signal_score_index.rb new file mode 100644 index 00000000..a09e38d5 --- /dev/null +++ b/db/migrate/20260223_add_signal_score_index.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class AddSignalScoreIndex < ActiveRecord::Migration[7.0] + def up + # Ensure metadata column exists (CI runs db:migrate from scratch; + # the squashed 001 migration doesn't include columns added later). + unless column_exists?(:projects, :metadata) + add_column :projects, :metadata, :jsonb, default: {} + add_index :projects, :metadata, using: :gin + end + + # Expression index on composite_score for efficient filtering and sorting. + # Without this, every WHERE/ORDER on the JSONB field does a full table scan. + execute <<~SQL + CREATE INDEX IF NOT EXISTS idx_projects_signal_score_composite + ON projects (((metadata->'signal_score'->>'composite_score')::float)) + WHERE metadata->'signal_score' IS NOT NULL; + SQL + end + + def down + execute "DROP INDEX IF EXISTS idx_projects_signal_score_composite;" + end +end diff --git a/docs/screenshots/signal-score/01-list-score-desc-2-projects.jpg b/docs/screenshots/signal-score/01-list-score-desc-2-projects.jpg new file mode 100644 index 00000000..1c71f15c Binary files /dev/null and b/docs/screenshots/signal-score/01-list-score-desc-2-projects.jpg differ diff --git a/docs/screenshots/signal-score/02-detail-scoring-factors-expanded.jpg b/docs/screenshots/signal-score/02-detail-scoring-factors-expanded.jpg new file mode 100644 index 00000000..32da30ab Binary files /dev/null and b/docs/screenshots/signal-score/02-detail-scoring-factors-expanded.jpg differ diff --git a/docs/screenshots/signal-score/03-detail-spam.jpg b/docs/screenshots/signal-score/03-detail-spam.jpg new file mode 100644 index 00000000..ce3ecb3c Binary files /dev/null and b/docs/screenshots/signal-score/03-detail-spam.jpg differ diff --git a/notebooks/01_eda.ipynb b/notebooks/01_eda.ipynb new file mode 100644 index 00000000..ecb26abd --- /dev/null +++ b/notebooks/01_eda.ipynb @@ -0,0 +1,276 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "39074e61", + "metadata": {}, + "source": [ + "# 01 - Exploratory Data Analysis\n", + "## Awesome Foundation Signal Score\n", + "\n", + "Corpus-level statistics, chapter distributions, label rates, and text field analysis\n", + "across 64,000+ grant applications.\n", + "\n", + "**Data source:** Parameterized via `AWESOMEBITS_DB` env var (defaults to `.scratch/data/awesomebits.duckdb`)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76102697", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.insert(0, '.')\n", + "from helpers import *\n", + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "setup_plotting()\n", + "con = connect()\n", + "print(f'Connected to: {DB_PATH}')\n", + "for t in con.execute('SHOW TABLES').fetchall():\n", + " cnt = con.execute(f'SELECT count(*) FROM {t[0]}').fetchone()[0]\n", + " print(f' {t[0]}: {cnt:,} rows')" + ] + }, + { + "cell_type": "markdown", + "id": "409c3c9f", + "metadata": {}, + "source": [ + "## Corpus Overview" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c2485e15", + "metadata": {}, + "outputs": [], + "source": [ + "df = con.execute('SELECT * FROM projects').df()\n", + "print(f'Total applications: {len(df):,}')\n", + "print(f'Date range: {df.created_at.min()} to {df.created_at.max()}')\n", + "print(f'Funded: {df.funded_on.notna().sum():,} ({df.funded_on.notna().mean():.1%})')\n", + "print(f'Hidden: {df.hidden_at.notna().sum():,} ({df.hidden_at.notna().mean():.1%})')\n", + "print(f'Unlabeled: {(df.funded_on.isna() & df.hidden_at.isna()).sum():,}')" + ] + }, + { + "cell_type": "markdown", + "id": "1d6b69eb", + "metadata": {}, + "source": [ + "## Chapter Distribution" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9c72c31d", + "metadata": {}, + "outputs": [], + "source": [ + "chapters = con.execute('''\n", + " SELECT c.name, c.country, c.inactive_at IS NOT NULL as inactive,\n", + " COUNT(*) as total,\n", + " COUNT(p.funded_on) as funded,\n", + " COUNT(p.hidden_at) as hidden,\n", + " COUNT(*) - COUNT(p.funded_on) - COUNT(p.hidden_at) as unlabeled\n", + " FROM projects p\n", + " JOIN chapters c ON p.chapter_id = c.id\n", + " GROUP BY c.name, c.country, inactive\n", + " ORDER BY total DESC\n", + "''').df()\n", + "\n", + "print(f'Active chapters: {(~chapters.inactive).sum()}')\n", + "print(f'Inactive chapters: {chapters.inactive.sum()}')\n", + "print(f'Countries: {chapters.country.nunique()}')\n", + "print()\n", + "print('Top 15 chapters by volume:')\n", + "print(chapters.head(15)[['name', 'country', 'total', 'funded', 'hidden']].to_string(index=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "008624c8", + "metadata": {}, + "outputs": [], + "source": [ + "# Applications per chapter (top 20)\n", + "top20 = chapters.head(20).copy()\n", + "fig, ax = plt.subplots(figsize=(12, 6))\n", + "ax.barh(top20['name'], top20['total'], color='steelblue')\n", + "ax.set_xlabel('Applications')\n", + "ax.set_title('Top 20 Chapters by Application Volume')\n", + "ax.invert_yaxis()\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "6060cfe5", + "metadata": {}, + "source": [ + "## Label Rates by Chapter\n", + "\n", + "Which chapters actively use the hide feature? This determines which chapters have enough ground truth for model validation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bc15043f", + "metadata": {}, + "outputs": [], + "source": [ + "# Chapters with enough labeled data for validation\n", + "validatable = chapters[(chapters.funded >= 10) & (chapters.hidden >= 10)].copy()\n", + "validatable['label_rate'] = ((validatable.funded + validatable.hidden) / validatable.total * 100).round(1)\n", + "validatable['fund_rate'] = (validatable.funded / (validatable.funded + validatable.hidden) * 100).round(1)\n", + "\n", + "print(f'Chapters with >= 10 funded AND >= 10 hidden: {len(validatable)}')\n", + "print()\n", + "print(validatable[['name', 'country', 'total', 'funded', 'hidden', 'label_rate', 'fund_rate']].to_string(index=False))" + ] + }, + { + "cell_type": "markdown", + "id": "bdacd01d", + "metadata": {}, + "source": [ + "## Temporal Distribution" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6a655908", + "metadata": {}, + "outputs": [], + "source": [ + "df['year'] = pd.to_datetime(df.created_at).dt.year\n", + "yearly = df.groupby('year').agg(\n", + " total=('id', 'count'),\n", + " funded=('funded_on', lambda x: x.notna().sum()),\n", + " hidden=('hidden_at', lambda x: x.notna().sum()),\n", + ").reset_index()\n", + "yearly['unlabeled'] = yearly.total - yearly.funded - yearly.hidden\n", + "\n", + "fig, ax = plt.subplots(figsize=(12, 5))\n", + "ax.bar(yearly.year, yearly.unlabeled, label='Unlabeled', color='lightgray')\n", + "ax.bar(yearly.year, yearly.hidden, bottom=yearly.unlabeled, label='Hidden', color='salmon')\n", + "ax.bar(yearly.year, yearly.funded, bottom=yearly.unlabeled + yearly.hidden, label='Funded', color='mediumseagreen')\n", + "ax.set_xlabel('Year')\n", + "ax.set_ylabel('Applications')\n", + "ax.set_title('Applications by Year and Label')\n", + "ax.legend()\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "0619e796", + "metadata": {}, + "source": [ + "## Text Field Analysis" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a98c24eb", + "metadata": {}, + "outputs": [], + "source": [ + "# Word counts per field\n", + "for field in TEXT_FIELDS:\n", + " df[f'wc_{field}'] = df[field].fillna('').apply(word_count)\n", + "df['wc_total'] = df['wc_about_me'] + df['wc_about_project'] + df['wc_use_for_money']\n", + "\n", + "print('Word count statistics:')\n", + "print(df[['wc_about_me', 'wc_about_project', 'wc_use_for_money', 'wc_total']].describe().round(1))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "48d91771", + "metadata": {}, + "outputs": [], + "source": [ + "# Word count distributions by label\n", + "df['label'] = df.apply(label_project, axis=1)\n", + "\n", + "fig, axes = plt.subplots(1, 3, figsize=(15, 5), sharey=True)\n", + "for ax, field in zip(axes, TEXT_FIELDS):\n", + " for label, color in [('funded', 'mediumseagreen'), ('hidden', 'salmon')]:\n", + " subset = df[df.label == label][f'wc_{field}']\n", + " ax.hist(subset, bins=50, alpha=0.6, color=color, label=label, range=(0, 1000))\n", + " ax.set_title(field.replace('_', ' ').title())\n", + " ax.set_xlabel('Word Count')\n", + " ax.legend()\n", + "axes[0].set_ylabel('Frequency')\n", + "plt.suptitle('Word Count Distribution: Funded vs Hidden', y=1.02)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a1352067", + "metadata": {}, + "outputs": [], + "source": [ + "# Empty field rates by label\n", + "for label in ['funded', 'hidden', 'unlabeled']:\n", + " subset = df[df.label == label]\n", + " empties = {f: (subset[f].fillna('').str.strip() == '').mean() for f in TEXT_FIELDS}\n", + " print(f'{label:>10}: ' + ' '.join(f'{f}={v:.1%}' for f, v in empties.items()))" + ] + }, + { + "cell_type": "markdown", + "id": "0585f18f", + "metadata": {}, + "source": [ + "## Geographic Distribution" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bdd7d773", + "metadata": {}, + "outputs": [], + "source": [ + "country_stats = con.execute('''\n", + " SELECT c.country, COUNT(*) as apps, COUNT(DISTINCT c.id) as chapters\n", + " FROM projects p JOIN chapters c ON p.chapter_id = c.id\n", + " GROUP BY c.country ORDER BY apps DESC\n", + "''').df()\n", + "\n", + "fig, ax = plt.subplots(figsize=(12, 6))\n", + "top_countries = country_stats.head(15)\n", + "ax.barh(top_countries.country, top_countries.apps, color='steelblue')\n", + "ax.set_xlabel('Applications')\n", + "ax.set_title('Applications by Country (Top 15)')\n", + "ax.invert_yaxis()\n", + "plt.tight_layout()\n", + "plt.show()\n", + "\n", + "print(f'\\nTotal countries: {len(country_stats)}')\n", + "print(f'English-speaking: {country_stats[country_stats.country.isin(ENGLISH_SPEAKING_COUNTRIES)].apps.sum():,} apps')" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/02_feature_engineering.ipynb b/notebooks/02_feature_engineering.ipynb new file mode 100644 index 00000000..2d74e0b4 --- /dev/null +++ b/notebooks/02_feature_engineering.ipynb @@ -0,0 +1,328 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "e1f3c814", + "metadata": {}, + "source": [ + "# 02 - Feature Engineering\n", + "## Pre-Scorer Features, TF-IDF, and Money Extraction\n", + "\n", + "Deterministic features computed before LLM scoring. Zero cost, runs locally.\n", + "\n", + "**Data source:** Parameterized via `AWESOMEBITS_DB` env var." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bb527170", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "sys.path.insert(0, '.')\n", + "from helpers import *\n", + "import pandas as pd\n", + "import numpy as np\n", + "from collections import Counter\n", + "from sklearn.feature_extraction.text import TfidfVectorizer\n", + "from price_parser import Price\n", + "import re\n", + "\n", + "setup_plotting()\n", + "con = connect()\n", + "df = con.execute('SELECT * FROM projects').df()\n", + "df['label'] = df.apply(label_project, axis=1)\n", + "df['text'] = df.apply(combined_text, axis=1)\n", + "print(f'Loaded {len(df):,} applications')" + ] + }, + { + "cell_type": "markdown", + "id": "4def555a", + "metadata": {}, + "source": [ + "## Pre-Scorer Feature Replication\n", + "\n", + "Replicate the Ruby `PreScorer` features in Python for analysis." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e59deda9", + "metadata": {}, + "outputs": [], + "source": [ + "def compute_features(row):\n", + " \"\"\"Replicate Ruby PreScorer features.\"\"\"\n", + " features = {}\n", + " texts = {f: str(row.get(f, '') or '') for f in TEXT_FIELDS}\n", + " all_text = ' '.join(t.strip() for t in texts.values() if t.strip())\n", + "\n", + " # Word counts\n", + " for f in TEXT_FIELDS:\n", + " features[f'wc_{f}'] = word_count(texts[f])\n", + " features['wc_total'] = sum(features[f'wc_{f}'] for f in TEXT_FIELDS)\n", + "\n", + " # Field length variance\n", + " counts = [features[f'wc_{f}'] for f in TEXT_FIELDS]\n", + " mean = np.mean(counts)\n", + " features['field_length_variance'] = np.var(counts)\n", + "\n", + " # Sentence stats\n", + " sentences = [s.strip() for s in re.split(r'[.!?]+', all_text) if s.strip()]\n", + " sent_lens = [len(s.split()) for s in sentences]\n", + " features['sentence_count'] = len(sentences)\n", + " features['avg_sentence_length'] = np.mean(sent_lens) if sent_lens else 0\n", + " features['sentence_length_variance'] = np.var(sent_lens) if sent_lens else 0\n", + "\n", + " # Punctuation\n", + " features['exclamation_count'] = all_text.count('!')\n", + " features['question_mark_count'] = all_text.count('?')\n", + "\n", + " # Content signals\n", + " features['url_count'] = len(re.findall(r'https?://\\S+', all_text))\n", + " features['money_mention_count'] = money_mention_count(all_text)\n", + " features['number_count'] = len(re.findall(r'\\b\\d[\\d,.]*\\b', all_text))\n", + " features['email_count'] = len(re.findall(r'\\S+@\\S+\\.\\S+', all_text))\n", + "\n", + " # Empty fields\n", + " features['empty_field_count'] = sum(1 for f in TEXT_FIELDS if not texts[f].strip())\n", + "\n", + " return pd.Series(features)\n", + "\n", + "# Compute for all projects\n", + "print('Computing pre-scorer features...')\n", + "feat_df = df.apply(compute_features, axis=1)\n", + "df = pd.concat([df, feat_df], axis=1)\n", + "print(f'Features computed: {list(feat_df.columns)}')" + ] + }, + { + "cell_type": "markdown", + "id": "ac4e513c", + "metadata": {}, + "source": [ + "## Feature Distributions: Funded vs Hidden" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6797c32", + "metadata": {}, + "outputs": [], + "source": [ + "# Compare feature distributions\n", + "labeled = df[df.label.isin(['funded', 'hidden'])].copy()\n", + "feature_cols = [c for c in feat_df.columns if c != 'empty_field_count']\n", + "\n", + "fig, axes = plt.subplots(3, 4, figsize=(16, 12))\n", + "for ax, col in zip(axes.flat, feature_cols):\n", + " for label, color in [('funded', 'mediumseagreen'), ('hidden', 'salmon')]:\n", + " data = labeled[labeled.label == label][col]\n", + " q99 = data.quantile(0.99)\n", + " ax.hist(data[data <= q99], bins=30, alpha=0.6, color=color, label=label, density=True)\n", + " ax.set_title(col, fontsize=10)\n", + " ax.legend(fontsize=8)\n", + "for ax in axes.flat[len(feature_cols):]:\n", + " ax.set_visible(False)\n", + "plt.suptitle('Feature Distributions: Funded vs Hidden', y=1.01)\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c00bdcf3", + "metadata": {}, + "outputs": [], + "source": [ + "# Feature means by label\n", + "summary = df.groupby('label')[feature_cols].mean().round(2).T\n", + "summary['funded_hidden_ratio'] = (summary['funded'] / summary['hidden'].replace(0, np.nan)).round(2)\n", + "print(summary[['funded', 'hidden', 'unlabeled', 'funded_hidden_ratio']].to_string())" + ] + }, + { + "cell_type": "markdown", + "id": "78117e43", + "metadata": {}, + "source": [ + "## Money Extraction Analysis\n", + "\n", + "Using regex pattern matching to find currency mentions across the full corpus. This feeds the `money_mention_count` pre-scorer feature." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "775f5158", + "metadata": {}, + "outputs": [], + "source": [ + "# Money mention distribution\n", + "print('Money mention counts by label:')\n", + "for label in ['funded', 'hidden', 'unlabeled']:\n", + " subset = df[df.label == label]\n", + " mc = subset.money_mention_count\n", + " print(f' {label:>10}: mean={mc.mean():.2f}, median={mc.median():.0f}, '\n", + " f'zero={((mc == 0).mean()):.1%}, 3+={((mc >= 3).mean()):.1%}')\n", + "\n", + "# Show examples of extracted money mentions\n", + "print('\\nSample money extractions (funded apps):')\n", + "funded_with_money = df[(df.label == 'funded') & (df.money_mention_count > 0)].sample(5, random_state=42)\n", + "for _, row in funded_with_money.iterrows():\n", + " mentions = extract_money_mentions(row.text)\n", + " print(f' [{row.money_mention_count} mentions] {mentions[:5]}')" + ] + }, + { + "cell_type": "markdown", + "id": "582f5869", + "metadata": {}, + "source": [ + "## TF-IDF Analysis\n", + "\n", + "Compute corpus-wide IDF values to identify distinctive vocabulary in funded vs hidden applications. The static IDF table can be exported for use in the Ruby pre-scorer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "79c73c21", + "metadata": {}, + "outputs": [], + "source": [ + "# Build TF-IDF on full corpus\n", + "tfidf = TfidfVectorizer(\n", + " max_features=10000,\n", + " stop_words='english',\n", + " min_df=10,\n", + " max_df=0.8,\n", + " ngram_range=(1, 2),\n", + ")\n", + "tfidf_matrix = tfidf.fit_transform(df.text.fillna(''))\n", + "feature_names = tfidf.get_feature_names_out()\n", + "idf_values = tfidf.idf_\n", + "\n", + "print(f'Vocabulary size: {len(feature_names):,}')\n", + "print(f'IDF range: {idf_values.min():.2f} to {idf_values.max():.2f}')\n", + "\n", + "# Most and least common terms\n", + "idf_df = pd.DataFrame({'term': feature_names, 'idf': idf_values}).sort_values('idf')\n", + "print('\\nMost common terms (lowest IDF):')\n", + "print(idf_df.head(20).to_string(index=False))\n", + "print('\\nRarest terms (highest IDF):')\n", + "print(idf_df.tail(20).to_string(index=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2f88f843", + "metadata": {}, + "outputs": [], + "source": [ + "# Terms most distinctive of funded vs hidden\n", + "funded_mask = (df.label == 'funded').values\n", + "hidden_mask = (df.label == 'hidden').values\n", + "\n", + "funded_mean = np.asarray(tfidf_matrix[funded_mask].mean(axis=0)).flatten()\n", + "hidden_mean = np.asarray(tfidf_matrix[hidden_mask].mean(axis=0)).flatten()\n", + "\n", + "diff = funded_mean - hidden_mean\n", + "diff_df = pd.DataFrame({'term': feature_names, 'funded_mean': funded_mean, 'hidden_mean': hidden_mean, 'diff': diff})\n", + "diff_df = diff_df.sort_values('diff', ascending=False)\n", + "\n", + "print('Terms most associated with FUNDED:')\n", + "print(diff_df.head(25)[['term', 'funded_mean', 'hidden_mean', 'diff']].to_string(index=False))\n", + "print('\\nTerms most associated with HIDDEN:')\n", + "print(diff_df.tail(25)[['term', 'funded_mean', 'hidden_mean', 'diff']].to_string(index=False))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ff6d2ec3", + "metadata": {}, + "outputs": [], + "source": [ + "# Visualize top distinctive terms\n", + "fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 8))\n", + "\n", + "top_funded = diff_df.head(20)\n", + "ax1.barh(top_funded.term, top_funded['diff'], color='mediumseagreen')\n", + "ax1.set_title('Top 20 Terms: Funded')\n", + "ax1.invert_yaxis()\n", + "\n", + "top_hidden = diff_df.tail(20).iloc[::-1]\n", + "ax2.barh(top_hidden.term, top_hidden['diff'].abs(), color='salmon')\n", + "ax2.set_title('Top 20 Terms: Hidden')\n", + "ax2.invert_yaxis()\n", + "\n", + "plt.suptitle('TF-IDF: Most Distinctive Terms by Label')\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "d19ed730", + "metadata": {}, + "source": [ + "## Export Static IDF Table\n", + "\n", + "Export the IDF values as JSON for use in the Ruby pre-scorer. This avoids recomputing TF-IDF at scoring time." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3d0637f0", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from pathlib import Path\n", + "\n", + "idf_export = dict(zip(feature_names.tolist(), idf_values.round(4).tolist()))\n", + "\n", + "out_path = Path(DB_PATH).parent / 'idf_table.json'\n", + "with open(out_path, 'w') as f:\n", + " json.dump(idf_export, f)\n", + "print(f'Exported {len(idf_export):,} IDF values to {out_path}')" + ] + }, + { + "cell_type": "markdown", + "id": "47052ed5", + "metadata": {}, + "source": [ + "## Feature Correlations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b21de230", + "metadata": {}, + "outputs": [], + "source": [ + "# Correlation matrix of pre-scorer features\n", + "corr = df[feature_cols].corr()\n", + "fig, ax = plt.subplots(figsize=(12, 10))\n", + "sns.heatmap(corr, annot=True, fmt='.2f', cmap='RdBu_r', center=0, ax=ax)\n", + "ax.set_title('Pre-Scorer Feature Correlations')\n", + "plt.tight_layout()\n", + "plt.show()" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/03_model_evaluation.ipynb b/notebooks/03_model_evaluation.ipynb new file mode 100644 index 00000000..3c9d3562 --- /dev/null +++ b/notebooks/03_model_evaluation.ipynb @@ -0,0 +1,404 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0eb0fd4e", + "metadata": {}, + "source": [ + "# 03 - Model Evaluation\n", + "## Batch Results, Model Comparison, and Cross-Chapter Analysis\n", + "\n", + "Analyze LLM scoring results from the Anthropic Batch API. Compare models, measure test-retest reliability, and validate across chapters.\n", + "\n", + "**Data source:** Parameterized via `AWESOMEBITS_DB` env var.\n", + "\n", + "**Score files:** Place batch result JSONs in `.scratch/data/` with names like `scores-haiku-run1.json`, `scores-sonnet-run1.json`, etc." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "661387a4", + "metadata": {}, + "outputs": [], + "source": [ + "import sys, json, glob\n", + "sys.path.insert(0, '.')\n", + "from helpers import *\n", + "import pandas as pd\n", + "import numpy as np\n", + "from pathlib import Path\n", + "from scipy import stats\n", + "\n", + "setup_plotting()\n", + "con = connect()\n", + "data_dir = Path(DB_PATH).parent\n", + "print(f'Data directory: {data_dir}')\n", + "print(f'Score files found:')\n", + "for f in sorted(data_dir.glob('scores-*.json')):\n", + " with open(f) as fh:\n", + " d = json.load(fh)\n", + " print(f' {f.name}: {len(d)} scores')" + ] + }, + { + "cell_type": "markdown", + "id": "2fcc822e", + "metadata": {}, + "source": [ + "## Load Scores\n", + "\n", + "Load all available score files and merge with project metadata." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4607f16c", + "metadata": {}, + "outputs": [], + "source": [ + "def load_scores(pattern='scores-*.json'):\n", + " \"\"\"Load score files into a dict of DataFrames keyed by filename stem.\"\"\"\n", + " results = {}\n", + " for path in sorted(data_dir.glob(pattern)):\n", + " with open(path) as f:\n", + " scores = json.load(f)\n", + " # Handle both list and dict formats\n", + " if isinstance(scores, dict):\n", + " rows = [{'project_id': int(k), **v} for k, v in scores.items()]\n", + " else:\n", + " rows = scores\n", + " sdf = pd.DataFrame(rows)\n", + " results[path.stem] = sdf\n", + " return results\n", + "\n", + "score_sets = load_scores()\n", + "print(f'Loaded {len(score_sets)} score sets: {list(score_sets.keys())}')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa9fc8f7", + "metadata": {}, + "outputs": [], + "source": [ + "# Merge scores with labels from DuckDB\n", + "projects = con.execute('''\n", + " SELECT p.id as project_id, p.title, p.funded_on, p.hidden_at,\n", + " c.name as chapter_name, c.country\n", + " FROM projects p\n", + " JOIN chapters c ON p.chapter_id = c.id\n", + "''').df()\n", + "projects['label'] = projects.apply(label_project, axis=1)\n", + "\n", + "# Merge each score set with labels\n", + "for name, sdf in score_sets.items():\n", + " merged = sdf.merge(projects, on='project_id', how='left')\n", + " score_sets[name] = merged\n", + " labeled = merged[merged.label.isin(['funded', 'hidden'])]\n", + " print(f'{name}: {len(merged)} total, {len(labeled)} labeled '\n", + " f'(funded={len(labeled[labeled.label==\"funded\"])}, '\n", + " f'hidden={len(labeled[labeled.label==\"hidden\"])})')" + ] + }, + { + "cell_type": "markdown", + "id": "3ad8aed5", + "metadata": {}, + "source": [ + "## Score Distributions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cfef3ce5", + "metadata": {}, + "outputs": [], + "source": [ + "# Composite score distribution per model\n", + "score_col = 'composite_score' # adjust if your JSON uses a different key\n", + "# Try to find the right column name\n", + "for name, sdf in score_sets.items():\n", + " candidates = [c for c in sdf.columns if 'score' in c.lower() and 'composite' in c.lower()]\n", + " if not candidates:\n", + " candidates = [c for c in sdf.columns if c in ('composite_score', 'score', 'total_score')]\n", + " if candidates:\n", + " score_col = candidates[0]\n", + " break\n", + "print(f'Using score column: {score_col}')\n", + "\n", + "fig, axes = plt.subplots(1, len(score_sets), figsize=(6 * len(score_sets), 5), squeeze=False)\n", + "for ax, (name, sdf) in zip(axes[0], score_sets.items()):\n", + " if score_col not in sdf.columns:\n", + " ax.set_title(f'{name} (no {score_col})')\n", + " continue\n", + " for label, color in [('funded', 'mediumseagreen'), ('hidden', 'salmon'), ('unlabeled', 'lightgray')]:\n", + " subset = sdf[sdf.label == label][score_col].dropna()\n", + " if len(subset):\n", + " ax.hist(subset, bins=20, alpha=0.6, color=color, label=f'{label} (n={len(subset)})', range=(0, 1))\n", + " ax.set_title(name)\n", + " ax.set_xlabel('Composite Score')\n", + " ax.legend()\n", + "plt.suptitle('Score Distributions by Model and Label')\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "c21b3810", + "metadata": {}, + "source": [ + "## Classification Metrics\n", + "\n", + "Using composite score thresholds, how well does each model separate funded from hidden?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2b4dee0d", + "metadata": {}, + "outputs": [], + "source": [ + "def evaluate_model(sdf, score_col, threshold=0.5):\n", + " \"\"\"Compute classification metrics at a given threshold.\"\"\"\n", + " labeled = sdf[sdf.label.isin(['funded', 'hidden'])].dropna(subset=[score_col]).copy()\n", + " labeled['predicted'] = (labeled[score_col] >= threshold).map({True: 'funded', False: 'hidden'})\n", + "\n", + " tp = ((labeled.label == 'funded') & (labeled.predicted == 'funded')).sum()\n", + " tn = ((labeled.label == 'hidden') & (labeled.predicted == 'hidden')).sum()\n", + " fp = ((labeled.label == 'hidden') & (labeled.predicted == 'funded')).sum()\n", + " fn = ((labeled.label == 'funded') & (labeled.predicted == 'hidden')).sum()\n", + "\n", + " precision = tp / (tp + fp) if (tp + fp) > 0 else 0\n", + " recall = tp / (tp + fn) if (tp + fn) > 0 else 0\n", + " f1 = 2 * precision * recall / (precision + recall) if (precision + recall) > 0 else 0\n", + " accuracy = (tp + tn) / len(labeled) if len(labeled) > 0 else 0\n", + "\n", + " return {\n", + " 'n': len(labeled), 'tp': tp, 'tn': tn, 'fp': fp, 'fn': fn,\n", + " 'precision': round(precision, 3), 'recall': round(recall, 3),\n", + " 'f1': round(f1, 3), 'accuracy': round(accuracy, 3),\n", + " 'funded_avg': round(labeled[labeled.label == 'funded'][score_col].mean(), 3),\n", + " 'hidden_avg': round(labeled[labeled.label == 'hidden'][score_col].mean(), 3),\n", + " }\n", + "\n", + "# Evaluate all models at multiple thresholds\n", + "thresholds = [0.3, 0.4, 0.5, 0.6, 0.7]\n", + "for name, sdf in score_sets.items():\n", + " if score_col not in sdf.columns:\n", + " print(f'{name}: no {score_col} column')\n", + " continue\n", + " print(f'\\n=== {name} ===')\n", + " rows = []\n", + " for t in thresholds:\n", + " m = evaluate_model(sdf, score_col, threshold=t)\n", + " rows.append({'threshold': t, **m})\n", + " print(pd.DataFrame(rows)[['threshold', 'precision', 'recall', 'f1', 'accuracy', 'fn', 'fp']].to_string(index=False))" + ] + }, + { + "cell_type": "markdown", + "id": "c3f10549", + "metadata": {}, + "source": [ + "## Model Comparison\n", + "\n", + "If multiple models scored the same applications, compare them head-to-head." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6536d1d", + "metadata": {}, + "outputs": [], + "source": [ + "if len(score_sets) >= 2:\n", + " names = list(score_sets.keys())\n", + " # Merge first two score sets on project_id\n", + " a_name, b_name = names[0], names[1]\n", + " a = score_sets[a_name][['project_id', score_col, 'label']].rename(columns={score_col: f'{a_name}'})\n", + " b = score_sets[b_name][['project_id', score_col]].rename(columns={score_col: f'{b_name}'})\n", + " comp = a.merge(b, on='project_id', how='inner')\n", + "\n", + " print(f'Comparing {a_name} vs {b_name}: {len(comp)} shared applications')\n", + "\n", + " # Correlation\n", + " r, p = stats.pearsonr(comp[a_name].dropna(), comp[b_name].dropna())\n", + " rho, _ = stats.spearmanr(comp[a_name].dropna(), comp[b_name].dropna())\n", + " print(f'Pearson r={r:.3f} (p={p:.2e})')\n", + " print(f'Spearman rho={rho:.3f}')\n", + "\n", + " # Score difference stats\n", + " comp['diff'] = comp[a_name] - comp[b_name]\n", + " print(f'Mean diff: {comp[\"diff\"].mean():.3f}')\n", + " print(f'Std diff: {comp[\"diff\"].std():.3f}')\n", + " print(f'Max abs diff: {comp[\"diff\"].abs().max():.3f}')\n", + "\n", + " fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 5))\n", + " colors = comp.label.map({'funded': 'mediumseagreen', 'hidden': 'salmon', 'unlabeled': 'lightgray'})\n", + " ax1.scatter(comp[a_name], comp[b_name], alpha=0.4, c=colors, s=20)\n", + " ax1.plot([0, 1], [0, 1], 'k--', alpha=0.3)\n", + " ax1.set_xlabel(a_name)\n", + " ax1.set_ylabel(b_name)\n", + " ax1.set_title(f'Score Comparison (r={r:.3f})')\n", + "\n", + " ax2.hist(comp['diff'], bins=40, color='steelblue', alpha=0.7)\n", + " ax2.axvline(0, color='black', linestyle='--', alpha=0.3)\n", + " ax2.set_xlabel('Score Difference')\n", + " ax2.set_title(f'Score Difference Distribution')\n", + " plt.tight_layout()\n", + " plt.show()\n", + "else:\n", + " print('Only one score set available. Submit additional batches for comparison.')" + ] + }, + { + "cell_type": "markdown", + "id": "baad6c12", + "metadata": {}, + "source": [ + "## Test-Retest Reliability\n", + "\n", + "If the same model was run twice on the same inputs, measure score stability. Look for score files like `scores-haiku-run1.json` and `scores-haiku-run2.json`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6627ca51", + "metadata": {}, + "outputs": [], + "source": [ + "# Find run pairs (same model, different runs)\n", + "import re as _re\n", + "run_pairs = {}\n", + "for name in score_sets:\n", + " m = _re.match(r'(scores-\\w+)-run(\\d+)', name)\n", + " if m:\n", + " base, run_num = m.group(1), int(m.group(2))\n", + " run_pairs.setdefault(base, {})[run_num] = name\n", + "\n", + "if run_pairs:\n", + " for base, runs in run_pairs.items():\n", + " if len(runs) < 2:\n", + " continue\n", + " r1_name = runs[min(runs)]\n", + " r2_name = runs[max(runs)]\n", + " r1 = score_sets[r1_name][['project_id', score_col]].rename(columns={score_col: 'run1'})\n", + " r2 = score_sets[r2_name][['project_id', score_col]].rename(columns={score_col: 'run2'})\n", + " paired = r1.merge(r2, on='project_id')\n", + "\n", + " r, _ = stats.pearsonr(paired.run1, paired.run2)\n", + " rho, _ = stats.spearmanr(paired.run1, paired.run2)\n", + " diff = (paired.run1 - paired.run2).abs()\n", + "\n", + " print(f'\\n=== {base} test-retest (n={len(paired)}) ===')\n", + " print(f'Pearson r={r:.4f}, Spearman rho={rho:.4f}')\n", + " print(f'Mean abs diff: {diff.mean():.4f}')\n", + " print(f'Max abs diff: {diff.max():.4f}')\n", + " print(f'Std diff: {(paired.run1 - paired.run2).std():.4f}')\n", + " # Threshold consistency\n", + " for t in [0.3, 0.5, 0.7]:\n", + " agree = ((paired.run1 >= t) == (paired.run2 >= t)).mean()\n", + " print(f' Threshold {t}: {agree:.1%} agreement')\n", + "else:\n", + " print('No test-retest pairs found. Run the same model twice and name files like:')\n", + " print(' scores-haiku-run1.json, scores-haiku-run2.json')" + ] + }, + { + "cell_type": "markdown", + "id": "1ef1db84", + "metadata": {}, + "source": [ + "## Cross-Chapter Analysis\n", + "\n", + "Do scores generalize across chapters, or are patterns locality-specific?\n", + "\n", + "Chapters with enough labeled data for validation:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0c22898", + "metadata": {}, + "outputs": [], + "source": [ + "# Show which chapters have enough data\n", + "chapter_stats = con.execute('''\n", + " SELECT c.name, c.country, c.inactive_at IS NOT NULL as inactive,\n", + " COUNT(*) as total, COUNT(p.funded_on) as funded, COUNT(p.hidden_at) as hidden\n", + " FROM projects p JOIN chapters c ON p.chapter_id = c.id\n", + " GROUP BY c.name, c.country, inactive\n", + " HAVING COUNT(p.funded_on) >= 10 AND COUNT(p.hidden_at) >= 10\n", + " ORDER BY total DESC\n", + "''').df()\n", + "print(chapter_stats.to_string(index=False))\n", + "print(f'\\n{len(chapter_stats)} chapters have enough labeled data for validation')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ca55338", + "metadata": {}, + "outputs": [], + "source": [ + "# Per-chapter score analysis (if scores include chapter info)\n", + "for name, sdf in score_sets.items():\n", + " if score_col not in sdf.columns or 'chapter_name' not in sdf.columns:\n", + " continue\n", + " print(f'\\n=== {name}: per-chapter scores ===')\n", + " ch_scores = sdf.groupby(['chapter_name', 'label'])[score_col].agg(['mean', 'count']).round(3)\n", + " ch_scores = ch_scores[ch_scores['count'] >= 5].reset_index()\n", + " # Pivot for readability\n", + " pivot = ch_scores.pivot_table(index='chapter_name', columns='label', values='mean')\n", + " if 'funded' in pivot.columns and 'hidden' in pivot.columns:\n", + " pivot['separation'] = (pivot['funded'] - pivot['hidden']).round(3)\n", + " print(pivot.sort_values('separation', ascending=False).to_string())\n", + " else:\n", + " print(pivot.to_string())" + ] + }, + { + "cell_type": "markdown", + "id": "65c481fa", + "metadata": {}, + "source": [ + "## Predicted Quality Ratio\n", + "\n", + "Does the ~28% quality ratio (from Chicago labeled data) hold when scoring unlabeled applications?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70edee5a", + "metadata": {}, + "outputs": [], + "source": [ + "for name, sdf in score_sets.items():\n", + " if score_col not in sdf.columns:\n", + " continue\n", + " print(f'\\n=== {name} ===')\n", + " for t in [0.3, 0.5, 0.7]:\n", + " unlabeled = sdf[sdf.label == 'unlabeled']\n", + " above = (unlabeled[score_col] >= t).mean()\n", + " print(f' Threshold {t}: {above:.1%} of unlabeled apps score above')\n", + " # Overall distribution\n", + " print(f' Unlabeled score stats: mean={sdf[sdf.label==\"unlabeled\"][score_col].mean():.3f}, '\n", + " f'median={sdf[sdf.label==\"unlabeled\"][score_col].median():.3f}')" + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks/README.md b/notebooks/README.md new file mode 100644 index 00000000..529c1d19 --- /dev/null +++ b/notebooks/README.md @@ -0,0 +1,38 @@ +# Signal Score Notebooks + +Jupyter notebooks for analyzing the Awesome Foundation Signal Score pipeline. + +## Setup + +```bash +pip install -r notebooks/requirements.txt +``` + +## Data + +Notebooks read from a DuckDB database. Default path: `.scratch/data/awesomebits.duckdb` + +Override with an environment variable: + +```bash +export AWESOMEBITS_DB=/path/to/your/awesomebits.duckdb +``` + +To build the database from CSV exports, run: + +```bash +ruby scripts/signal-score/import_data.rb +``` + +## Notebooks + +| Notebook | Description | +|----------|-------------| +| `01_eda.ipynb` | Corpus stats, chapter distributions, label rates, text analysis | +| `02_feature_engineering.ipynb` | Pre-scorer features, TF-IDF, money extraction, feature correlations | +| `03_model_evaluation.ipynb` | Batch results, model comparison, test-retest reliability, cross-chapter | + +## No Real Data + +These notebooks are git-tracked. They contain no real grant application text. +All analysis reads from a local DuckDB database that lives in `.scratch/` (gitignored). diff --git a/notebooks/helpers.py b/notebooks/helpers.py new file mode 100644 index 00000000..0dd9fcce --- /dev/null +++ b/notebooks/helpers.py @@ -0,0 +1,115 @@ +"""Shared helpers for Signal Score notebooks.""" + +import os +import re +from pathlib import Path + +import duckdb +import matplotlib.pyplot as plt +import seaborn as sns + +# --- Data source --- +DEFAULT_DB_PATH = os.path.join( + os.path.dirname(__file__), "..", ".scratch", "data", "awesomebits.duckdb" +) +DB_PATH = os.environ.get("AWESOMEBITS_DB", DEFAULT_DB_PATH) + + +def connect(read_only: bool = True) -> duckdb.DuckDBPyConnection: + """Open the DuckDB database.""" + path = str(Path(DB_PATH).resolve()) + return duckdb.connect(path, read_only=read_only) + + +# --- Plotting defaults --- +def setup_plotting(): + """Configure consistent plot styling.""" + sns.set_theme(style="whitegrid", palette="muted", font_scale=1.1) + plt.rcParams.update( + { + "figure.figsize": (10, 6), + "figure.dpi": 100, + "axes.titlesize": 14, + "axes.labelsize": 12, + } + ) + + +# --- Text helpers --- +TEXT_FIELDS = ["about_me", "about_project", "use_for_money"] + + +def combined_text(row) -> str: + """Combine the three text fields into one string.""" + parts = [str(row.get(f, "") or "") for f in TEXT_FIELDS] + return " ".join(p.strip() for p in parts if p.strip()) + + +def word_count(text: str) -> int: + """Count words in text.""" + return len(text.split()) if text.strip() else 0 + + +# --- Money extraction --- +# Currency symbols and written forms +CURRENCY_SYMBOLS = r"[\$\£\€\¥\₹\₩\₫\₱\₺\₽\฿\₿]" +CURRENCY_PREFIXED = r"(?:A|NZ|CA|HK|S|US|C)\$" +CURRENCY_CODES = ( + r"\b(?:USD|GBP|EUR|AUD|NZD|CAD|CHF|JPY|CNY|INR|BRL|MXN|ZAR|SEK|NOK|DKK|" + r"KRW|SGD|HKD|TWD|THB|MYR|PHP|IDR|VND|CZK|PLN|HUF|RON|BGN|HRK|RUB|TRY|" + r"ILS|AED|SAR|QAR|KWD|BHD|OMR|CLP|COP|PEN|ARS|UYU)\b" +) +CURRENCY_WORDS = ( + r"\b(?:dollars?|pounds?|euros?|yen|yuan|rupees?|rand|krona|kronor|" + r"francs?|pesos?|reais?|real|won|baht|ringgit|dong|crowns?|zloty|forints?|" + r"lira|shekels?|dirhams?|riyals?)\b" +) + +# Number patterns (handles 1,000 and 1.000 and 1 000) +NUMBER_PATTERN = r"\d[\d,.\s]*\d|\d+" + +# Combined money pattern +MONEY_PATTERN = re.compile( + rf"(?:" + rf"(?:{CURRENCY_PREFIXED}|{CURRENCY_SYMBOLS})\s*(?:{NUMBER_PATTERN})" # symbol before number + rf"|(?:{NUMBER_PATTERN})\s*(?:{CURRENCY_PREFIXED}|{CURRENCY_SYMBOLS})" # number before symbol + rf"|(?:{NUMBER_PATTERN})\s*(?:{CURRENCY_WORDS})" # number + written currency + rf"|(?:{CURRENCY_CODES})\s*(?:{NUMBER_PATTERN})" # ISO code + number + rf"|(?:{NUMBER_PATTERN})\s*(?:{CURRENCY_CODES})" # number + ISO code + rf")", + re.IGNORECASE, +) + + +def extract_money_mentions(text: str) -> list[str]: + """Extract money/currency mentions from free text.""" + return MONEY_PATTERN.findall(text) + + +def money_mention_count(text: str) -> int: + """Count money mentions in text.""" + return len(extract_money_mentions(text)) + + +# --- Label helpers --- +def label_project(row) -> str: + """Assign a label based on funded_on / hidden_at.""" + import pandas as pd + + if pd.notna(row.get("funded_on")): + return "funded" + if pd.notna(row.get("hidden_at")): + return "hidden" + return "unlabeled" + + +# --- Chapter helpers --- +ENGLISH_SPEAKING_COUNTRIES = { + "United States", + "Canada", + "Australia", + "United Kingdom", + "Ireland", + "New Zealand", + "Worldwide", +} diff --git a/notebooks/requirements.txt b/notebooks/requirements.txt new file mode 100644 index 00000000..14e81819 --- /dev/null +++ b/notebooks/requirements.txt @@ -0,0 +1,9 @@ +duckdb>=1.0 +pandas>=2.0 +numpy>=1.24 +matplotlib>=3.7 +seaborn>=0.13 +scikit-learn>=1.3 +price-parser>=0.3 +scipy>=1.11 +jupyterlab>=4.0 diff --git a/scripts/signal-score/anthropic_cache.rb b/scripts/signal-score/anthropic_cache.rb new file mode 100644 index 00000000..c8dbbe3c --- /dev/null +++ b/scripts/signal-score/anthropic_cache.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require "digest" +require "json" +require "fileutils" + +# File-based passthrough cache for Anthropic API responses. +# Cache key = SHA256(model + system_prompt + tool_schema + user_message) +# +# Usage: +# cache = AnthropicCache.new(".scratch/cache/anthropic") +# response = cache.fetch(request_params) { call_anthropic(request_params) } +# +class AnthropicCache + attr_reader :cache_dir, :stats + + def initialize(cache_dir = nil) + @cache_dir = cache_dir || File.expand_path("../../.scratch/cache/anthropic", __dir__) + FileUtils.mkdir_p(@cache_dir) + @stats = { hits: 0, misses: 0 } + end + + # Fetch a cached response or execute the block and cache the result. + def fetch(request_params) + key = cache_key(request_params) + path = cache_path(key) + + if File.exist?(path) + @stats[:hits] += 1 + JSON.parse(File.read(path)) + else + response = yield + File.write(path, JSON.pretty_generate(response)) + @stats[:misses] += 1 + response + end + end + + # Check if a response is cached without fetching. + def cached?(request_params) + File.exist?(cache_path(cache_key(request_params))) + end + + # Write a result directly (e.g., from batch API result parsing). + def write(request_params, response) + key = cache_key(request_params) + File.write(cache_path(key), JSON.pretty_generate(response)) + end + + # Read a cached result without calling API. + def read(request_params) + key = cache_key(request_params) + path = cache_path(key) + return nil unless File.exist?(path) + + JSON.parse(File.read(path)) + end + + # Number of cached responses. + def size + Dir.glob(File.join(@cache_dir, "*.json")).length + end + + # Clear all cached responses. + def clear! + Dir.glob(File.join(@cache_dir, "*.json")).each { |f| File.delete(f) } + end + + private + + def cache_key(params) + # Extract the parts that determine the response + model = params[:model] || params["model"] || "" + system = params[:system] || params["system"] || "" + tools = params[:tools] || params["tools"] || [] + messages = params[:messages] || params["messages"] || [] + + # Stable serialization for hashing + content = JSON.generate({ + model: model, + system: system, + tools: tools, + messages: messages, + }) + + Digest::SHA256.hexdigest(content) + end + + def cache_path(key) + File.join(@cache_dir, "#{key}.json") + end +end diff --git a/scripts/signal-score/import_data.rb b/scripts/signal-score/import_data.rb new file mode 100644 index 00000000..012080c3 --- /dev/null +++ b/scripts/signal-score/import_data.rb @@ -0,0 +1,119 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# Import CSV data into Parquet and DuckDB for Signal Score analysis. +# +# Prerequisites: +# - DuckDB CLI: `brew install duckdb` or `mise install duckdb` +# - CSV files in .scratch/data/ (see DATA SOURCES below) +# +# DATA SOURCES: +# Historical application data is exported from the Awesome Foundation +# production database and stored in a privileged Google Drive folder. +# Contact @divideby0 for access. +# +# Expected CSV files: +# - projects.csv (~64K rows) — all applications across all chapters +# - chapters.csv (~220 rows) — chapter metadata +# - comments.csv — trustee comments on applications +# +# Usage: +# ruby scripts/signal-score/import_data.rb +# ruby scripts/signal-score/import_data.rb --csv-only # Skip DuckDB +# ruby scripts/signal-score/import_data.rb --force # Overwrite existing + +require "fileutils" +require "open3" + +DATA_DIR = File.expand_path("../../.scratch/data", __dir__) +DB_PATH = File.join(DATA_DIR, "awesomebits.duckdb") + +TABLES = { + "projects" => "projects.csv", + "chapters" => "chapters.csv", +}.freeze + +def run(cmd) + puts " $ #{cmd}" + stdout, stderr, status = Open3.capture3(cmd) + unless status.success? + warn "ERROR: #{stderr}" + exit 1 + end + stdout +end + +def csv_to_parquet(csv_file, parquet_file, force: false) + if File.exist?(parquet_file) && !force + puts " ✓ #{File.basename(parquet_file)} exists (skip)" + return + end + + sql = <<~SQL + COPY (SELECT * FROM read_csv_auto('#{csv_file}')) + TO '#{parquet_file}' (FORMAT PARQUET); + SQL + run("duckdb -c \"#{sql}\"") + puts " ✓ #{File.basename(csv_file)} → #{File.basename(parquet_file)}" +end + +def load_to_duckdb(parquet_file, table_name, force: false) + sql = if force + "DROP TABLE IF EXISTS #{table_name}; CREATE TABLE #{table_name} AS SELECT * FROM read_parquet('#{parquet_file}');" + else + "CREATE TABLE IF NOT EXISTS #{table_name} AS SELECT * FROM read_parquet('#{parquet_file}');" + end + run("duckdb #{DB_PATH} -c \"#{sql}\"") + count = run("duckdb #{DB_PATH} -c \"SELECT count(*) FROM #{table_name};\"").strip.split("\n").last.strip + puts " ✓ #{table_name}: #{count} rows" +end + +def main + force = ARGV.include?("--force") + csv_only = ARGV.include?("--csv-only") + + puts "Signal Score Data Import" + puts "========================" + puts "Data dir: #{DATA_DIR}" + puts + + # Check for CSV files + missing = TABLES.values.reject { |f| File.exist?(File.join(DATA_DIR, f)) } + unless missing.empty? + warn "Missing CSV files in #{DATA_DIR}:" + missing.each { |f| warn " - #{f}" } + warn "\nCSV data must be downloaded from the privileged Google Drive folder." + warn "Contact @divideby0 for access." + exit 1 + end + + # CSV → Parquet + puts "Converting CSV → Parquet..." + TABLES.each do |table, csv| + csv_path = File.join(DATA_DIR, csv) + parquet_path = File.join(DATA_DIR, "#{table}.parquet") + csv_to_parquet(csv_path, parquet_path, force: force) + end + puts + + return if csv_only + + # Parquet → DuckDB + puts "Loading into DuckDB (#{DB_PATH})..." + TABLES.each do |table, _| + parquet_path = File.join(DATA_DIR, "#{table}.parquet") + load_to_duckdb(parquet_path, table, force: force) + end + + # Also load any additional parquet files + %w[chicago_validation sample_100].each do |extra| + parquet = File.join(DATA_DIR, "#{extra}.parquet") + next unless File.exist?(parquet) + load_to_duckdb(parquet, extra, force: force) + end + + puts + puts "Done. Query with: duckdb #{DB_PATH}" +end + +main diff --git a/scripts/signal-score/pre_scorer.rb b/scripts/signal-score/pre_scorer.rb new file mode 100644 index 00000000..5366965f --- /dev/null +++ b/scripts/signal-score/pre_scorer.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +# PreScorer — deterministic text features computed before LLM scoring. +# Zero cost, runs locally on application text fields. + +class PreScorer + FIELDS = %w[about_me about_project use_for_money].freeze + + attr_reader :features + + def initialize(project) + @project = project + @features = {} + end + + def analyze! + @features = {} + + # Word counts per field + FIELDS.each do |field| + text = field_text(field) + @features["word_count_#{field}"] = word_count(text) + end + @features["word_count_total"] = FIELDS.sum { |f| @features["word_count_#{f}"] } + + # Field length variance (low = suspicious uniformity, AI template signal) + counts = FIELDS.map { |f| @features["word_count_#{f}"].to_f } + @features["field_length_variance"] = variance(counts).round(2) + + # Sentence-level stats (AI tends toward uniform sentence length) + all_text = combined_text + sentences = all_text.split(/[.!?]+/).map(&:strip).reject(&:empty?) + lengths = sentences.map { |s| s.split(/\s+/).length } + @features["sentence_count"] = sentences.length + @features["avg_sentence_length"] = lengths.empty? ? 0.0 : (lengths.sum.to_f / lengths.length).round(1) + @features["sentence_length_variance"] = variance(lengths.map(&:to_f)).round(2) + + # Punctuation signals + @features["exclamation_count"] = all_text.count("!") + @features["question_mark_count"] = all_text.count("?") + + # Content signals + @features["url_count"] = all_text.scan(%r{https?://\S+}).length + @features["number_count"] = all_text.scan(/\b\d[\d,.]*\b/).length + @features["email_count"] = all_text.scan(/\S+@\S+\.\S+/).length + + # Money extraction (international currencies) + money_matches = extract_money_mentions(all_text) + @features["money_mention_count"] = money_matches.length + + # Capitalized words (proxy for proper nouns / named entities) + words = all_text.split(/\s+/) + # Skip first word of each sentence + mid_sentence_caps = words.each_cons(2).count { |prev, word| + !prev.match?(/[.!?]\z/) && word.match?(/\A[A-Z][a-z]/) + } + @features["capitalized_word_count"] = mid_sentence_caps + @features["capitalized_word_density"] = if words.length > 10 + (mid_sentence_caps.to_f / words.length * 100).round(1) + else + 0.0 + end + + # Empty field detection + @features["empty_field_count"] = FIELDS.count { |f| field_text(f).strip.empty? } + + # Has images (title/about fields mention image/photo/picture URLs) + @features["has_image_reference"] = all_text.match?(/\.(jpg|jpeg|png|gif|webp|bmp)/i) ? 1 : 0 + + # TF-IDF features (requires static IDF table) + if self.class.idf_table + tfidf_scores = compute_tfidf(all_text) + @features["tfidf_mean"] = tfidf_scores.empty? ? 0.0 : (tfidf_scores.values.sum / tfidf_scores.length.to_f).round(4) + @features["tfidf_max"] = tfidf_scores.empty? ? 0.0 : tfidf_scores.values.max.round(4) + @features["tfidf_unique_terms"] = tfidf_scores.count { |_, v| v > 0 } + end + + @features + end + + # Load static IDF table (computed from full corpus via notebook 02) + def self.idf_table + @idf_table + end + + def self.load_idf(path) + require "json" + @idf_table = JSON.parse(File.read(path)) + end + + private + + def field_text(name) + @project[name].to_s + end + + def combined_text + FIELDS.map { |f| field_text(f).strip }.reject(&:empty?).join(" ") + end + + def word_count(text) + text.strip.empty? ? 0 : text.strip.split(/\s+/).length + end + + def variance(values) + return 0.0 if values.length < 2 + mean = values.sum / values.length.to_f + values.sum { |v| (v - mean)**2 } / values.length.to_f + end + + # International money/currency extraction + CURRENCY_SYMBOLS = /[\$\£\€\¥\₹\₩\₫\₱\₺\₽\฿]/ + CURRENCY_PREFIXED = /(?:A|NZ|CA|HK|S|US|C)\$/ + CURRENCY_CODES = /\b(?:USD|GBP|EUR|AUD|NZD|CAD|CHF|JPY|CNY|INR|BRL|MXN|ZAR|SEK|NOK|DKK|KRW|SGD|HKD|TWD|THB|MYR|PHP|IDR|VND|CZK|PLN|HUF|RON|RUB|TRY|ILS|AED|SAR|CLP|COP|PEN|ARS)\b/i + CURRENCY_WORDS = /\b(?:dollars?|pounds?|euros?|yen|yuan|rupees?|rand|krona|kronor|francs?|pesos?|reais?|real|won|baht|ringgit|crowns?|zloty|forints?|lira|shekels?|dirhams?|riyals?)\b/i + NUMBER_PAT = /\d[\d,.\s]*\d|\d+/ + + MONEY_PATTERN = Regexp.union( + /(?:#{CURRENCY_PREFIXED}|#{CURRENCY_SYMBOLS})\s*#{NUMBER_PAT}/, + /#{NUMBER_PAT}\s*(?:#{CURRENCY_PREFIXED}|#{CURRENCY_SYMBOLS})/, + /#{NUMBER_PAT}\s*#{CURRENCY_WORDS}/, + /#{CURRENCY_CODES}\s*#{NUMBER_PAT}/, + /#{NUMBER_PAT}\s*#{CURRENCY_CODES}/ + ) + + def extract_money_mentions(text) + text.scan(MONEY_PATTERN) + end + + # TF-IDF: compute term frequencies against static IDF table + def compute_tfidf(text) + idf = self.class.idf_table + return {} unless idf + + # Tokenize: lowercase, split on non-word, reject short + words = text.downcase.split(/\W+/).reject { |w| w.length < 2 } + return {} if words.empty? + + # Term frequency (normalized) + tf = Hash.new(0) + words.each { |w| tf[w] += 1 } + tf.transform_values! { |c| c.to_f / words.length } + + # TF-IDF for terms in our IDF table + scores = {} + tf.each do |term, freq| + scores[term] = freq * idf[term] if idf.key?(term) + end + scores + end +end diff --git a/scripts/signal-score/prompt_builder.rb b/scripts/signal-score/prompt_builder.rb new file mode 100644 index 00000000..9a65d979 --- /dev/null +++ b/scripts/signal-score/prompt_builder.rb @@ -0,0 +1,259 @@ +# frozen_string_literal: true + +require "json" + +# Assembles scoring prompts with chapter-specific config overrides. +# +# Config resolution order: +# 1. Chapter-specific overrides (from DB or .scratch/prompts/{slug}.json) +# 2. Global default (.scratch/prompts/default.json or hardcoded) +# +# Usage: +# builder = PromptBuilder.new(chapter: "Chicago, IL") +# system_prompt = builder.system_prompt +# tool_schema = builder.score_tool +# few_shot = builder.few_shot_text(funded_projects, hidden_projects) +# +class PromptBuilder + CATEGORIES_PATH = File.expand_path("../../config/signal_score/categories.json", __dir__) + PROMPTS_DIR = File.expand_path("../../.scratch/prompts", __dir__) + + attr_reader :config + + def initialize(chapter: nil, config_override: nil) + @config = resolve_config(chapter, config_override) + end + + # --- Prompt assembly --- + + def system_prompt + # Full override takes precedence + return @config["system_prompt"] if @config["system_prompt"] + + base = default_system_prompt + addendum = @config["rubric_addendum"] + addendum ? "#{base}\n\n#{addendum}" : base + end + + def score_tool + SCORE_TOOL + end + + def category_tool + CATEGORY_TOOL + end + + def categories + @categories ||= begin + custom = @config.dig("category_config", "custom") || [] + disabled = @config.dig("category_config", "disabled") || [] + base = canonical_categories.reject { |c| disabled.include?(c["slug"]) } + base + custom + end + end + + def category_slugs + categories.map { |c| c["slug"] } + end + + def model + @config["model"] || "claude-haiku-4-5-20251001" + end + + def grant_amount + @config["grant_amount"] || 1000 + end + + def currency + @config["currency"] || "USD" + end + + def few_shot_count + @config["few_shot_count"] || 15 + end + + # Build the few-shot examples text block. + def few_shot_text(funded_projects, hidden_projects) + n = few_shot_count + + text = "## EXAMPLES OF FUNDED (WINNING) APPLICATIONS\n\n" + funded_projects.first(n).each do |p| + text += "### Project #{p['id']} — FUNDED\n#{format_application(p)}\n\n" + end + + text += "## EXAMPLES OF HIDDEN (REJECTED/FILTERED) APPLICATIONS\n\n" + hidden_projects.first(n).each do |p| + reason = p["hidden_reason"] ? " (reason: #{p['hidden_reason']})" : "" + text += "### Project #{p['id']} — HIDDEN#{reason}\n#{format_application(p)}\n\n" + end + + text + end + + def format_application(project) + parts = ["Title: #{project['title'] || '(empty)'}"] + parts << "Chapter: #{project['chapter_name']}" if project["chapter_name"] + %w[about_me about_project use_for_money].each do |f| + val = project[f].to_s.strip + parts << "#{f.tr('_', ' ').split.map(&:capitalize).join(' ')}: #{val}" unless val.empty? + end + parts.join("\n") + end + + private + + def resolve_config(chapter, override) + base = load_default_config + if chapter + chapter_config = load_chapter_config(chapter) + base = deep_merge(base, chapter_config) if chapter_config + end + base = deep_merge(base, override) if override + base + end + + def load_default_config + path = File.join(PROMPTS_DIR, "default.json") + File.exist?(path) ? JSON.parse(File.read(path)) : {} + end + + def load_chapter_config(chapter) + slug = chapter.downcase.gsub(/[^a-z0-9]+/, "-").gsub(/-+$/, "") + path = File.join(PROMPTS_DIR, "chapters", "#{slug}.json") + File.exist?(path) ? JSON.parse(File.read(path)) : nil + end + + def canonical_categories + @canonical_categories ||= JSON.parse(File.read(CATEGORIES_PATH)) + end + + def deep_merge(base, overlay) + base.merge(overlay) do |_key, old_val, new_val| + if old_val.is_a?(Hash) && new_val.is_a?(Hash) + deep_merge(old_val, new_val) + else + new_val + end + end + end + + # --- Default system prompt (hardcoded fallback) --- + + def default_system_prompt + amount = grant_amount + curr = currency == "USD" ? "$" : currency + + <<~PROMPT + You are an expert grant application screener for the Awesome Foundation. + + The Awesome Foundation is a global network of volunteer "micro-trustees" who each chip in to award #{curr}#{amount} grants for awesome projects. No strings attached — the money goes to creative, community-benefiting, unique ideas. + + Score each application using the score_application tool. Extract structured features to help trustees prioritize their review. + + ## Scoring Rubric (composite_score: 0.0 to 1.0) + + - 0.0–0.1: Clear spam, gibberish, test submissions, or AI-generated mass submissions + - 0.1–0.3: Real but very weak — business pitches, personal fundraising, vague ideas, wrong location + - 0.3–0.5: Borderline — decent concept but missing details, unclear community benefit, generic + - 0.5–0.7: Solid — clear project, community benefit, actionable plan, reasonable for #{curr}#{amount} + - 0.7–0.9: Strong — creative, specific, well-articulated, exactly what AF funds + - 0.9–1.0: Exceptional — innovative, clearly impactful, inspiring, would excite any trustee + + ## Feature Dimensions + + ### Trust Equation: T = (Credibility + Reliability + Intimacy) / (1 + Self-Interest) + + **Numerator (higher = better):** + - credibility: Clear budget, realistic plan, relevant expertise. (0=vague, 1=detailed budget + clear execution) + - reliability: Track record, prior work, organizational backing. (0=no evidence, 1=strong track record) + - intimacy: Connection to community, local ties, authentic voice. Named neighborhoods, specific orgs, personal anecdotes. Sub-city precision (ward/zip) scores higher than city-level. (0=generic, 1=deeply embedded) + + **Denominator (higher = worse):** + - self_interest: Money primarily benefits applicant? Living expenses, tuition, business startup, self-payment >50% of budget. (0=community-benefiting, 1=self-serving) + + ### Additional Signals + + - specificity: Concrete details — costs, addresses, timelines, partner names. (0=vague, 1=highly specific) + - creativity: How original, unique, fun, or surprising? "The opposite of whimsy is boring, not serious." (0=generic, 1=delightfully novel) + - budget_alignment: Can #{curr}#{amount} meaningfully complete this? Is it a "drop in the bucket" or perfectly sized? (0=wildly mismatched, 1=perfectly scoped) + - catalytic_potential: Does #{curr}#{amount} unlock something bigger — prototype, proof-of-concept, career catalyst? (0=standalone purchase, 1=unlocks outsized outcomes) + - community_benefit: Clear benefit beyond the applicant? Creating new connections scores higher than serving existing community. (0=purely personal, 1=broad community impact + new connections) + - personal_voice: Does it sound like a real person? Quirky details, informal language, personal anecdotes = POSITIVE. Over-polished corporate language = NEGATIVE. (0=robotic/templated, 1=authentic and personal) + + ### AI Detection (two separate signals) + + - ai_spam_likelihood: Mass-generated generic proposal — no personal details, no local knowledge. (0=clearly genuine, 1=almost certainly mass-generated) + - ai_writing_likelihood: Exhibits AI writing patterns? Uniform sentences, significance inflation, promotional language. INFORMATIONAL ONLY — do not factor into composite_score. (0=clearly human, 1=heavily AI-styled) + + ## Flags (include any that apply) + spam, ai_spam, duplicate, incomplete, wrong_location, business_pitch, personal_fundraising, low_effort + + ## Key Principles + - AF values creativity, community impact, and fun + - #{curr}#{amount} is small — projects should be scoped appropriately + - Professional credentials are neutral-to-negative + - Projects "too weird for traditional funders" = MORE awesome, not less + - ~28% of applications are typically review-worthy + PROMPT + end + + # --- Tool schemas --- + + SCORE_TOOL = { + "name" => "score_application", + "description" => "Score a grant application and extract structured features", + "input_schema" => { + "type" => "object", + "required" => %w[composite_score reason flags features], + "properties" => { + "composite_score" => { "type" => "number", "description" => "Overall score 0.0-1.0" }, + "reason" => { "type" => "string", "description" => "One-sentence explanation" }, + "flags" => { "type" => "array", "items" => { "type" => "string" } }, + "features" => { + "type" => "object", + "required" => %w[ + credibility reliability intimacy self_interest + specificity creativity budget_alignment catalytic_potential + community_benefit personal_voice + ai_spam_likelihood ai_writing_likelihood + ], + "properties" => %w[ + credibility reliability intimacy self_interest + specificity creativity budget_alignment catalytic_potential + community_benefit personal_voice + ai_spam_likelihood ai_writing_likelihood + ].each_with_object({}) { |k, h| h[k] = { "type" => "number" } }, + }, + }, + }, + }.freeze + + def self.build_category_tool + cats = JSON.parse(File.read(CATEGORIES_PATH)) + { + "name" => "categorize_application", + "description" => "Categorize a grant application with thematic scores and freeform tags", + "input_schema" => { + "type" => "object", + "required" => %w[categories tags primary_category], + "properties" => { + "primary_category" => { "type" => "string", "description" => "The single most fitting category slug" }, + "categories" => { + "type" => "object", + "description" => "Confidence score 0-1 for each category (omit below 0.2)", + "properties" => cats.each_with_object({}) { |c, h| + h[c["slug"]] = { "type" => "number", "description" => c["description"] } + }, + }, + "tags" => { + "type" => "array", + "items" => { "type" => "string" }, + "description" => "3-5 freeform keyword tags for themes not in canonical categories", + }, + }, + }, + }.freeze + end + + CATEGORY_TOOL = build_category_tool +end diff --git a/scripts/signal-score/score_grants.rb b/scripts/signal-score/score_grants.rb new file mode 100644 index 00000000..516a3cc3 --- /dev/null +++ b/scripts/signal-score/score_grants.rb @@ -0,0 +1,395 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# score_grants.rb — Signal Score batch scoring via Anthropic API +# +# Usage: +# ruby scripts/signal-score/score_grants.rb build-sample --count 400 +# ruby scripts/signal-score/score_grants.rb build-batch [--model claude-haiku-4-5-20251001] +# ruby scripts/signal-score/score_grants.rb submit --input .scratch/data/batch-haiku.jsonl +# ruby scripts/signal-score/score_grants.rb results --batch-id +# ruby scripts/signal-score/score_grants.rb pre-score # Run PreScorer on sample +# +# Requires: ANTHROPIC_API_KEY in .env.local + +require "json" +require "net/http" +require "uri" +require "fileutils" +require_relative "pre_scorer" +require_relative "prompt_builder" +require_relative "anthropic_cache" + +DATA_DIR = File.expand_path("../../.scratch/data", __dir__) +DB_PATH = File.join(DATA_DIR, "awesomebits.duckdb") + +MODELS = { + "haiku" => "claude-haiku-4-5-20251001", + "sonnet" => "claude-sonnet-4-20250514", + "opus" => "claude-opus-4-0-20250514", +}.freeze + +CACHE = AnthropicCache.new + +# --- DuckDB helper --- + +def duckdb(sql) + result = `duckdb #{DB_PATH} -json -c #{Shellwords.escape(sql)} 2>&1` + raise "DuckDB error: #{result}" unless $?.success? + JSON.parse(result.empty? ? "[]" : result) +end + +# --- Tool schema for structured output --- + +# --- Prompt builder instance --- + +def prompt_builder(chapter: nil) + PromptBuilder.new(chapter: chapter) +end + +def build_user_message(builder, project, examples_text = nil) + app_text = builder.format_application(project) + if examples_text + "Here are examples of previously funded and hidden applications:\n\n#{examples_text}\n\n---\n\nNow score this application:\n\n#{app_text}" + else + "Score this application:\n\n#{app_text}" + end +end + +def build_few_shot_examples(builder) + labeled = duckdb(<<~SQL) + SELECT cv.*, c.name as chapter_name + FROM chicago_validation cv + LEFT JOIN chapters c ON cv.chapter_id = c.id + ORDER BY actual_label, hidden_reason + SQL + + funded = labeled.select { |p| p["actual_label"] == "funded" } + hidden = labeled.select { |p| p["actual_label"] == "hidden" } + + builder.few_shot_text(funded, hidden) +end + +# --- Commands --- + +def cmd_build_sample(count:) + puts "Building stratified sample of #{count} Chicago apps..." + + labeled = duckdb("SELECT count(*) as n FROM chicago_validation")[0]["n"] + unlabeled_needed = count - labeled + puts " #{labeled} labeled + #{unlabeled_needed} unlabeled needed" + + # Stratified by year + years = duckdb(<<~SQL) + SELECT extract(year from p.created_at::date) as yr, count(*) as n + FROM projects p JOIN chapters c ON p.chapter_id = c.id + WHERE c.name = 'Chicago, IL' + AND p.funded_on IS NULL AND p.hidden_at IS NULL + GROUP BY 1 ORDER BY 1 + SQL + + total_unlabeled = years.sum { |y| y["n"] } + sample_ids = [] + + years.each do |y| + yr_count = [(y["n"].to_f / total_unlabeled * unlabeled_needed).round, y["n"]].min + next if yr_count == 0 + + ids = duckdb(<<~SQL) + SELECT p.id FROM projects p JOIN chapters c ON p.chapter_id = c.id + WHERE c.name = 'Chicago, IL' + AND p.funded_on IS NULL AND p.hidden_at IS NULL + AND extract(year from p.created_at::date) = #{y['yr']} + ORDER BY random() + LIMIT #{yr_count} + SQL + sample_ids += ids.map { |r| r["id"] } + puts " #{y['yr']}: #{yr_count}/#{y['n']} unlabeled" + end + + # Build the full sample: labeled + unlabeled + sql = <<~SQL + CREATE OR REPLACE TABLE sample_400 AS + SELECT p.*, c.name as chapter_name, + CASE + WHEN p.funded_on IS NOT NULL THEN 'funded' + WHEN p.hidden_at IS NOT NULL THEN 'hidden' + ELSE 'unlabeled' + END as actual_label + FROM projects p + LEFT JOIN chapters c ON p.chapter_id = c.id + WHERE p.id IN ( + SELECT id FROM chicago_validation + UNION ALL + SELECT unnest([#{sample_ids.join(",")}]) + ) + SQL + duckdb(sql) + + result = duckdb("SELECT actual_label, count(*) as n FROM sample_400 GROUP BY 1 ORDER BY 1") + result.each { |r| puts " #{r['actual_label']}: #{r['n']}" } + + total = duckdb("SELECT count(*) as n FROM sample_400")[0]["n"] + puts "\nSample: #{total} apps saved to sample_400 table" + + # Export to parquet + duckdb("COPY sample_400 TO '#{DATA_DIR}/sample_400.parquet' (FORMAT PARQUET)") + puts "Exported to #{DATA_DIR}/sample_400.parquet" +end + +def cmd_pre_score + puts "Running PreScorer on sample_400..." + + projects = duckdb("SELECT * FROM sample_400") + results = {} + + projects.each do |p| + scorer = PreScorer.new(p) + scorer.analyze! + results[p["id"]] = scorer.features + end + + output = File.join(DATA_DIR, "pre_scores.json") + File.write(output, JSON.pretty_generate(results)) + puts "Wrote #{results.length} pre-scores to #{output}" + + # Summary stats + totals = results.values.map { |f| f["word_count_total"] } + puts "\nWord count stats:" + puts " Min: #{totals.min}, Max: #{totals.max}, Avg: #{(totals.sum.to_f / totals.length).round(0)}" + puts " Median: #{totals.sort[totals.length / 2]}" + + empty = results.values.count { |f| f["empty_field_count"] > 0 } + puts " Apps with empty fields: #{empty}/#{results.length}" +end + +def cmd_build_batch(model_alias:, chapter: nil, no_fewshot: false) + builder = prompt_builder(chapter: chapter) + model = MODELS[model_alias] || model_alias + puts "Building batch for model: #{model} (#{model_alias})" + puts " Chapter: #{chapter || 'default'}, few-shot: #{!no_fewshot}" + + examples_text = no_fewshot ? nil : build_few_shot_examples(builder) + projects = duckdb("SELECT * FROM sample_400") + puts " #{projects.length} projects to score" + + # Skip cached requests + skipped = 0 + lines = [] + projects.each do |p| + request_params = { + model: model, + system: builder.system_prompt, + tools: [builder.score_tool], + messages: [{ role: "user", content: build_user_message(builder, p, examples_text) }], + } + + if CACHE.cached?(request_params) + skipped += 1 + next + end + + lines << { + custom_id: "project-#{p['id']}", + params: { + model: model, + max_tokens: 1024, + system: builder.system_prompt, + tools: [builder.score_tool], + tool_choice: { type: "tool", name: "score_application" }, + messages: [{ role: "user", content: build_user_message(builder, p, examples_text) }], + }, + }.to_json + end + puts " Skipped #{skipped} cached requests" if skipped > 0 + + output = File.join(DATA_DIR, "batch-#{model_alias}.jsonl") + File.write(output, lines.join("\n") + "\n") + puts "Wrote #{lines.length} requests to #{output}" + + # Cost estimate + total_chars = lines.sum(&:length) + input_tokens = total_chars / 4 + case model_alias + when "haiku" + cost = (input_tokens / 1_000_000.0 * 0.80 + projects.length * 200 / 1_000_000.0 * 4.0) * 0.5 + when "sonnet" + cost = (input_tokens / 1_000_000.0 * 3.0 + projects.length * 200 / 1_000_000.0 * 15.0) * 0.5 + when "opus" + cost = (input_tokens / 1_000_000.0 * 15.0 + projects.length * 200 / 1_000_000.0 * 75.0) * 0.5 + end + puts "Estimated batch cost (50% discount): ~$#{'%.2f' % cost}" if cost +end + +def cmd_submit(input:) + api_key = ENV["ANTHROPIC_API_KEY"] + raise "ANTHROPIC_API_KEY not set — add to .env.local" unless api_key + + requests = File.readlines(input).map { |l| JSON.parse(l) } + puts "Submitting #{requests.length} requests to Anthropic Batch API..." + + uri = URI("https://api.anthropic.com/v1/messages/batches") + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + + req = Net::HTTP::Post.new(uri) + req["x-api-key"] = api_key + req["anthropic-version"] = "2023-06-01" + req["anthropic-beta"] = "message-batches-2024-09-24" + req["content-type"] = "application/json" + req.body = { requests: requests }.to_json + + resp = http.request(req) + unless resp.is_a?(Net::HTTPSuccess) + warn "Error #{resp.code}: #{resp.body}" + exit 1 + end + + result = JSON.parse(resp.body) + puts "Batch created: #{result['id']}" + puts "Status: #{result['processing_status']}" + + # Save batch ID + id_file = input.sub(".jsonl", "-batch-id.txt") + File.write(id_file, result["id"] + "\n") + puts "Batch ID saved to #{id_file}" + result["id"] +end + +def cmd_results(batch_id:, output: nil) + api_key = ENV["ANTHROPIC_API_KEY"] + raise "ANTHROPIC_API_KEY not set" unless api_key + + headers = { + "x-api-key" => api_key, + "anthropic-version" => "2023-06-01", + "anthropic-beta" => "message-batches-2024-09-24", + } + + # Check status + uri = URI("https://api.anthropic.com/v1/messages/batches/#{batch_id}") + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + req = Net::HTTP::Get.new(uri, headers) + resp = http.request(req) + status = JSON.parse(resp.body) + + puts "Status: #{status['processing_status']}" + counts = status["request_counts"] || {} + puts "Progress: #{counts['succeeded']}/#{counts['processing']} succeeded" + + unless status["processing_status"] == "ended" + puts "Batch still processing. Try again later." + return + end + + # Fetch results + uri = URI("https://api.anthropic.com/v1/messages/batches/#{batch_id}/results") + req = Net::HTTP::Get.new(uri, headers) + resp = http.request(req) + + scores = {} + resp.body.strip.split("\n").each do |line| + item = JSON.parse(line) + project_id = item["custom_id"].sub("project-", "").to_i + begin + # With tool_use, result is in tool call input + tool_call = item.dig("result", "message", "content")&.find { |c| c["type"] == "tool_use" } + if tool_call + parsed = tool_call["input"] + else + # Fallback: try text content + text = item.dig("result", "message", "content", 0, "text") || "" + json_match = text.match(/\{[\s\S]*\}/) + parsed = json_match ? JSON.parse(json_match[0]) : nil + end + scores[project_id] = parsed.merge("project_id" => project_id) if parsed + rescue => e + warn "Failed to parse result for project-#{project_id}: #{e.message}" + end + end + + output ||= File.join(DATA_DIR, "scores-#{batch_id[0..12]}.json") + File.write(output, JSON.pretty_generate(scores)) + puts "\nWrote #{scores.length} scores to #{output}" + + # Summary + score_values = scores.values.map { |s| s["composite_score"] }.compact + avg = score_values.sum / score_values.length.to_f + low = score_values.count { |s| s <= 0.3 } + mid = score_values.count { |s| s > 0.3 && s <= 0.7 } + high = score_values.count { |s| s > 0.7 } + puts "\nScore distribution:" + puts " Low (≤0.3): #{low} apps" + puts " Mid (0.3-0.7): #{mid} apps" + puts " High (>0.7): #{high} apps" + puts " Average: #{'%.3f' % avg}" +end + +# --- CLI --- + +require "shellwords" + +command = ARGV.shift + +# Load .env.local if present +env_file = File.expand_path("../../.env.local", __dir__) +if File.exist?(env_file) + File.readlines(env_file, encoding: "UTF-8").each do |line| + line = line.strip + next if line.empty? || line.start_with?("#") + key, value = line.split("=", 2) + ENV[key] = value if key && value + end +end + +case command +when "build-sample" + i = ARGV.index("--count") + count = i ? ARGV[i + 1].to_i : 400 + cmd_build_sample(count: count) + +when "pre-score" + cmd_pre_score + +when "build-batch" + i = ARGV.index("--model") + model = i ? ARGV[i + 1] : "haiku" + i = ARGV.index("--chapter") + chapter = i ? ARGV[i + 1] : nil + no_fewshot = ARGV.include?("--no-fewshot") + cmd_build_batch(model_alias: model, chapter: chapter, no_fewshot: no_fewshot) + +when "submit" + i = ARGV.index("--input") + input = i ? ARGV[i + 1] : nil + raise "--input required" unless input + cmd_submit(input: input) + +when "results" + i = ARGV.index("--batch-id") + batch_id = i ? ARGV[i + 1] : nil + raise "--batch-id required" unless batch_id + cmd_results(batch_id: batch_id) + +when "submit-all" + %w[haiku sonnet opus].each do |m| + input = File.join(DATA_DIR, "batch-#{m}.jsonl") + next unless File.exist?(input) + puts "\n=== Submitting #{m} ===" + cmd_submit(input: input) + end + +else + puts <<~USAGE + Usage: ruby scripts/signal-score/score_grants.rb + + Commands: + build-sample Build stratified 400-app sample in DuckDB + pre-score Run PreScorer on sample (deterministic features) + build-batch Build batch JSONL [--model haiku|sonnet|opus] [--chapter name] [--no-fewshot] + submit Submit batch [--input path.jsonl] + submit-all Submit all built batches (haiku, sonnet, opus) + results Fetch results [--batch-id ] + USAGE +end diff --git a/spec/extras/signal_scorer_spec.rb b/spec/extras/signal_scorer_spec.rb new file mode 100644 index 00000000..ae58ca0d --- /dev/null +++ b/spec/extras/signal_scorer_spec.rb @@ -0,0 +1,163 @@ +require 'spec_helper' +require 'webmock/rspec' + +describe SignalScorer do + let(:chapter) { FactoryBot.create(:chapter) } + let(:project) { FactoryBot.create(:project, chapter: chapter) } + + let(:api_key) { "sk-ant-test-key" } + let(:scorer) { described_class.new(project, api_key: api_key) } + + let(:score_response) do + { + "content" => [ + { + "type" => "tool_use", + "id" => "tool_123", + "name" => "score_application", + "input" => { + "composite_score" => 0.72, + "reason" => "Strong community project with clear budget.", + "flags" => [], + "features" => { + "credibility" => 0.8, + "reliability" => 0.7, + "intimacy" => 0.9, + "self_interest" => 0.2, + "specificity" => 0.8, + "creativity" => 0.7, + "budget_alignment" => 0.9, + "catalytic_potential" => 0.6, + "community_benefit" => 0.8, + "personal_voice" => 0.7, + "ai_spam_likelihood" => 0.1, + "ai_writing_likelihood" => 0.2, + } + } + } + ] + } + end + + before do + # Create a global config + SignalScoreConfig.create!( + chapter_id: nil, + scoring_config: { "model" => "claude-haiku-4-5-20251001" }, + prompt_overrides: {}, + category_config: {}, + enabled: true + ) + end + + describe '#score!' do + before do + stub_request(:post, "https://api.anthropic.com/v1/messages") + .to_return(status: 200, body: score_response.to_json, headers: { "Content-Type" => "application/json" }) + end + + it 'calls the Anthropic API and persists the score' do + result = scorer.score! + + expect(result["composite_score"]).to eq(0.72) + expect(result["reason"]).to eq("Strong community project with clear budget.") + + project.reload + expect(project.metadata["signal_score"]["composite_score"]).to eq(0.72) + expect(project.metadata["signal_score"]["scored_at"]).to be_present + expect(project.metadata["signal_score"]["model"]).to eq("claude-haiku-4-5-20251001") + expect(project.metadata["signal_score"]["variant"]).to eq("live") + end + + it 'does not clobber existing metadata' do + project.update_column(:metadata, { "other_key" => "preserved" }) + + scorer.score! + project.reload + + expect(project.metadata["other_key"]).to eq("preserved") + expect(project.metadata["signal_score"]["composite_score"]).to eq(0.72) + end + + it 'raises ScoringError without an API key' do + scorer = described_class.new(project, api_key: nil) + expect { scorer.score! }.to raise_error(SignalScorer::ScoringError, /No Anthropic API key/) + end + + it 'raises ScoringError on API failure without leaking response body' do + stub_request(:post, "https://api.anthropic.com/v1/messages") + .to_return(status: 429, body: { "error" => { "type" => "rate_limit_error", "message" => "secret stuff" } }.to_json) + + expect { scorer.score! }.to raise_error(SignalScorer::ScoringError, /429.*rate_limit_error/) + expect { scorer.score! }.to raise_error { |e| expect(e.message).not_to include("secret stuff") } + end + + it 'raises ScoringError when no tool_use block returned' do + stub_request(:post, "https://api.anthropic.com/v1/messages") + .to_return(status: 200, body: { "content" => [{ "type" => "text", "text" => "oops" }] }.to_json) + + expect { scorer.score! }.to raise_error(SignalScorer::ScoringError, /No tool_use block/) + end + end + + describe '#composite_score' do + it 'returns nil before scoring' do + expect(scorer.composite_score).to be_nil + end + end + + describe '#score_label' do + before do + stub_request(:post, "https://api.anthropic.com/v1/messages") + .to_return(status: 200, body: score_response.to_json, headers: { "Content-Type" => "application/json" }) + scorer.score! + end + + it 'returns the correct label' do + expect(scorer.score_label).to eq("Strong") + end + end + + describe '#score_color' do + before do + stub_request(:post, "https://api.anthropic.com/v1/messages") + .to_return(status: 200, body: score_response.to_json, headers: { "Content-Type" => "application/json" }) + scorer.score! + end + + it 'returns green for strong scores' do + expect(scorer.score_color).to eq("green") + end + end + + describe '#format_application' do + it 'includes all present fields' do + project = FactoryBot.create(:project, + title: "Test Project", + about_me: "About me text", + about_project: "About project text", + use_for_money: "$500 for supplies", + url: "https://example.com" + ) + scorer = described_class.new(project, api_key: api_key) + text = scorer.send(:format_application) + + expect(text).to include("Title: Test Project") + expect(text).to include("About Me: About me text") + expect(text).to include("Use for Money: $500 for supplies") + expect(text).to include("URL: https://example.com") + end + + it 'skips blank fields' do + project = FactoryBot.create(:project, title: "Test") + # Set fields to blank after creation to bypass validations + project.update_columns(about_me: nil, url: "") + scorer = described_class.new(project.reload, api_key: api_key) + text = scorer.send(:format_application) + + expect(text).to include("Title: Test") + expect(text).not_to include("About Me:") + expect(text).not_to include("URL:") + end + end +end diff --git a/spec/jobs/score_grant_job_spec.rb b/spec/jobs/score_grant_job_spec.rb new file mode 100644 index 00000000..7c4930d2 --- /dev/null +++ b/spec/jobs/score_grant_job_spec.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'webmock/rspec' + +describe ScoreGrantJob do + let(:chapter) { FactoryBot.create(:chapter) } + let(:project) { FactoryBot.create(:project, chapter: chapter) } + + let(:score_response) do + { + "content" => [ + { + "type" => "tool_use", + "id" => "tool_123", + "name" => "score_application", + "input" => { + "composite_score" => 0.65, + "reason" => "Solid project.", + "flags" => [], + "features" => { + "credibility" => 0.7, "reliability" => 0.6, "intimacy" => 0.7, + "self_interest" => 0.3, "specificity" => 0.6, "creativity" => 0.5, + "budget_alignment" => 0.7, "catalytic_potential" => 0.5, + "community_benefit" => 0.6, "personal_voice" => 0.6, + "ai_spam_likelihood" => 0.1, "ai_writing_likelihood" => 0.2, + } + } + } + ] + } + end + + before do + SignalScoreConfig.create!( + chapter_id: nil, + scoring_config: { "model" => "claude-haiku-4-5-20251001" }, + prompt_overrides: {}, + category_config: {}, + enabled: true + ) + @original_key = ENV["ANTHROPIC_API_KEY"] + ENV["ANTHROPIC_API_KEY"] = "sk-ant-test" + + stub_request(:post, "https://api.anthropic.com/v1/messages") + .to_return(status: 200, body: score_response.to_json, headers: { "Content-Type" => "application/json" }) + end + + after do + if @original_key + ENV["ANTHROPIC_API_KEY"] = @original_key + else + ENV.delete("ANTHROPIC_API_KEY") + end + end + + describe '#perform' do + it 'scores the project' do + described_class.new.perform(project.id) + + project.reload + assert_equal 0.65, project.metadata["signal_score"]["composite_score"] + end + + it 'skips if project already has a score' do + project.update_column(:metadata, { "signal_score" => { "composite_score" => 0.5 } }) + + described_class.new.perform(project.id) + + project.reload + assert_equal 0.5, project.metadata["signal_score"]["composite_score"] + end + + it 'skips if project does not exist' do + expect { described_class.new.perform(999999) }.not_to raise_error + end + + it 'skips and releases claim if scoring is disabled' do + SignalScoreConfig.update_all(enabled: false) + + described_class.new.perform(project.id) + + project.reload + assert_nil project.metadata&.dig("signal_score", "composite_score") + end + + it 'prevents double-scoring via atomic claim' do + project.update_column(:metadata, { "signal_score" => { "status" => "scoring" } }) + + described_class.new.perform(project.id) + + project.reload + assert_equal "scoring", project.metadata["signal_score"]["status"] + assert_nil project.metadata["signal_score"]["composite_score"] + end + + it 'cleans up claim on API failure after retries' do + stub_request(:post, "https://api.anthropic.com/v1/messages") + .to_return(status: 500, body: { "error" => { "type" => "server_error" } }.to_json) + + # Stub sleep to avoid slow tests + ScoreGrantJob.any_instance.stubs(:sleep) + + described_class.new.perform(project.id) + + project.reload + # Claim should be cleaned up after exhausting retries + assert_nil project.metadata&.dig("signal_score") + end + end +end diff --git a/spec/lib/project_filter_spec.rb b/spec/lib/project_filter_spec.rb index 21bf5a35..490af079 100644 --- a/spec/lib/project_filter_spec.rb +++ b/spec/lib/project_filter_spec.rb @@ -43,4 +43,79 @@ project_filter = ProjectFilter.new(Project) expect(project_filter.funded.result).to eq([winner]) end + + describe '#signal_score_above' do + it 'filters projects at or above the threshold' do + high = FactoryBot.create(:project) + high.update_column(:metadata, { "signal_score" => { "composite_score" => 0.8 } }) + low = FactoryBot.create(:project) + low.update_column(:metadata, { "signal_score" => { "composite_score" => 0.3 } }) + unscored = FactoryBot.create(:project) + + result = ProjectFilter.new(Project).signal_score_above(0.5).result + expect(result).to include(high) + expect(result).not_to include(low) + expect(result).not_to include(unscored) + end + end + + describe '#sort_by_signal_score' do + it 'sorts descending by default (highest first)' do + low = FactoryBot.create(:project) + low.update_column(:metadata, { "signal_score" => { "composite_score" => 0.3 } }) + high = FactoryBot.create(:project) + high.update_column(:metadata, { "signal_score" => { "composite_score" => 0.9 } }) + mid = FactoryBot.create(:project) + mid.update_column(:metadata, { "signal_score" => { "composite_score" => 0.6 } }) + + result = ProjectFilter.new(Project).sort_by_signal_score.result + expect(result.map(&:id)).to eq([high, mid, low].map(&:id)) + end + + it 'sorts ascending when specified' do + high = FactoryBot.create(:project) + high.update_column(:metadata, { "signal_score" => { "composite_score" => 0.9 } }) + low = FactoryBot.create(:project) + low.update_column(:metadata, { "signal_score" => { "composite_score" => 0.2 } }) + + result = ProjectFilter.new(Project).sort_by_signal_score(:asc).result + expect(result.map(&:id)).to eq([low, high].map(&:id)) + end + + it 'puts unscored projects last in descending order' do + scored = FactoryBot.create(:project) + scored.update_column(:metadata, { "signal_score" => { "composite_score" => 0.5 } }) + unscored = FactoryBot.create(:project) + + result = ProjectFilter.new(Project).sort_by_signal_score.result.to_a + expect(result.first).to eq(scored) + expect(result.last).to eq(unscored) + end + end + + describe '#sort_by_date' do + it 'sorts by created_at in the given direction' do + Timecop.freeze(Time.now) do + old = FactoryBot.create(:project, created_at: 3.days.ago) + new_proj = FactoryBot.create(:project, created_at: 1.hour.ago) + + asc_result = ProjectFilter.new(Project).sort_by_date(:asc).result + expect(asc_result.map(&:id)).to eq([old, new_proj].map(&:id)) + + desc_result = ProjectFilter.new(Project).sort_by_date(:desc).result + expect(desc_result.map(&:id)).to eq([new_proj, old].map(&:id)) + end + end + end + + describe '#sort_by_random' do + it 'returns a stable ordering based on project id hash' do + 5.times { FactoryBot.create(:project) } + + first_run = ProjectFilter.new(Project).sort_by_random.result.map(&:id) + second_run = ProjectFilter.new(Project).sort_by_random.result.map(&:id) + + expect(first_run).to eq(second_run) + end + end end diff --git a/spec/lib/signal_score/pre_scorer_spec.rb b/spec/lib/signal_score/pre_scorer_spec.rb new file mode 100644 index 00000000..eb25d129 --- /dev/null +++ b/spec/lib/signal_score/pre_scorer_spec.rb @@ -0,0 +1,194 @@ +# frozen_string_literal: true + +require_relative "../../../scripts/signal-score/pre_scorer" + +RSpec.describe PreScorer do + describe "#analyze!" do + context "with a typical funded application" do + let(:project) do + { + "about_me" => "I'm a community organizer in Pilsen, Chicago. I've been running after-school programs for 5 years at Benito Juarez High School.", + "about_project" => "We want to build a community garden on the vacant lot at 18th and Ashland. We've already gotten permission from the alderman's office and have 30 volunteers signed up.", + "use_for_money" => "Seeds and soil: $400. Raised beds (lumber): $350. Tools: $150. Water hookup: $100.", + } + end + + subject { described_class.new(project).tap(&:analyze!).features } + + it "counts words per field" do + expect(subject["word_count_about_me"]).to be > 10 + expect(subject["word_count_about_project"]).to be > 10 + expect(subject["word_count_use_for_money"]).to be > 5 + end + + it "detects money mentions" do + expect(subject["money_mention_count"]).to eq(4) # $400, $350, $150, $100 + end + + it "finds URLs" do + expect(subject["url_count"]).to eq(0) + end + + it "detects no empty fields" do + expect(subject["empty_field_count"]).to eq(0) + end + + it "calculates field length variance" do + expect(subject["field_length_variance"]).to be > 0 + end + + it "has reasonable sentence stats" do + expect(subject["sentence_count"]).to be > 3 + expect(subject["avg_sentence_length"]).to be > 5 + end + end + + context "with a spam/low-effort application" do + let(:project) do + { + "about_me" => "me", + "about_project" => "need money", + "use_for_money" => "stuff", + } + end + + subject { described_class.new(project).tap(&:analyze!).features } + + it "has very low word counts" do + expect(subject["word_count_total"]).to be < 10 + end + + it "has zero money mentions" do + expect(subject["money_mention_count"]).to eq(0) + end + + it "has low sentence count" do + expect(subject["sentence_count"]).to be <= 3 + end + end + + context "with international currencies" do + let(:project) do + { + "about_me" => "Artist from Melbourne.", + "about_project" => "Budget: A$800 for paint, 200 euros for brushes, 500 pounds for venue hire, NZ$300 for shipping.", + "use_for_money" => "Materials and transport costs.", + } + end + + subject { described_class.new(project).tap(&:analyze!).features } + + it "detects multiple currency formats" do + expect(subject["money_mention_count"]).to be >= 4 + end + end + + context "with empty fields" do + let(:project) do + { + "about_me" => "I'm a student.", + "about_project" => "", + "use_for_money" => nil, + } + end + + subject { described_class.new(project).tap(&:analyze!).features } + + it "counts empty fields" do + expect(subject["empty_field_count"]).to eq(2) + end + end + + context "with TF-IDF features" do + let(:project) do + { + "about_me" => "community garden volunteer", + "about_project" => "build raised beds for neighborhood", + "use_for_money" => "soil and seeds", + } + end + + before do + # Load a minimal IDF table + PreScorer.instance_variable_set(:@idf_table, { + "community" => 2.5, + "garden" => 4.0, + "volunteer" => 3.5, + "build" => 2.0, + "raised" => 5.0, + "beds" => 5.5, + "neighborhood" => 3.8, + "soil" => 6.0, + "seeds" => 5.5, + }) + end + + after do + PreScorer.instance_variable_set(:@idf_table, nil) + end + + subject { described_class.new(project).tap(&:analyze!).features } + + it "computes TF-IDF mean" do + expect(subject["tfidf_mean"]).to be > 0 + end + + it "computes TF-IDF max" do + expect(subject["tfidf_max"]).to be >= subject["tfidf_mean"] + end + + it "counts unique matching terms" do + expect(subject["tfidf_unique_terms"]).to be > 0 + end + end + end + + describe "seeded data", :signal_score_data do + # These tests run against real data from .scratch/data/ + # Enable with: SIGNAL_SCORE_DATA=1 rspec --tag signal_score_data + + let(:duckdb_path) { File.expand_path("../../../.scratch/data/awesomebits.duckdb", __dir__) } + + before(:all) do + skip "DuckDB not found at .scratch/data/" unless File.exist?( + File.expand_path("../../../.scratch/data/awesomebits.duckdb", __dir__) + ) + require "json" + # Load projects via duckdb CLI + raw = `duckdb #{File.expand_path("../../../.scratch/data/awesomebits.duckdb", __dir__)} -json -c "SELECT * FROM sample_400 LIMIT 50"` + @projects = JSON.parse(raw) + end + + it "processes real funded projects without errors" do + funded = @projects.select { |p| p["actual_label"] == "funded" } + skip "No funded projects in seed data" if funded.empty? + + funded.each do |p| + scorer = PreScorer.new(p) + expect { scorer.analyze! }.not_to raise_error + expect(scorer.features["word_count_total"]).to be >= 0 + end + end + + it "processes real hidden projects without errors" do + hidden = @projects.select { |p| p["actual_label"] == "hidden" } + skip "No hidden projects in seed data" if hidden.empty? + + hidden.each do |p| + scorer = PreScorer.new(p) + expect { scorer.analyze! }.not_to raise_error + end + end + + it "funded projects tend to have higher word counts than hidden" do + funded = @projects.select { |p| p["actual_label"] == "funded" } + hidden = @projects.select { |p| p["actual_label"] == "hidden" } + skip "Not enough labeled data" if funded.length < 5 || hidden.length < 5 + + funded_avg = funded.sum { |p| PreScorer.new(p).tap(&:analyze!).features["word_count_total"] } / funded.length.to_f + hidden_avg = hidden.sum { |p| PreScorer.new(p).tap(&:analyze!).features["word_count_total"] } / hidden.length.to_f + + expect(funded_avg).to be > hidden_avg + end + end +end diff --git a/spec/models/signal_score_config_spec.rb b/spec/models/signal_score_config_spec.rb new file mode 100644 index 00000000..e30489bd --- /dev/null +++ b/spec/models/signal_score_config_spec.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe SignalScoreConfig do + it { is_expected.to belong_to(:chapter).class_name("Chapter") } + + describe "validations" do + it "allows one global default (chapter_id = nil)" do + config = described_class.create!(scoring_config: { "model" => "haiku" }) + expect(config).to be_persisted + expect(config.chapter_id).to be_nil + end + + it "prevents duplicate global defaults" do + described_class.create!(scoring_config: { "model" => "haiku" }) + duplicate = described_class.new(scoring_config: { "model" => "sonnet" }) + expect(duplicate).not_to be_valid + expect(duplicate.errors[:chapter_id]).to include("global default already exists") + end + + it "allows multiple chapter-specific configs" do + c1 = FactoryBot.create(:chapter) + c2 = FactoryBot.create(:chapter) + config1 = described_class.create!(chapter: c1, scoring_config: {}) + config2 = described_class.create!(chapter: c2, scoring_config: {}) + expect(config1).to be_persisted + expect(config2).to be_persisted + end + + it "prevents duplicate chapter configs" do + chapter = FactoryBot.create(:chapter) + described_class.create!(chapter: chapter, scoring_config: {}) + duplicate = described_class.new(chapter: chapter, scoring_config: {}) + expect(duplicate).not_to be_valid + end + end + + describe ".resolve" do + let!(:global) do + described_class.create!( + scoring_config: { "model" => "haiku", "grant_amount" => 1000 }, + prompt_overrides: { "rubric_addendum" => "Be strict." }, + category_config: { "disabled" => [] }, + enabled: true, + ) + end + + it "returns global config when no chapter specified" do + result = described_class.resolve + expect(result["scoring_config"]["model"]).to eq("haiku") + expect(result["scoring_config"]["grant_amount"]).to eq(1000) + end + + it "deep-merges chapter overrides onto global" do + chapter = FactoryBot.create(:chapter) + described_class.create!( + chapter: chapter, + scoring_config: { "model" => "sonnet", "temperature" => 0.5 }, + prompt_overrides: {}, + category_config: {}, + enabled: true, + ) + + result = described_class.resolve(chapter) + # Chapter override wins + expect(result["scoring_config"]["model"]).to eq("sonnet") + expect(result["scoring_config"]["temperature"]).to eq(0.5) + # Global default preserved where chapter doesn't override + expect(result["scoring_config"]["grant_amount"]).to eq(1000) + # Global prompt preserved (chapter has empty overrides) + expect(result["prompt_overrides"]["rubric_addendum"]).to eq("Be strict.") + end + + it "returns empty hash when no global default exists" do + global.destroy! + expect(described_class.resolve).to eq({}) + end + end +end