From aa09569c67e587c0ee0798d2c3710ce44f00f245 Mon Sep 17 00:00:00 2001 From: Ian Baker Date: Tue, 12 Aug 2025 19:45:48 -0700 Subject: [PATCH 1/5] Add behavior service, endpoint, firestore config, basic browser events, time-to-first-interaction --- firestore/firestore.rules | 20 +- .../participant_view/cohort_landing.ts | 22 ++ frontend/src/service_provider.ts | 4 + frontend/src/services/behavior.service.ts | 255 ++++++++++++++++++ frontend/src/services/participant.service.ts | 15 ++ frontend/src/shared/callables.ts | 28 ++ functions/src/behavior.endpoints.ts | 61 +++++ functions/src/index.ts | 1 + utils/src/behavior.ts | 20 ++ utils/src/index.ts | 3 + utils/src/shared.validation.ts | 15 ++ 11 files changed, 437 insertions(+), 7 deletions(-) create mode 100644 frontend/src/services/behavior.service.ts create mode 100644 functions/src/behavior.endpoints.ts create mode 100644 utils/src/behavior.ts diff --git a/firestore/firestore.rules b/firestore/firestore.rules index e3329c495..63050c41e 100644 --- a/firestore/firestore.rules +++ b/firestore/firestore.rules @@ -113,7 +113,7 @@ service cloud.firestore { match /agents/{agentId} { allow get: if isExperimenter(); allow list: if isExperimenter(); - // Experimenters can use cloud function endpoints + // Experimenters can use cloud function endpoints allow write: if false; match /chatPrompts/{promptId} { @@ -131,13 +131,13 @@ service cloud.firestore { allow get: if true; // Public read allow list: if canEditExperiment(experimentId); - // Experimenters can use cloud function endpoints + // Experimenters can use cloud function endpoints allow write: if false; match /publicStageData/{stageId} { allow read: if true; - // TODO: Triggered by cloud function triggers + // TODO: Triggered by cloud function triggers allow write: if false; match /chats/{chatId} { @@ -169,7 +169,7 @@ service cloud.firestore { allow get: if true; allow list: if true; - // Use cloud function endpoints + // Use cloud function endpoints allow update: if false; } @@ -180,13 +180,13 @@ service cloud.firestore { allow list: if true; allow get: if true; // Public read - // Participants can use cloud function endpoints + // Participants can use cloud function endpoints allow update: if false; match /stageData/{stageId} { allow read: if true; - // Participants can use cloud function endpoints + // Participants can use cloud function endpoints allow write: if false; } @@ -194,13 +194,19 @@ service cloud.firestore { allow read: if true; allow write: if false; // Use cloud functions } + + // Append-only participant behavior events + match /behavior/{eventId} { + allow read: if isExperimenter(); + allow write: if false; // Use cloud functions + } } match /participantPublicData/{participantPublicId} { allow list: if true; allow get: if true; - // Triggered by cloud function triggers + // Triggered by cloud function triggers allow write: if false; } } diff --git a/frontend/src/components/participant_view/cohort_landing.ts b/frontend/src/components/participant_view/cohort_landing.ts index c3c3f0e62..90c98f232 100644 --- a/frontend/src/components/participant_view/cohort_landing.ts +++ b/frontend/src/components/participant_view/cohort_landing.ts @@ -9,6 +9,7 @@ import {AnalyticsService, ButtonClick} from '../../services/analytics.service'; import {ExperimentService} from '../../services/experiment.service'; import {FirebaseService} from '../../services/firebase.service'; import {Pages, RouterService} from '../../services/router.service'; +import {BehaviorService} from '../../services/behavior.service'; import {StageKind} from '@deliberation-lab/utils'; @@ -26,8 +27,24 @@ export class CohortLanding extends MobxLitElement { private readonly experimentService = core.getService(ExperimentService); private readonly firebaseService = core.getService(FirebaseService); private readonly routerService = core.getService(RouterService); + private readonly behaviorService = core.getService(BehaviorService); @state() isLoading = false; + private hasLoggedCtaRender = false; + + override updated() { + // Log first time the join CTA is actually available to the user + const params = this.routerService.activeRoute.params; + const exp = this.experimentService.experiment; + const isLocked = !!(exp && exp.cohortLockMap[params['cohort']]); + if (!isLocked && !this.hasLoggedCtaRender) { + this.behaviorService.log('cta_render', { + cta: 'join_experiment', + page: 'cohort_landing', + }); + this.hasLoggedCtaRender = true; + } + } override render() { const isLockedCohort = () => { @@ -67,6 +84,11 @@ export class CohortLanding extends MobxLitElement { } private async joinExperiment() { + // Log CTA click for time-to-first-interaction + this.behaviorService.log('cta_click', { + cta: 'join_experiment', + page: 'cohort_landing', + }); this.isLoading = true; this.analyticsService.trackButtonClick(ButtonClick.PARTICIPANT_JOIN); diff --git a/frontend/src/service_provider.ts b/frontend/src/service_provider.ts index fcad2cd1f..13295c593 100644 --- a/frontend/src/service_provider.ts +++ b/frontend/src/service_provider.ts @@ -16,6 +16,7 @@ import {SettingsService} from './services/settings.service'; import {ExperimentEditor} from './services/experiment.editor'; import {ExperimentManager} from './services/experiment.manager'; import {PresenceService} from './services/presence.service'; +import {BehaviorService} from './services/behavior.service'; /** * Defines a map of services to their identifier @@ -67,6 +68,9 @@ export function makeServiceProvider(self: Core) { get presenceService() { return self.getService(PresenceService); }, + get behaviorService() { + return self.getService(BehaviorService); + }, // Editors get experimentEditor() { return self.getService(ExperimentEditor); diff --git a/frontend/src/services/behavior.service.ts b/frontend/src/services/behavior.service.ts new file mode 100644 index 000000000..32077aff5 --- /dev/null +++ b/frontend/src/services/behavior.service.ts @@ -0,0 +1,255 @@ +import {makeObservable} from 'mobx'; +import {FirebaseService} from './firebase.service'; +import {ParticipantService} from './participant.service'; +import {addBehaviorEventsCallable} from '../shared/callables'; +import {Service} from './service'; + +interface ServiceProvider { + firebaseService: FirebaseService; + participantService: ParticipantService; +} + +/** Collects client interaction events and sends in small batches. */ +export class BehaviorService extends Service { + constructor(private readonly sp: ServiceProvider) { + super(); + makeObservable(this); + } + + private experimentId: string | null = null; + private participantPrivateId: string | null = null; + private buffer: { + eventType: string; + relativeTimestamp: number; + stageId: string; + metadata: Record; + }[] = []; + private maxBufferSize = 50; + private flushIntervalMs = 5000; + private intervalHandle: number | null = null; + private listenersAttached = false; + + start(experimentId: string, participantPrivateId: string) { + // Ignore if already tracking same participant + if ( + this.experimentId === experimentId && + this.participantPrivateId === participantPrivateId && + this.listenersAttached + ) { + return; + } + this.stop(); + this.experimentId = experimentId; + this.participantPrivateId = participantPrivateId; + this.attachListeners(); + // Periodic flush + this.intervalHandle = window.setInterval( + () => this.flush(), + this.flushIntervalMs, + ); + // Flush when page hidden/unloaded + document.addEventListener('visibilitychange', this.onVisibilityChange, { + capture: true, + }); + window.addEventListener('beforeunload', this.onBeforeUnload, { + capture: true, + }); + } + + stop() { + if (this.listenersAttached) { + document.removeEventListener('click', this.onClick, true); + document.removeEventListener('keydown', this.onKeyDown, true); + document.removeEventListener('copy', this.onCopy, true); + document.removeEventListener('paste', this.onPaste, true); + window.removeEventListener('blur', this.onWindowBlur, true); + window.removeEventListener('focus', this.onWindowFocus, true); + this.listenersAttached = false; + } + document.removeEventListener( + 'visibilitychange', + this.onVisibilityChange, + true, + ); + window.removeEventListener('beforeunload', this.onBeforeUnload, true); + if (this.intervalHandle) { + window.clearInterval(this.intervalHandle); + this.intervalHandle = null; + } + // Do not flush after clearing identifiers + void this.flush(); + this.experimentId = null; + this.participantPrivateId = null; + this.buffer = []; + } + + /** Manually log a custom behavior event. */ + log(eventType: string, metadata: Record = {}) { + const stageId = + this.sp.participantService.currentStageViewId ?? + this.sp.participantService.profile?.currentStageId ?? + 'unknown'; + const evt = { + eventType, + relativeTimestamp: performance.now(), + stageId, + metadata, + }; + this.buffer.push(evt); + if (this.buffer.length >= this.maxBufferSize) { + void this.flush(); + } + } + + // Internal + private attachListeners() { + if (this.listenersAttached) return; + document.addEventListener('click', this.onClick, true); + document.addEventListener('keydown', this.onKeyDown, true); + document.addEventListener('copy', this.onCopy, true); + document.addEventListener('paste', this.onPaste, true); + window.addEventListener('blur', this.onWindowBlur, true); + window.addEventListener('focus', this.onWindowFocus, true); + this.listenersAttached = true; + } + + private onClick = (e: MouseEvent) => { + // Keep minimal metadata; avoid PII + const target = e.target as HTMLElement | null; + const tag = target?.tagName || ''; + const id = target?.id || ''; + const cls = target?.className?.toString?.().slice(0, 200) || ''; + this.log('click', { + x: e.clientX, + y: e.clientY, + button: e.button, + isTrusted: e.isTrusted, + targetTag: tag, + targetId: id, + targetClass: cls, + }); + }; + + private onKeyDown = (e: KeyboardEvent) => { + // Do not capture actual key values; only log whether it was backspace/delete + const keyName = (e.key || '').toLowerCase(); + const isBackspace = keyName === 'backspace'; + const isDelete = keyName === 'delete' || keyName === 'del'; + this.log('keydown', { + isBackspace, + isDelete, + ctrlKey: e.ctrlKey, + metaKey: e.metaKey, + altKey: e.altKey, + shiftKey: e.shiftKey, + repeat: e.repeat, + isTrusted: e.isTrusted, + targetTag: (e.target as HTMLElement | null)?.tagName || '', + }); + }; + + private onCopy = (e: ClipboardEvent) => { + // Try to capture the actual copied text safely + let text = ''; + const active = document.activeElement as HTMLElement | null; + // If copying from an input or textarea, use selection range + if ( + active && + (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA') + ) { + const el = active as HTMLInputElement | HTMLTextAreaElement; + if ( + typeof el.selectionStart === 'number' && + typeof el.selectionEnd === 'number' + ) { + text = el.value.substring(el.selectionStart, el.selectionEnd); + } + } + // Otherwise, use the document selection + if (!text) { + text = document.getSelection()?.toString() ?? ''; + } + + const MAX_COPY_LEN = 10000; + const truncated = text.length > MAX_COPY_LEN; + const storedText = truncated ? text.slice(0, MAX_COPY_LEN) : text; + + this.log('copy', { + text: storedText, + length: text.length, + truncated, + isTrusted: e.isTrusted, + targetTag: active?.tagName || '', + contentEditable: !!active?.isContentEditable, + }); + }; + + private onPaste = (e: ClipboardEvent) => { + // Read pasted text from clipboard data (covers keyboard, menu, context menu) + let text = ''; + if (e.clipboardData) { + text = e.clipboardData.getData('text/plain') || ''; + } + + // Cap length to avoid huge payloads + const MAX_PASTE_LEN = 10000; + const truncated = text.length > MAX_PASTE_LEN; + const storedText = truncated ? text.slice(0, MAX_PASTE_LEN) : text; + + const active = document.activeElement as HTMLElement | null; + this.log('paste', { + text: storedText, + length: text.length, + truncated, + isTrusted: e.isTrusted, + targetTag: active?.tagName || '', + contentEditable: !!active?.isContentEditable, + }); + }; + + private onVisibilityChange = () => { + if (document.visibilityState === 'hidden') { + void this.flush(); + } + }; + + private onBeforeUnload = () => { + void this.flush(); + }; + + // Public flush for callers that need to ensure the buffer is persisted now + public async flush() { + if (!this.experimentId || !this.participantPrivateId) return; + if (this.buffer.length === 0) return; + const events = this.buffer.slice(); + this.buffer = []; + const payload = { + experimentId: this.experimentId, + participantId: this.participantPrivateId, + events, + }; + try { + await addBehaviorEventsCallable( + this.sp.firebaseService.functions, + payload, + ); + } catch (_) { + // re-queue on failure (cap size to avoid growth) + this.buffer = events.concat(this.buffer).slice(-200); + } + } + + private onWindowBlur = () => { + this.log('window_blur', { + visibility: document.visibilityState, + hasFocus: document.hasFocus(), + }); + }; + + private onWindowFocus = () => { + this.log('window_focus', { + visibility: document.visibilityState, + hasFocus: document.hasFocus(), + }); + }; +} diff --git a/frontend/src/services/participant.service.ts b/frontend/src/services/participant.service.ts index 9d45fdb56..42664930c 100644 --- a/frontend/src/services/participant.service.ts +++ b/frontend/src/services/participant.service.ts @@ -66,12 +66,14 @@ import { isParticipantEndedExperiment, } from '../shared/participant.utils'; import {ElectionStrategy} from '@deliberation-lab/utils'; +import {BehaviorService} from './behavior.service'; interface ServiceProvider { cohortService: CohortService; experimentService: ExperimentService; firebaseService: FirebaseService; participantAnswerService: ParticipantAnswerService; + behaviorService: BehaviorService; } export class ParticipantService extends Service { @@ -112,6 +114,12 @@ export class ParticipantService extends Service { this.participantId = participantId; this.isLoading = true; this.loadParticipantData(); + // Start behavior logging when both IDs are known + if (experimentId && participantId) { + this.sp.behaviorService.start(experimentId, participantId); + } else { + this.sp.behaviorService.stop(); + } } // True if currently in the experiment (not dropped out, not transfer pending) @@ -302,6 +310,7 @@ export class ParticipantService extends Service { this.participantId = null; this.unsubscribeAll(); this.sp.cohortService.reset(); + this.sp.behaviorService.stop(); } // *********************************************************************** // @@ -331,6 +340,8 @@ export class ParticipantService extends Service { } async routeToEndExperiment(currentStatus: ParticipantStatus) { + await this.sp.behaviorService.flush(); + const config = this.sp.experimentService.experiment?.prolificConfig; // Redirect to Prolific if prolific integration is set up @@ -374,6 +385,8 @@ export class ParticipantService extends Service { return; } + await this.sp.behaviorService.flush(); + const result = await updateParticipantToNextStageCallable( this.sp.firebaseService.functions, { @@ -477,6 +490,8 @@ export class ParticipantService extends Service { participantId: this.profile.privateId, }, ); + // Ensure behavior logging is active after start + this.sp.behaviorService.start(this.experimentId, this.profile.privateId); } /** Accept attention check. */ diff --git a/frontend/src/shared/callables.ts b/frontend/src/shared/callables.ts index 44c175bf0..b62219a06 100644 --- a/frontend/src/shared/callables.ts +++ b/frontend/src/shared/callables.ts @@ -511,3 +511,31 @@ export const ackAlertMessageCallable = async ( )(config); return data; }; + +/** Participant behavior logging (batched). */ +type BehaviorEventInput = { + eventType: string; + relativeTimestamp: number; + stageId: string; + metadata: Record; +}; + +type AddBehaviorEventsData = { + experimentId: string; + participantId: string; // private id + events: BehaviorEventInput[]; +}; + +export const addBehaviorEventsCallable = async ( + functions: Functions, + config: AddBehaviorEventsData, +) => { + const {data} = await httpsCallable< + AddBehaviorEventsData, + {success: boolean; count: number} + >( + functions, + 'addBehaviorEvents', + )(config); + return data; +}; diff --git a/functions/src/behavior.endpoints.ts b/functions/src/behavior.endpoints.ts new file mode 100644 index 000000000..f0ae86adc --- /dev/null +++ b/functions/src/behavior.endpoints.ts @@ -0,0 +1,61 @@ +import {onCall} from 'firebase-functions/v2/https'; +import * as functions from 'firebase-functions'; +import {app} from './app'; +import {Type} from '@sinclair/typebox'; +import {Value} from '@sinclair/typebox/value'; + +// Local schema to avoid depending on utils build output +const BehaviorEventInputSchema = Type.Object({ + eventType: Type.String({minLength: 1}), + relativeTimestamp: Type.Number(), + stageId: Type.String({minLength: 1}), + metadata: Type.Record(Type.String(), Type.Any()), +}); + +const AddBehaviorEventsDataSchema = Type.Object({ + experimentId: Type.String({minLength: 1}), + participantId: Type.String({minLength: 1}), + events: Type.Array(BehaviorEventInputSchema, {minItems: 1, maxItems: 200}), +}); + +/** + * addBehaviorEvents: batched append-only event log under participant. + * Path: experiments/{experimentId}/participants/{participantPrivateId}/behavior/{autoId} + */ +export const addBehaviorEvents = onCall(async (request) => { + const {data} = request as { + data: {experimentId: string; participantId: string; events: unknown[]}; + }; + + if (!Value.Check(AddBehaviorEventsDataSchema, data)) { + throw new functions.https.HttpsError('invalid-argument', 'Invalid data'); + } + + const batch = app.firestore().batch(); + const base = app + .firestore() + .collection('experiments') + .doc(data.experimentId) + .collection('participants') + .doc(data.participantId) + .collection('behavior'); + + for (const evt of data.events as Array<{ + eventType: string; + relativeTimestamp: number; + stageId: string; + metadata?: Record; + }>) { + const ref = base.doc(); + batch.set(ref, { + type: evt.eventType, + relativeTimestamp: evt.relativeTimestamp, + stageId: evt.stageId, + metadata: evt.metadata ?? {}, + timestamp: new Date(), // server time + }); + } + + await batch.commit(); + return {success: true, count: data.events.length}; +}); diff --git a/functions/src/index.ts b/functions/src/index.ts index 521664f69..99077b402 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -22,6 +22,7 @@ export * from './agent.utils'; export * from './log.utils'; export * from './mediator.endpoints'; +export * from './behavior.endpoints'; export * from './stages/chat.endpoints'; export * from './stages/chat.triggers'; diff --git a/utils/src/behavior.ts b/utils/src/behavior.ts new file mode 100644 index 000000000..8313b7415 --- /dev/null +++ b/utils/src/behavior.ts @@ -0,0 +1,20 @@ +/** Behavioral event types for participant interaction logging. */ + +export interface BehaviorEventInput { + /** Event type identifier, e.g., 'click', 'keydown'. */ + eventType: string; + /** High-resolution relative timestamp in milliseconds (e.g., performance.now()). */ + relativeTimestamp: number; + /** Current stage ID when event was captured. */ + stageId: string; + /** Arbitrary metadata map; contents depend on eventType. */ + metadata: Record; +} + +export interface AddBehaviorEventsData { + experimentId: string; + /** Participant private ID. */ + participantId: string; + /** Events to add in a single batch. */ + events: BehaviorEventInput[]; +} diff --git a/utils/src/index.ts b/utils/src/index.ts index 366b313c8..b161c67e0 100644 --- a/utils/src/index.ts +++ b/utils/src/index.ts @@ -80,3 +80,6 @@ export * from './utils/cache.utils'; export * from './utils/object.utils'; export * from './utils/random.utils'; export * from './utils/string.utils'; + +// Behavior +export * from './behavior'; diff --git a/utils/src/shared.validation.ts b/utils/src/shared.validation.ts index 3c4a2f9fe..73f1d1c43 100644 --- a/utils/src/shared.validation.ts +++ b/utils/src/shared.validation.ts @@ -1,5 +1,6 @@ import {Type} from '@sinclair/typebox'; import {Visibility} from './shared'; +import type {BehaviorEventInput} from './behavior'; /** UnifiedTimestamp input validation. */ export const UnifiedTimestampSchema = Type.Object({ @@ -27,3 +28,17 @@ export const PermissionsConfigSchema = Type.Object({ ]), readers: Type.Array(Type.String()), }); + +// Behavior event validation +export const BehaviorEventInputSchema = Type.Object({ + eventType: Type.String({minLength: 1}), + relativeTimestamp: Type.Number(), + stageId: Type.String({minLength: 1}), + metadata: Type.Record(Type.String(), Type.Any()), +}); + +export const AddBehaviorEventsDataSchema = Type.Object({ + experimentId: Type.String({minLength: 1}), + participantId: Type.String({minLength: 1}), + events: Type.Array(BehaviorEventInputSchema, {minItems: 1, maxItems: 200}), +}); From e01817497d4a837c948885cef2a777309c549771 Mon Sep 17 00:00:00 2001 From: Ian Baker Date: Wed, 13 Aug 2025 15:36:17 -0700 Subject: [PATCH 2/5] Add input event listener for survey textarea and chat, for differentiating speech-to-text, etc --- eslint.config.mjs | 87 +++++----- .../src/components/stages/chat_interface.ts | 28 ++++ frontend/src/components/stages/survey_view.ts | 47 +++++- frontend/src/services/behavior.service.ts | 158 +++++++++++++++++- 4 files changed, 267 insertions(+), 53 deletions(-) diff --git a/eslint.config.mjs b/eslint.config.mjs index 40fbf1b2c..36682b1fd 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,56 +1,65 @@ -import typescriptEslint from "@typescript-eslint/eslint-plugin"; -import globals from "globals"; -import tsParser from "@typescript-eslint/parser"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import js from "@eslint/js"; -import { FlatCompat } from "@eslint/eslintrc"; +import typescriptEslint from '@typescript-eslint/eslint-plugin'; +import globals from 'globals'; +import tsParser from '@typescript-eslint/parser'; +import path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import js from '@eslint/js'; +import {FlatCompat} from '@eslint/eslintrc'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const compat = new FlatCompat({ - baseDirectory: __dirname, - recommendedConfig: js.configs.recommended, - allConfig: js.configs.all + baseDirectory: __dirname, + recommendedConfig: js.configs.recommended, + allConfig: js.configs.all, }); -export default [{ +export default [ + { ignores: [ - "**/dist/", - "**/node_modules/", - "**/webpack.config.js", - "**/lit-css-loader.js", + '**/dist/', + '**/node_modules/', + '**/webpack.config.js', + '**/lit-css-loader.js', ], -}, ...compat.extends( - "eslint:recommended", - "plugin:@typescript-eslint/eslint-recommended", - "plugin:@typescript-eslint/recommended", -), { + }, + ...compat.extends( + 'eslint:recommended', + 'plugin:@typescript-eslint/eslint-recommended', + 'plugin:@typescript-eslint/recommended', + ), + { plugins: { - "@typescript-eslint": typescriptEslint, + '@typescript-eslint': typescriptEslint, }, languageOptions: { - globals: { - ...globals.browser, - }, + globals: { + ...globals.browser, + }, - parser: tsParser, - ecmaVersion: 2020, - sourceType: "module", + parser: tsParser, + ecmaVersion: 2020, + sourceType: 'module', }, rules: { - "no-case-declarations": "off", - "no-prototype-builtins": "off", - "@typescript-eslint/ban-types": "off", - "@typescript-eslint/explicit-function-return-type": "off", - "@typescript-eslint/explicit-module-boundary-types": "off", - "@typescript-eslint/no-explicit-any": "error", - "@typescript-eslint/no-empty-function": "off", - "@typescript-eslint/no-non-null-assertion": "off", - "@typescript-eslint/no-unused-vars": ["warn", { - argsIgnorePattern: "^_", - }], + 'no-case-declarations': 'off', + 'no-prototype-builtins': 'off', + '@typescript-eslint/ban-types': 'off', + '@typescript-eslint/explicit-function-return-type': 'off', + '@typescript-eslint/explicit-module-boundary-types': 'off', + '@typescript-eslint/no-explicit-any': 'error', + '@typescript-eslint/no-empty-function': 'off', + '@typescript-eslint/no-non-null-assertion': 'off', + '@typescript-eslint/no-unused-vars': [ + 'warn', + { + argsIgnorePattern: '^_', + varsIgnorePattern: '^_', + caughtErrorsIgnorePattern: '^_', + }, + ], }, -}]; \ No newline at end of file + }, +]; diff --git a/frontend/src/components/stages/chat_interface.ts b/frontend/src/components/stages/chat_interface.ts index c9e8abd1a..484fe91c9 100644 --- a/frontend/src/components/stages/chat_interface.ts +++ b/frontend/src/components/stages/chat_interface.ts @@ -9,6 +9,7 @@ import './chat_message'; import {MobxLitElement} from '@adobe/lit-mobx'; import {CSSResultGroup, html, nothing} from 'lit'; +import {ref} from 'lit/directives/ref.js'; import {customElement, property, state} from 'lit/decorators.js'; import {core} from '../../core/core'; @@ -17,6 +18,7 @@ import {CohortService} from '../../services/cohort.service'; import {ExperimentService} from '../../services/experiment.service'; import {ParticipantService} from '../../services/participant.service'; import {ParticipantAnswerService} from '../../services/participant.answer'; +import {BehaviorService} from '../../services/behavior.service'; import {RouterService} from '../../services/router.service'; import { @@ -41,6 +43,7 @@ export class ChatInterface extends MobxLitElement { private readonly participantAnswerService = core.getService( ParticipantAnswerService, ); + private readonly behaviorService = core.getService(BehaviorService); private readonly routerService = core.getService(RouterService); @property() stage: ChatStageConfig | undefined = undefined; @@ -49,6 +52,30 @@ export class ChatInterface extends MobxLitElement { @state() readyToEndDiscussionLoading = false; @state() isAlertLoading = false; + private _chatInputUnsub: (() => void) | null = null; + + private onChatRef = (el: Element | undefined) => { + // Detach previous listener if any + if (this._chatInputUnsub) { + this._chatInputUnsub(); + this._chatInputUnsub = null; + } + if (!el) return; + const stageId = this.stage?.id ?? 'unknown'; + this._chatInputUnsub = this.behaviorService.attachTextInput( + el, + `chat_stage:${stageId}`, + ); + }; + + override disconnectedCallback(): void { + if (this._chatInputUnsub) { + this._chatInputUnsub(); + this._chatInputUnsub = null; + } + super.disconnectedCallback(); + } + private sendUserInput() { if (!this.stage) return; @@ -210,6 +237,7 @@ export class ChatInterface extends MobxLitElement { this.isConversationOver()} @keyup=${handleKeyUp} @input=${handleInput} + ${ref(this.onChatRef)} > void> = new Map(); + private _refHandlers: Map void> = + new Map(); + + private getTextareaRef(questionId: string) { + let handler = this._refHandlers.get(questionId); + if (handler) return handler; + handler = (el: Element | undefined) => { + // Detach any previous subscription for this question + const prev = this._inputUnsubs.get(questionId); + if (prev) { + prev(); + this._inputUnsubs.delete(questionId); + } + if (!el) return; + const unsub = this.behaviorService.attachTextInput(el, questionId); + this._inputUnsubs.set(questionId, unsub); + }; + this._refHandlers.set(questionId, handler); + return handler; + } + + override disconnectedCallback(): void { + // Clean up all per-question input listeners + for (const unsub of this._inputUnsubs.values()) { + unsub(); + } + this._inputUnsubs.clear(); + this._refHandlers.clear(); + super.disconnectedCallback(); + } + override render() { if (!this.stage) { return nothing; @@ -145,7 +179,7 @@ export class SurveyView extends MobxLitElement { >
- ${unsafeHTML(convertMarkdownToHTML(question.questionTitle + "*"))} + ${unsafeHTML(convertMarkdownToHTML(question.questionTitle + '*'))}
@@ -183,13 +217,14 @@ export class SurveyView extends MobxLitElement { return html`
- ${unsafeHTML(convertMarkdownToHTML(question.questionTitle + "*"))} + ${unsafeHTML(convertMarkdownToHTML(question.questionTitle + '*'))}
@@ -239,7 +274,7 @@ export class SurveyView extends MobxLitElement { private renderRadioButton(choice: MultipleChoiceItem, questionId: string) { const id = `${questionId}-${choice.id}`; - const handleMultipleChoiceClick = (e: Event) => { + const handleMultipleChoiceClick = (_e: Event) => { const answer: MultipleChoiceSurveyAnswer = { id: questionId, kind: SurveyQuestionKind.MULTIPLE_CHOICE, @@ -312,7 +347,7 @@ export class SurveyView extends MobxLitElement { return html`
- ${unsafeHTML(convertMarkdownToHTML(question.questionTitle + "*"))} + ${unsafeHTML(convertMarkdownToHTML(question.questionTitle + '*'))}
${question.lowerText}
diff --git a/frontend/src/services/behavior.service.ts b/frontend/src/services/behavior.service.ts index 32077aff5..3958ca77e 100644 --- a/frontend/src/services/behavior.service.ts +++ b/frontend/src/services/behavior.service.ts @@ -28,6 +28,8 @@ export class BehaviorService extends Service { private flushIntervalMs = 5000; private intervalHandle: number | null = null; private listenersAttached = false; + private lastValueMap: WeakMap = new WeakMap(); + private inputSubscriptions: Set<() => void> = new Set(); start(experimentId: string, participantPrivateId: string) { // Ignore if already tracking same participant @@ -76,6 +78,11 @@ export class BehaviorService extends Service { window.clearInterval(this.intervalHandle); this.intervalHandle = null; } + // Tear down any per-field input subscriptions + for (const unsub of this.inputSubscriptions) { + unsub(); + } + this.inputSubscriptions.clear(); // Do not flush after clearing identifiers void this.flush(); this.experimentId = null; @@ -133,15 +140,14 @@ export class BehaviorService extends Service { private onKeyDown = (e: KeyboardEvent) => { // Do not capture actual key values; only log whether it was backspace/delete const keyName = (e.key || '').toLowerCase(); - const isBackspace = keyName === 'backspace'; - const isDelete = keyName === 'delete' || keyName === 'del'; + const modifiers: string[] = []; + if (e.ctrlKey) modifiers.push('ctrl'); + if (e.metaKey) modifiers.push('meta'); + if (e.altKey) modifiers.push('alt'); + if (e.shiftKey) modifiers.push('shift'); this.log('keydown', { - isBackspace, - isDelete, - ctrlKey: e.ctrlKey, - metaKey: e.metaKey, - altKey: e.altKey, - shiftKey: e.shiftKey, + keyName, + modifiers, repeat: e.repeat, isTrusted: e.isTrusted, targetTag: (e.target as HTMLElement | null)?.tagName || '', @@ -207,6 +213,142 @@ export class BehaviorService extends Service { }); }; + // used for searching in shadow dom for the actual text input element + private isAllowedTextInput( + el: Element, + ): el is HTMLInputElement | HTMLTextAreaElement { + if (el.tagName === 'TEXTAREA') return true; + if (el.tagName === 'INPUT') { + const input = el as HTMLInputElement; + const type = (input.type || 'text').toLowerCase(); + // Exclude password fields; allow common text types + return ( + type !== 'password' && + ['text', 'search', 'url', 'email', 'tel', 'number'].includes(type) + ); + } + return false; + } + + private getValue(el: HTMLInputElement | HTMLTextAreaElement): string { + return el.value ?? ''; + } + + private computeDiff(oldVal: string, newVal: string) { + // Simple diff: longest common prefix/suffix + let start = 0; + const oldLen = oldVal.length; + const newLen = newVal.length; + while ( + start < oldLen && + start < newLen && + oldVal[start] === newVal[start] + ) { + start++; + } + let end = 0; + while ( + end < oldLen - start && + end < newLen - start && + oldVal[oldLen - 1 - end] === newVal[newLen - 1 - end] + ) { + end++; + } + const deletedText = oldVal.slice(start, oldLen - end); + const addedText = newVal.slice(start, newLen - end); + return {addedText, deletedText, start, end}; + } + + // change events intentionally not tracked; per-input granularity only + + /** + * Attach input tracking to a specific text field host element. + * Works with native inputs/textareas and custom elements like pr-textarea. + * Returns an unsubscribe function to remove the listener. + */ + public attachTextInput(target: Element, fieldId: string): () => void { + const handler = (e: Event) => { + // Minimal resolution: prefer target itself, then its shadow root, then light DOM descendants + let el: HTMLInputElement | HTMLTextAreaElement | null = null; + if (this.isAllowedTextInput(target)) { + el = target as HTMLInputElement | HTMLTextAreaElement; + } else { + const sr = target.shadowRoot as ShadowRoot | undefined; + if (sr) { + const candidate = sr.querySelector('textarea, input'); + if (candidate && this.isAllowedTextInput(candidate)) { + el = candidate as HTMLInputElement | HTMLTextAreaElement; + } + } + if (!el && (target as Element).querySelector) { + const candidate = (target as Element).querySelector( + 'textarea, input', + ); + if (candidate && this.isAllowedTextInput(candidate)) { + el = candidate as HTMLInputElement | HTMLTextAreaElement; + } + } + } + if (!el) return; + + const newVal = this.getValue(el); + const oldVal = this.lastValueMap.get(el) ?? ''; + const {addedText, deletedText, start} = this.computeDiff(oldVal, newVal); + + if (!addedText && !deletedText) return; + + const ie = e as unknown as InputEvent; + const inputType = ie?.inputType || 'unknown'; + const data = ie?.data ?? null; + const isComposing = ie?.isComposing === true; + + const selStart = + (el as HTMLInputElement | HTMLTextAreaElement).selectionStart ?? null; + const selEnd = + (el as HTMLInputElement | HTMLTextAreaElement).selectionEnd ?? null; + + const MAX_DELTA_LEN = 10000; + const addedTrunc = addedText.length > MAX_DELTA_LEN; + const deletedTrunc = deletedText.length > MAX_DELTA_LEN; + + this.log('text_change', { + domEvent: 'input', + fieldId, + inputType, + data: typeof data === 'string' ? data.slice(0, MAX_DELTA_LEN) : data, + isComposing, + addedText: addedTrunc ? addedText.slice(0, MAX_DELTA_LEN) : addedText, + deletedText: deletedTrunc + ? deletedText.slice(0, MAX_DELTA_LEN) + : deletedText, + diffStart: start, + oldLength: oldVal.length, + newLength: newVal.length, + selectionStart: selStart, + selectionEnd: selEnd, + isTrusted: e.isTrusted === true, + targetTag: el.tagName, + inputTypeAttr: (el as HTMLInputElement).type || 'text', + }); + + this.lastValueMap.set(el, newVal); + }; + + // Listen to both input (in case event composes) and custom change emitted by pr-textarea + target.addEventListener('input', handler, {capture: true}); + target.addEventListener('change', handler, {capture: true}); + + const unsubscribe = () => { + target.removeEventListener('input', handler, true); + target.removeEventListener('change', handler, true); + }; + this.inputSubscriptions.add(unsubscribe); + return () => { + unsubscribe(); + this.inputSubscriptions.delete(unsubscribe); + }; + } + private onVisibilityChange = () => { if (document.visibilityState === 'hidden') { void this.flush(); From 97967a81f610a6fd8611dea3b5d1ae12b8af7a0b Mon Sep 17 00:00:00 2001 From: Ian Baker Date: Wed, 13 Aug 2025 15:40:55 -0700 Subject: [PATCH 3/5] remove small cruft --- frontend/src/services/behavior.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/services/behavior.service.ts b/frontend/src/services/behavior.service.ts index 3958ca77e..22628dc8b 100644 --- a/frontend/src/services/behavior.service.ts +++ b/frontend/src/services/behavior.service.ts @@ -326,7 +326,7 @@ export class BehaviorService extends Service { newLength: newVal.length, selectionStart: selStart, selectionEnd: selEnd, - isTrusted: e.isTrusted === true, + isTrusted: e.isTrusted, targetTag: el.tagName, inputTypeAttr: (el as HTMLInputElement).type || 'text', }); From 30cd79033d0df5c7e9c3a8ba4cbcfea783d14d66 Mon Sep 17 00:00:00 2001 From: Ian Baker Date: Wed, 13 Aug 2025 16:25:55 -0700 Subject: [PATCH 4/5] Added a config flag to enable behavior data collection --- .../experiment_settings_editor.ts | 29 ++++++++++++++----- frontend/src/services/behavior.service.ts | 20 ++++++++++--- frontend/src/services/experiment.editor.ts | 8 ++--- frontend/src/services/experiment.service.ts | 17 +++++------ utils/src/experiment.ts | 5 +++- utils/src/experiment.validation.ts | 1 + 6 files changed, 55 insertions(+), 25 deletions(-) diff --git a/frontend/src/components/experiment_builder/experiment_settings_editor.ts b/frontend/src/components/experiment_builder/experiment_settings_editor.ts index 6403ac6e2..cf89c2eea 100644 --- a/frontend/src/components/experiment_builder/experiment_settings_editor.ts +++ b/frontend/src/components/experiment_builder/experiment_settings_editor.ts @@ -1,15 +1,11 @@ import {MobxLitElement} from '@adobe/lit-mobx'; import {CSSResultGroup, html, nothing} from 'lit'; -import {customElement, property} from 'lit/decorators.js'; +import {customElement} from 'lit/decorators.js'; import '@material/web/textfield/filled-text-field.js'; import '@material/web/checkbox/checkbox.js'; import {core} from '../../core/core'; -import {ButtonClick, AnalyticsService} from '../../services/analytics.service'; -import {AuthService} from '../../services/auth.service'; -import {HomeService} from '../../services/home.service'; -import {Pages, RouterService} from '../../services/router.service'; import {ExperimentEditor} from '../../services/experiment.editor'; import {ExperimentManager} from '../../services/experiment.manager'; @@ -22,7 +18,6 @@ import {styles} from './experiment_settings_editor.scss'; export class ExperimentSettingsEditor extends MobxLitElement { static override styles: CSSResultGroup = [styles]; - private readonly analyticsService = core.getService(AnalyticsService); private readonly experimentEditor = core.getService(ExperimentEditor); private readonly experimentManager = core.getService(ExperimentManager); @@ -92,9 +87,16 @@ export class ExperimentSettingsEditor extends MobxLitElement { this.experimentEditor.updatePermissions({visibility}); }; + const isBehaviorEnabled = + this.experimentEditor.experiment.collectBehaviorData; + + const updateBehavior = () => { + this.experimentEditor.updateCollectBehaviorData(!isBehaviorEnabled); + }; + return html`
-
+
+
+ + +
+ Collect behavior data for bot detection (may incur additional + database costs) +
+
`; } diff --git a/frontend/src/services/behavior.service.ts b/frontend/src/services/behavior.service.ts index 22628dc8b..fe9fc4d9b 100644 --- a/frontend/src/services/behavior.service.ts +++ b/frontend/src/services/behavior.service.ts @@ -1,12 +1,14 @@ import {makeObservable} from 'mobx'; import {FirebaseService} from './firebase.service'; import {ParticipantService} from './participant.service'; +import {ExperimentService} from './experiment.service'; import {addBehaviorEventsCallable} from '../shared/callables'; import {Service} from './service'; interface ServiceProvider { firebaseService: FirebaseService; participantService: ParticipantService; + experimentService: ExperimentService; } /** Collects client interaction events and sends in small batches. */ @@ -30,8 +32,13 @@ export class BehaviorService extends Service { private listenersAttached = false; private lastValueMap: WeakMap = new WeakMap(); private inputSubscriptions: Set<() => void> = new Set(); + private enabled = false; start(experimentId: string, participantPrivateId: string) { + const exp = this.sp.experimentService.experiment; + this.enabled = exp?.collectBehaviorData === true; + if (!this.enabled) return; + // Ignore if already tracking same participant if ( this.experimentId === experimentId && @@ -45,10 +52,12 @@ export class BehaviorService extends Service { this.participantPrivateId = participantPrivateId; this.attachListeners(); // Periodic flush - this.intervalHandle = window.setInterval( - () => this.flush(), - this.flushIntervalMs, - ); + if (!this.intervalHandle) { + this.intervalHandle = window.setInterval( + () => this.flush(), + this.flushIntervalMs, + ); + } // Flush when page hidden/unloaded document.addEventListener('visibilitychange', this.onVisibilityChange, { capture: true, @@ -92,6 +101,7 @@ export class BehaviorService extends Service { /** Manually log a custom behavior event. */ log(eventType: string, metadata: Record = {}) { + if (!this.enabled) return; const stageId = this.sp.participantService.currentStageViewId ?? this.sp.participantService.profile?.currentStageId ?? @@ -111,6 +121,7 @@ export class BehaviorService extends Service { // Internal private attachListeners() { if (this.listenersAttached) return; + if (!this.enabled) return; document.addEventListener('click', this.onClick, true); document.addEventListener('keydown', this.onKeyDown, true); document.addEventListener('copy', this.onCopy, true); @@ -268,6 +279,7 @@ export class BehaviorService extends Service { */ public attachTextInput(target: Element, fieldId: string): () => void { const handler = (e: Event) => { + if (!this.enabled) return; // Minimal resolution: prefer target itself, then its shadow root, then light DOM descendants let el: HTMLInputElement | HTMLTextAreaElement | null = null; if (this.isAllowedTextInput(target)) { diff --git a/frontend/src/services/experiment.editor.ts b/frontend/src/services/experiment.editor.ts index 9e56fba14..63b55b6ea 100644 --- a/frontend/src/services/experiment.editor.ts +++ b/frontend/src/services/experiment.editor.ts @@ -7,10 +7,6 @@ import { StageConfig, StageKind, createExperimentConfig, - createMetadataConfig, - createPermissionsConfig, - createProlificConfig, - generateId, } from '@deliberation-lab/utils'; import {Timestamp} from 'firebase/firestore'; import {computed, makeObservable, observable} from 'mobx'; @@ -117,6 +113,10 @@ export class ExperimentEditor extends Service { }; } + updateCollectBehaviorData(enabled: boolean) { + this.experiment.collectBehaviorData = enabled; + } + setCurrentStageId(id: string | undefined) { this.currentStageId = id; } diff --git a/frontend/src/services/experiment.service.ts b/frontend/src/services/experiment.service.ts index d25db6fff..e35869189 100644 --- a/frontend/src/services/experiment.service.ts +++ b/frontend/src/services/experiment.service.ts @@ -1,17 +1,11 @@ import {computed, makeObservable, observable} from 'mobx'; -import { - collection, - doc, - getDocs, - onSnapshot, - Unsubscribe, -} from 'firebase/firestore'; +import {collection, doc, onSnapshot, Unsubscribe} from 'firebase/firestore'; import {FirebaseService} from './firebase.service'; import {AgentEditor} from './agent.editor'; -import {Pages, RouterService} from './router.service'; +import {RouterService} from './router.service'; import {Service} from './service'; -import {Experiment, StageConfig, StageKind} from '@deliberation-lab/utils'; +import {Experiment, StageConfig} from '@deliberation-lab/utils'; import {getPublicExperimentName} from '../shared/experiment.utils'; interface ServiceProvider { @@ -32,6 +26,8 @@ export class ExperimentService extends Service { // Experiment configs @observable experiment: Experiment | undefined = undefined; + // Temporary local flag until type updates propagate + @observable collectBehaviorData = false; @observable stageConfigMap: Record = {}; // TODO: Add roleConfigMap @@ -69,6 +65,9 @@ export class ExperimentService extends Service { cohortLockMap: {}, // for experiments version <= 11 ...doc.data(), } as Experiment; + // Keep a local mirror of the flag if present + this.collectBehaviorData = + (doc.data() as Experiment).collectBehaviorData === true; this.isExperimentLoading = false; }, ), diff --git a/utils/src/experiment.ts b/utils/src/experiment.ts index ba671bdcb..1c65b7d94 100644 --- a/utils/src/experiment.ts +++ b/utils/src/experiment.ts @@ -33,8 +33,9 @@ import {StageConfig} from './stages/stage'; * VERSION 16 - switch to new mediator workflow including updated ChatMessage * VERSION 17 - add structured output config to agent prompt configs * VERSION 18 - add agent participant config to ParticipantProfileExtended + * VERSION 19 - add collectBehaviorData flag to Experiment */ -export const EXPERIMENT_VERSION_ID = 18; +export const EXPERIMENT_VERSION_ID = 19; /** Experiment. */ export interface Experiment { @@ -47,6 +48,7 @@ export interface Experiment { prolificConfig: ProlificConfig; stageIds: string[]; // Ordered list of stage IDs cohortLockMap: Record; // maps cohort ID to is locked + collectBehaviorData: boolean; // enable behavior logging for bot detection } /** Experiment config for participant options. */ @@ -92,6 +94,7 @@ export function createExperimentConfig( prolificConfig: config.prolificConfig ?? createProlificConfig(), stageIds: stages.map((stage) => stage.id), cohortLockMap: config.cohortLockMap ?? {}, + collectBehaviorData: config.collectBehaviorData ?? false, }; } diff --git a/utils/src/experiment.validation.ts b/utils/src/experiment.validation.ts index 2e6ea0798..036019c96 100644 --- a/utils/src/experiment.validation.ts +++ b/utils/src/experiment.validation.ts @@ -82,6 +82,7 @@ export const ExperimentCreationData = Type.Object( prolificConfig: ProlificConfigSchema, stageIds: Type.Array(Type.String()), cohortLockMap: Type.Record(Type.String(), Type.Boolean()), + collectBehaviorData: Type.Boolean(), }, strict, ), From d0a707f83b439b7eda243904de521684a9f44751 Mon Sep 17 00:00:00 2001 From: Ian Baker Date: Wed, 13 Aug 2025 16:44:53 -0700 Subject: [PATCH 5/5] Add behavior log to json data export --- frontend/src/shared/file.utils.ts | 19 +++++++++++++++++++ utils/src/behavior.ts | 11 +++++++++++ utils/src/data.ts | 4 ++++ 3 files changed, 34 insertions(+) diff --git a/frontend/src/shared/file.utils.ts b/frontend/src/shared/file.utils.ts index e794853e4..5c08298ad 100644 --- a/frontend/src/shared/file.utils.ts +++ b/frontend/src/shared/file.utils.ts @@ -44,6 +44,7 @@ import { SurveyQuestion, SurveyQuestionKind, UnifiedTimestamp, + BehaviorEvent, calculatePayoutResult, calculatePayoutTotal, createCohortDownload, @@ -145,6 +146,24 @@ export async function getExperimentDownload( for (const stage of stageAnswers) { participantDownload.answerMap[stage.id] = stage; } + + // Fetch behavior events (ordered by server timestamp) + const behaviorDocs = await getDocs( + query( + collection( + firestore, + 'experiments', + experimentId, + 'participants', + profile.privateId, + 'behavior', + ), + orderBy('timestamp', 'asc'), + ), + ); + participantDownload.behavior = behaviorDocs.docs.map( + (doc) => doc.data() as BehaviorEvent, + ); // Add ParticipantDownload to ExperimentDownload experimentDownload.participantMap[profile.publicId] = participantDownload; } diff --git a/utils/src/behavior.ts b/utils/src/behavior.ts index 8313b7415..5ca043992 100644 --- a/utils/src/behavior.ts +++ b/utils/src/behavior.ts @@ -11,6 +11,17 @@ export interface BehaviorEventInput { metadata: Record; } +// Event shape as stored/exported from Firestore +// Note: backend writes field name `type` in documents. +import type {UnifiedTimestamp} from './shared'; +export interface BehaviorEvent { + type: string; + relativeTimestamp: number; + stageId: string; + metadata: Record; + timestamp: UnifiedTimestamp; +} + export interface AddBehaviorEventsData { experimentId: string; /** Participant private ID. */ diff --git a/utils/src/data.ts b/utils/src/data.ts index acf1bd875..e94d8e8ad 100644 --- a/utils/src/data.ts +++ b/utils/src/data.ts @@ -2,6 +2,7 @@ import {AgentDataObject} from './agent'; import {CohortConfig} from './cohort'; import {Experiment} from './experiment'; import {ParticipantProfileExtended} from './participant'; +import {BehaviorEvent} from './behavior'; import {ChatMessage} from './stages/chat_stage'; import { StageConfig, @@ -33,6 +34,8 @@ export interface ParticipantDownload { profile: ParticipantProfileExtended; // Maps from stage ID to participant's stage answer answerMap: Record; + // Ordered list of behavior events (if any) + behavior: BehaviorEvent[]; } export interface CohortDownload { @@ -63,6 +66,7 @@ export function createParticipantDownload( return { profile, answerMap: {}, + behavior: [], }; }