diff --git a/.smriti/CLAUDE.md b/.smriti/CLAUDE.md new file mode 100644 index 0000000..e879240 --- /dev/null +++ b/.smriti/CLAUDE.md @@ -0,0 +1,75 @@ +# Team Knowledge + +This directory contains shared knowledge from development sessions. +Generated by `smriti share`. Do not edit manually. + +## Project Context + +> Auto-generated by `smriti context` on 2026-02-11. Do not edit manually. + +### Recent Sessions (last 7 days) +- **3m ago** How does the ingestion pipeline work? What tables does it write to? (9 turns) +- **3m ago** How does the ingestion pipeline work? What tables does it write to? (9 turns) +- **13m ago** [Request interrupted by user for tool use] (68 turns) +- **13m ago** Let's plan the iggestion pipeline for all kinds of the agents. Here are my thoughts about how to pro (19 turns) +- **13m ago** [Request interrupted by user for tool use] (94 turns) + +### Hot Files +`src/context.ts` (16 ops), `src/db.ts` (12 ops), `src/ingest/index.ts` (7 ops), `~/.claude/plans/inherited-foraging-pebble.md` (6 ops), `src/index.ts` (6 ops), `src/ingest/claude.ts` (6 ops), `src/qmd.ts` (5 ops), `src/config.ts` (4 ops), `src/ingest/types.ts` (4 ops), `src/ingest/blocks.ts` (2 ops) + +### Git Activity +- commit (2026-02-11) + +### Usage +6 sessions, 301 turns, ~14K input / ~29K output tokens + +## architecture + +- [2026-02-09 the-session-addressed-gpu-recommendations-for-runn](knowledge/architecture/2026-02-09_the-session-addressed-gpu-recommendations-for-runn.md) + +## bug + +- [2026-02-10 the-session-focused-on-preparing-the-avkash-projec](knowledge/bug/2026-02-10_the-session-focused-on-preparing-the-avkash-projec.md) +- [2026-02-10 the-session-focused-on-resolving-critical-dependen](knowledge/bug/2026-02-10_the-session-focused-on-resolving-critical-dependen.md) + +## code + +- [2026-02-10 the-session-focused-on-implementing-a-knowledge-ma](knowledge/code/2026-02-10_the-session-focused-on-implementing-a-knowledge-ma.md) +- [2026-02-10 the-session-focused-on-implementing-a-conversation](knowledge/code/2026-02-10_the-session-focused-on-implementing-a-conversation.md) +- [2026-02-10 the-session-completed-the-full-implementation-of-t](knowledge/code/2026-02-10_the-session-completed-the-full-implementation-of-t.md) +- [2026-02-10 this-session-focused-on-preparing-the-avkashio-pro](knowledge/code/2026-02-10_this-session-focused-on-preparing-the-avkashio-pro.md) +- [2026-02-10 this-session-focused-on-transitioning-a-project-to](knowledge/code/2026-02-10_this-session-focused-on-transitioning-a-project-to.md) +- [2026-02-10 this-session-implemented-a-scalable-organizational](knowledge/code/2026-02-10_this-session-implemented-a-scalable-organizational.md) +- [2026-02-10 the-session-focused-on-implementing-and-validating](knowledge/code/2026-02-10_the-session-focused-on-implementing-and-validating.md) +- [2026-02-10 this-session-implemented-deputation-and-attendance](knowledge/code/2026-02-10_this-session-implemented-deputation-and-attendance.md) + +## feature + +- [2026-02-10 a-persistent-conversation-memory-layer-was-impleme](knowledge/feature/2026-02-10_a-persistent-conversation-memory-layer-was-impleme.md) +- [2026-02-10 this-session-focused-on-finalizing-project-documen](knowledge/feature/2026-02-10_this-session-focused-on-finalizing-project-documen.md) +- [2026-02-10 this-session-implemented-a-secure-https-api-server](knowledge/feature/2026-02-10_this-session-implemented-a-secure-https-api-server.md) +- [2026-02-09 this-session-focused-on-building-a-robust-authoriz](knowledge/feature/2026-02-09_this-session-focused-on-building-a-robust-authoriz.md) + +## project + +- [2026-02-10 the-session-involved-setting-up-a-new-project-name](knowledge/project/2026-02-10_the-session-involved-setting-up-a-new-project-name.md) +- [2026-02-10 the-session-clarified-that-the-claude-cli-is-incom](knowledge/project/2026-02-10_the-session-clarified-that-the-claude-cli-is-incom.md) +- [2026-02-10 this-session-resolved-opencodes-tool-calling-issue](knowledge/project/2026-02-10_this-session-resolved-opencodes-tool-calling-issue.md) +- [2026-02-10 this-session-established-automated-cicd-pipelines](knowledge/project/2026-02-10_this-session-established-automated-cicd-pipelines.md) +- [2026-02-10 this-session-implemented-an-org-management-system](knowledge/project/2026-02-10_this-session-implemented-an-org-management-system.md) +- [2026-02-10 this-session-established-a-production-ready-setup](knowledge/project/2026-02-10_this-session-established-a-production-ready-setup.md) +- [2026-02-10 a-new-local-rag-project-named-smriti-was-created-u](knowledge/project/2026-02-10_a-new-local-rag-project-named-smriti-was-created-u.md) +- [2026-02-09 the-session-focused-on-configuring-ollama-by-modif](knowledge/project/2026-02-09_the-session-focused-on-configuring-ollama-by-modif.md) + +## topic + +- [2026-02-10 the-session-established-a-plan-to-build-smriti-a-u](knowledge/topic/2026-02-10_the-session-established-a-plan-to-build-smriti-a-u.md) + +## uncategorized + +- [2026-02-10 this-session-documented-the-tagging-and-categoriza](knowledge/uncategorized/2026-02-10_this-session-documented-the-tagging-and-categoriza.md) +- [2026-02-10 this-session-focused-on-refining-smritis-documenta](knowledge/uncategorized/2026-02-10_this-session-focused-on-refining-smritis-documenta.md) +- [2026-02-10 the-session-focused-on-transforming-raw-ai-convers](knowledge/uncategorized/2026-02-10_the-session-focused-on-transforming-raw-ai-convers.md) +- [2026-02-10 this-session-focused-on-finalizing-the-smriti-tool](knowledge/uncategorized/2026-02-10_this-session-focused-on-finalizing-the-smriti-tool.md) +- [2026-02-10 the-session-focused-on-documenting-a-memory-system](knowledge/uncategorized/2026-02-10_the-session-focused-on-documenting-a-memory-system.md) +- [2026-02-10 the-session-involved-verifying-if-previous-convers](knowledge/uncategorized/2026-02-10_the-session-involved-verifying-if-previous-convers.md) diff --git a/.smriti/config.json b/.smriti/config.json new file mode 100644 index 0000000..7f3d982 --- /dev/null +++ b/.smriti/config.json @@ -0,0 +1,7 @@ +{ + "version": 1, + "allowedCategories": [ + "*" + ], + "autoSync": false +} \ No newline at end of file diff --git a/.smriti/index.json b/.smriti/index.json new file mode 100644 index 0000000..e119e70 --- /dev/null +++ b/.smriti/index.json @@ -0,0 +1,182 @@ +[ + { + "id": "2e5f420a-e376-4ad4-8b35-ad94838cbc42", + "category": "project", + "file": "knowledge/project/2026-02-10_the-session-involved-setting-up-a-new-project-name.md", + "shared_at": "2026-02-10T11:29:44.517Z" + }, + { + "id": "598764f9-ce18-4f90-b951-210e7f6afd1c", + "category": "project", + "file": "knowledge/project/2026-02-10_the-session-clarified-that-the-claude-cli-is-incom.md", + "shared_at": "2026-02-10T11:30:13.128Z" + }, + { + "id": "ec2a9411-039d-4cae-83e0-99913c290bbc", + "category": "code", + "file": "knowledge/code/2026-02-10_the-session-focused-on-implementing-a-knowledge-ma.md", + "shared_at": "2026-02-10T11:31:17.212Z" + }, + { + "id": "bc0a47ce-db71-4cf0-87bc-ea467c9f6ce0", + "category": "topic", + "file": "knowledge/topic/2026-02-10_the-session-established-a-plan-to-build-smriti-a-u.md", + "shared_at": "2026-02-10T11:32:09.496Z" + }, + { + "id": "04321d7f-1ce4-41c6-823c-344026795afa", + "category": "project", + "file": "knowledge/project/2026-02-10_this-session-resolved-opencodes-tool-calling-issue.md", + "shared_at": "2026-02-10T11:33:12.809Z" + }, + { + "id": "84aa0a49-6d65-455d-87d9-b53023cf06cd", + "category": "feature", + "file": "knowledge/feature/2026-02-10_a-persistent-conversation-memory-layer-was-impleme.md", + "shared_at": "2026-02-10T11:34:06.080Z" + }, + { + "id": "cc920155-7aba-40e5-897d-53a9ae566c7f", + "category": "code", + "file": "knowledge/code/2026-02-10_the-session-focused-on-implementing-a-conversation.md", + "shared_at": "2026-02-10T11:34:49.433Z" + }, + { + "id": "dc3a6584", + "category": "uncategorized", + "file": "knowledge/uncategorized/2026-02-10_this-session-documented-the-tagging-and-categoriza.md", + "shared_at": "2026-02-10T17:53:50.801Z" + }, + { + "id": "96102237", + "category": "uncategorized", + "file": "knowledge/uncategorized/2026-02-10_this-session-focused-on-refining-smritis-documenta.md", + "shared_at": "2026-02-10T17:54:38.723Z" + }, + { + "id": "a8255f26", + "category": "uncategorized", + "file": "knowledge/uncategorized/2026-02-10_the-session-focused-on-transforming-raw-ai-convers.md", + "shared_at": "2026-02-10T17:55:38.712Z" + }, + { + "id": "c84adc84", + "category": "uncategorized", + "file": "knowledge/uncategorized/2026-02-10_this-session-focused-on-finalizing-the-smriti-tool.md", + "shared_at": "2026-02-10T17:56:35.452Z" + }, + { + "id": "ec2a9411", + "category": "code", + "file": "knowledge/code/2026-02-10_the-session-completed-the-full-implementation-of-t.md", + "shared_at": "2026-02-10T17:57:31.204Z" + }, + { + "id": "a9a45641-1bf9-41da-9aa4-1f61815d71ab", + "category": "code", + "file": "knowledge/code/2026-02-10_this-session-focused-on-preparing-the-avkashio-pro.md", + "shared_at": "2026-02-10T17:58:23.431Z" + }, + { + "id": "9028693f-3fb1-47b9-8a2b-a5d6771d5059", + "category": "bug", + "file": "knowledge/bug/2026-02-10_the-session-focused-on-preparing-the-avkash-projec.md", + "shared_at": "2026-02-10T17:59:24.778Z" + }, + { + "id": "fd956621-8cae-423b-8b42-3c397d5a9434", + "category": "code", + "file": "knowledge/code/2026-02-10_this-session-focused-on-transitioning-a-project-to.md", + "shared_at": "2026-02-10T18:00:23.780Z" + }, + { + "id": "7d2fb4ba-5b2e-4e34-9d40-951aaaa7a1de", + "category": "project", + "file": "knowledge/project/2026-02-10_this-session-established-automated-cicd-pipelines.md", + "shared_at": "2026-02-10T18:01:13.418Z" + }, + { + "id": "7a03996f-d04d-46b7-a30e-b69ea3770a5e", + "category": "bug", + "file": "knowledge/bug/2026-02-10_the-session-focused-on-resolving-critical-dependen.md", + "shared_at": "2026-02-10T18:02:06.001Z" + }, + { + "id": "94d5d582-f9d5-481f-bc59-42291c79f8a8", + "category": "project", + "file": "knowledge/project/2026-02-10_this-session-implemented-an-org-management-system.md", + "shared_at": "2026-02-10T18:03:04.684Z" + }, + { + "id": "0a03e5ef-f35c-481b-9dac-b6eee7422ff2", + "category": "feature", + "file": "knowledge/feature/2026-02-10_this-session-focused-on-finalizing-project-documen.md", + "shared_at": "2026-02-10T18:03:57.086Z" + }, + { + "id": "2ecff2c6-8821-4d3f-8f87-66d4bd29a4e1", + "category": "code", + "file": "knowledge/code/2026-02-10_this-session-implemented-a-scalable-organizational.md", + "shared_at": "2026-02-10T18:04:56.943Z" + }, + { + "id": "e479ed40-79cb-4b2a-a959-3e3f85ae7047", + "category": "project", + "file": "knowledge/project/2026-02-10_this-session-established-a-production-ready-setup.md", + "shared_at": "2026-02-10T18:05:52.975Z" + }, + { + "id": "40b3e2ee-e169-40cb-8085-a8f04cb303d3", + "category": "feature", + "file": "knowledge/feature/2026-02-10_this-session-implemented-a-secure-https-api-server.md", + "shared_at": "2026-02-10T18:06:41.729Z" + }, + { + "id": "44fff7a5-fafb-4939-b032-de20721d57bc", + "category": "code", + "file": "knowledge/code/2026-02-10_the-session-focused-on-implementing-and-validating.md", + "shared_at": "2026-02-10T18:07:35.568Z" + }, + { + "id": "e96025a3-0459-4eae-964c-74dd8c004e1c", + "category": "code", + "file": "knowledge/code/2026-02-10_this-session-implemented-deputation-and-attendance.md", + "shared_at": "2026-02-10T18:08:30.294Z" + }, + { + "id": "3c9485f4-67bf-41e0-8eb4-6a4413e8b7dd", + "category": "uncategorized", + "file": "knowledge/uncategorized/2026-02-10_the-session-focused-on-documenting-a-memory-system.md", + "shared_at": "2026-02-10T18:09:00.058Z" + }, + { + "id": "2e5f420a", + "category": "project", + "file": "knowledge/project/2026-02-10_a-new-local-rag-project-named-smriti-was-created-u.md", + "shared_at": "2026-02-10T18:09:49.711Z" + }, + { + "id": "3c9485f4", + "category": "uncategorized", + "file": "knowledge/uncategorized/2026-02-10_the-session-involved-verifying-if-previous-convers.md", + "shared_at": "2026-02-10T18:10:17.083Z" + }, + { + "id": "84aa0a49", + "category": "feature", + "file": "knowledge/feature/2026-02-09_this-session-focused-on-building-a-robust-authoriz.md", + "shared_at": "2026-02-10T18:11:05.728Z" + }, + { + "id": "7c130ccd", + "category": "architecture", + "file": "knowledge/architecture/2026-02-09_the-session-addressed-gpu-recommendations-for-runn.md", + "shared_at": "2026-02-10T18:11:36.937Z" + }, + { + "id": "f1543e51", + "category": "project", + "file": "knowledge/project/2026-02-09_the-session-focused-on-configuring-ollama-by-modif.md", + "shared_at": "2026-02-10T18:12:06.194Z" + } +] \ No newline at end of file diff --git a/.smriti/knowledge/architecture/2026-02-09_the-session-addressed-gpu-recommendations-for-runn.md b/.smriti/knowledge/architecture/2026-02-09_the-session-addressed-gpu-recommendations-for-runn.md new file mode 100644 index 0000000..d6871f9 --- /dev/null +++ b/.smriti/knowledge/architecture/2026-02-09_the-session-addressed-gpu-recommendations-for-runn.md @@ -0,0 +1,34 @@ +--- +id: 7c130ccd +category: architecture +project: +agent: +author: zero8 +shared_at: 2026-02-10T18:11:36.927Z +tags: ["architecture"] +--- + +# The session addressed GPU recommendations for running local large language mo... + +> The session addressed GPU recommendations for running local large language models (LLMs), emphasizing NVIDIA and Apple Silicon options. It highlighted hardware requirements, software compatibility, and performance trade-offs for different architectures. + +## Changes + +- **N/A** + +## Decisions + +- **Recommended NVIDIA GPUs**: RTX 3060 12GB and RTX 4060 Ti 16GB for their CUDA ecosystem support and VRAM adequacy. +- **AMD GPUs as alternative**: Acknowledged but noted limited software tooling compared to NVIDIA. +- **Apple Silicon (M1/M2/M3/M4)**: Prioritized for unified memory architecture, enabling larger model sizes than discrete GPUs with equivalent VRAM. +- **M2 Pro 32GB as benchmark**: Selected for running 13B models, balancing cost and performance. + +## Insights + +- **Unified memory advantage**: Apple Silicon’s architecture allows efficient memory management, reducing VRAM bottlenecks for LLMs. +- **NVIDIA dominance in LLM inference**: CUDA ecosystem and driver maturity make NVIDIA GPUs more reliable for production workloads. +- **AMD’s niche role**: Suitable for cost-sensitive setups but may require additional optimization for LLMs. + +## Context + +Prior state: User needed guidance on hardware for local LLM deployment. Constraints included VRAM limitations and software ecosystem compatibility. Gotchas: Apple Silicon’s lack of x86 support but strong performance for single-machine inference; AMD’s potential for lower costs but higher complexity in LLM optimization. diff --git a/.smriti/knowledge/bug/2026-02-10_the-session-focused-on-preparing-the-avkash-projec.md b/.smriti/knowledge/bug/2026-02-10_the-session-focused-on-preparing-the-avkash-projec.md new file mode 100644 index 0000000..fbdd34f --- /dev/null +++ b/.smriti/knowledge/bug/2026-02-10_the-session-focused-on-preparing-the-avkash-projec.md @@ -0,0 +1,54 @@ +--- +id: 9028693f-3fb1-47b9-8a2b-a5d6771d5059 +category: bug +project: zero8-dev-avkash +agent: claude-code +author: zero8 +shared_at: 2026-02-10T17:59:24.766Z +tags: ["bug", "bug/report"] +--- + +# The session focused on preparing the Avkash project for open-source release b... + +> The session focused on preparing the Avkash project for open-source release by implementing standardized documentation, licensing, and security practices. Key actions included rewriting the README, creating a contribution guide, adding a Business Source License 1.1, removing Vercel-specific integrations, and cleaning up committed secrets. These changes ensure the project is accessible, compliant, and secure for open-source collaboration. + +--- + +## Changes + +- **Modified `README.md`**: Updated to include project description, features, tech stack, prerequisites, installation steps, environment variables table, and links to CONTRIBUTING.md/LICENSE. +- **Created `CONTRIBUTING.md`**: Defined fork/clone process, branch naming conventions, commitlint rules (`type(scope): message`), PR workflow, and pre-commit hooks (husky + lint-staged). +- **Added `LICENSE`**: Implemented BSL 1.1 with licensor `zero8.dev`, licensed work `Avkash`, and restrictions on reselling. +- **Removed Vercel integrations**: + - Deleted `@vercel/analytics` from `package.json` + - Removed `Analytics` import and component from `src/app/layout.tsx` + - Deleted `vercel.sh` and removed `.vercel` from `.gitignore` +- **Cleaned secrets**: + - Removed `.env` from git tracking via `git rm --cached .env` + - Updated `.gitignore` to explicitly ignore `.env` + - Renamed `.env.local.sample` → `.env.example` and stripped real Slack/Razorpay values + - Added `data_dump.sql` to `.gitignore` + +--- + +## Decisions + +- **BSL 1.1 License**: Chosen to allow self-hosting for internal use while restricting commercial resale, with a transition to Apache 2.0 after 4 years. This balances permissiveness and control. +- **Secret Cleanup**: Removed `.env` from git to prevent exposure of real credentials, even though historical commits may still contain them. +- **Commitlint Rules**: Enforced structured commit messages to improve code review clarity and auditability. +- **Pre-Commit Hooks**: Integrated husky + lint-staged to automate formatting, linting, and type-checking, ensuring code quality before commits. + +--- + +## Insights + +- **Secret Management**: Committed secrets (e.g., Slack/Razorpay keys) in git history pose a risk, even if rotated. Future steps may require `git filter-repo` to purge historical data. +- **Build Dependencies**: The `sharp` module caused pre-push build failures, highlighting the need for environment-specific dependency management. +- **License Strategy**: BSL 1.1 is ideal for projects requiring both open-source flexibility and commercial restrictions, but its complexity demands clear documentation. +- **Open-Source Readiness**: Standardized documentation and contributor workflows are critical for attracting and onboarding collaborators. + +--- + +## Context + +The project was transitioning from a private to open-source repository, requiring strict adherence to licensing, security, and documentation standards. Constraints included avoiding Vercel-specific dependencies, preventing secret exposure, and diff --git a/.smriti/knowledge/bug/2026-02-10_the-session-focused-on-resolving-critical-dependen.md b/.smriti/knowledge/bug/2026-02-10_the-session-focused-on-resolving-critical-dependen.md new file mode 100644 index 0000000..c301125 --- /dev/null +++ b/.smriti/knowledge/bug/2026-02-10_the-session-focused-on-resolving-critical-dependen.md @@ -0,0 +1,49 @@ +--- +id: 7a03996f-d04d-46b7-a30e-b69ea3770a5e +category: bug +project: zero8-dev-avkash-regulation-hub +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:02:05.991Z +tags: ["bug", "bug/fix"] +--- + +# The session focused on resolving critical dependency issues, ensuring commit ... + +> The session focused on resolving critical dependency issues, ensuring commit attribution, and setting up deployment infrastructure for the Avkash Regulation Hub project. Key actions included fixing missing `lucuide-react` dependency, updating the CONTRIBUTING.md URL, committing changes without AI attribution, and configuring Cloudflare and GitHub secrets for automated deployment. These steps ensure the project is deployable, secure, and aligned with the team's workflow. + +--- + +## Changes + +- **Modified `package.json`**: Added `lucide-react` as a dependency (v0.563.0) to resolve build failures. +- **Updated `CONTRIBUTING.md`**: Changed placeholder URL from `your-org/avkash-regulation-hub` to `zero8dotdev/avkash-regulation-hub`. +- **Committed changes**: Hash `6ea014d` with authorship attributed to the user, excluding AI co-authorship. +- **Executed Cloudflare CLI commands**: + - Retrieved Cloudflare account ID `f0ecc60e106966433010db7b9800a0cc`. + - Attempted to refresh OAuth token for GitHub Actions integration. +- **Executed GitHub CLI commands**: + - Saved Cloudflare API tokens as GitHub secrets for CI/CD pipelines. + +--- + +## Decisions + +- **Retained Vite over Bun.serve**: Despite CLAUDE.md suggesting Bun.serve, the team opted to maintain Vite for consistency with the existing project setup. +- **Enforced user commit attribution**: All future commits will be authored by the user, ensuring accountability and avoiding AI-assisted contributions. +- **Token management approach**: Used Cloudflare API tokens for GitHub Actions instead of short-lived OAuth tokens to ensure long-term CI/CD reliability. + +--- + +## Insights + +- **Dependency management**: Missing dependencies like `lucide-react` can block builds entirely, requiring explicit package.json updates. +- **Commit attribution**: Central to maintaining ownership and traceability in collaborative workflows. +- **Token lifecycle**: Short-lived OAuth tokens are unsuitable for CI/CD; permanent API tokens are necessary for automation. +- **Cloudflare integration**: Requires careful token handling to avoid exposure of sensitive credentials in workflows. + +--- + +## Context + +The project is a Preact + Vite-based static site for an Indian labor regulation platform. Initial issues included unresolved dependencies, placeholder URLs, and architectural misalignment with CLAUDE.md. Deployment required Cloudflare CLI for API tokens and GitHub CLI to secure secrets. The team prioritized consistency with existing tools (Vite) over proposed diff --git a/.smriti/knowledge/code/2026-02-10_the-session-completed-the-full-implementation-of-t.md b/.smriti/knowledge/code/2026-02-10_the-session-completed-the-full-implementation-of-t.md new file mode 100644 index 0000000..225a107 --- /dev/null +++ b/.smriti/knowledge/code/2026-02-10_the-session-completed-the-full-implementation-of-t.md @@ -0,0 +1,48 @@ +--- +id: ec2a9411 +category: code +project: +agent: +author: zero8 +shared_at: 2026-02-10T17:57:31.196Z +tags: ["code", "code/implementation", "decision"] +--- + +# The session completed the full implementation of the Smriti CLI tool, integra... + +> The session completed the full implementation of the Smriti CLI tool, integrating QMD for memory management, SQLite for storage, and modular architecture for ingestion, categorization, search, and team collaboration. All 38 tests passed, and end-to-end workflows (ingestion, categorization, search, sharing) were validated with real data, ensuring robustness and usability for knowledge management in development workflows. + +--- + +## Changes + +- **Files Created/Modified** + - `src/config.ts` - Configuration for paths, environment variables, and defaults. + - `src/db.ts` - Database schema initialization, category seeding, and CRUD helpers using Bun:sqlite. + - `src/ingest/claude.ts` - Parser for Claude Code JSONL format. + - `src/ingest/codex.ts` - Parser for Codex CLI output. + - `src/ingest/cursor.ts` - Parser for Cursor IDE logs. + - `src/ingest/generic.ts` - Generic file importer wrapping QMD’s `importTranscript`. + - `src/ingest/index.ts` - Ingest orchestrator with deduplication logic. + - `src/categorize/schema.ts` - Category taxonomy definitions and CRUD operations. + - `src/categorize/classifier.ts` - Rule-based classifier with optional LLM integration. + - `src/search/index.ts` - Filtered full-text search with category/project/agent filters. + - `src/search/recall.ts` - Enhanced recall with synthesis and filtering. + - `src/team/share.ts` - Team knowledge export to `.smriti/` with YAML frontmatter. + - `src/team/sync.ts` - Team knowledge import with deduplication. + - `src/format.ts` - CLI output formatting (table, JSON, markdown). + - `src/index.ts` - CLI entry point wiring all 14 commands. + - `test/db.test.ts`, `test/ingest.test.ts`, `test/categorize.test.ts`, `test/search.test.ts`, `test/team.test.ts` - Unit tests for each module. + - `package.json` - Added `bin` entry for CLI and dependencies (`bun:sqlite`, `Bun.file`, `Bun.glob`). + +- **CLI Commands Added** + - `smriti ingest claude`, `smriti categorize`, `smriti search`, `smriti recall`, `smriti share`, `smriti sync`, `smriti status`, `smriti list`, `smriti projects`, `smriti format`, `smriti export`, `smriti import`, `smriti help`, `smriti version`. + +--- + +## Decisions + +- **SQLite as Primary DB** + Chose Bun’s built-in `bun:sqlite` for lightweight, embedded storage, avoiding external DB setup. +- **Modular Architecture** + Separated ingestion, categorization, search, and team features into distinct modules for maintainability and testability diff --git a/.smriti/knowledge/code/2026-02-10_the-session-focused-on-implementing-a-conversation.md b/.smriti/knowledge/code/2026-02-10_the-session-focused-on-implementing-a-conversation.md new file mode 100644 index 0000000..429ae0b --- /dev/null +++ b/.smriti/knowledge/code/2026-02-10_the-session-focused-on-implementing-a-conversation.md @@ -0,0 +1,39 @@ +--- +id: cc920155-7aba-40e5-897d-53a9ae566c7f +category: code +project: smriti +agent: claude-code +author: zero8 +shared_at: 2026-02-10T11:34:49.424Z +tags: ["code", "code/implementation"] +--- + +# The session focused on implementing a conversation memory layer for a local O... + +> The session focused on implementing a conversation memory layer for a local Ollama setup, enabling persistent chat history across sessions. The solution extends QMD's existing architecture with a CLI tool, leveraging SQLite for storage and integrating with the MCP server. This ensures LLMs retain context from prior interactions without requiring external dependencies. + +## Changes + +- Created `memory.js` (logic for persisting/chat history) +- Created `cli.js` (CLI tool for interacting with memory layer) +- Modified `qmd.config.js` (added memory plugin configuration) +- Updated `qmd.js` (integrated memory layer with MCP server) +- Added `memory-plugin.js` (QMD plugin for memory persistence) + +## Decisions + +- **CLI over API**: Chose CLI for simplicity in local development workflows rather than building a separate API service. +- **SQLite integration**: Used QMD's existing SQLite backend for persistence instead of external databases to avoid duplication. +- **Session-based storage**: Stored chat history per session in SQLite to balance persistence and privacy. +- **Plugin architecture**: Designed memory layer as a QMD plugin to maintain compatibility with existing workflows. + +## Insights + +- QMD's SQLite backend is already optimized for vector search, making it a natural fit for memory persistence. +- CLI tools are critical for local development but may require additional safeguards for production use. +- Storing session data in SQLite introduces risks of data fragmentation; regular cleanup is recommended. +- The MCP server's event-driven architecture simplifies integrating memory persistence without overhauling existing workflows. + +## Context + +Prior state: Ollama v0.15.6 with QMD's hybrid search (BM25 + sqlite-vec) and Mastra AI Workflow's RAG module. Constraints: Avoid external dependencies, ensure compatibility with existing MCP server. Gotchas: Handling concurrent access to SQLite, ensuring memory data doesn't conflict with QMD's vector search index. diff --git a/.smriti/knowledge/code/2026-02-10_the-session-focused-on-implementing-a-knowledge-ma.md b/.smriti/knowledge/code/2026-02-10_the-session-focused-on-implementing-a-knowledge-ma.md new file mode 100644 index 0000000..99ef645 --- /dev/null +++ b/.smriti/knowledge/code/2026-02-10_the-session-focused-on-implementing-a-knowledge-ma.md @@ -0,0 +1,46 @@ +--- +id: ec2a9411-039d-4cae-83e0-99913c290bbc +category: code +project: smriti +agent: claude-code +author: zero8 +shared_at: 2026-02-10T11:31:17.198Z +tags: ["code", "code/implementation"] +--- + +# The session focused on implementing a knowledge management system called "smr... + +> The session focused on implementing a knowledge management system called "smriti" with core features for categorization, search, team sharing, and CLI integration. These capabilities enable structured storage, intelligent retrieval, and collaborative knowledge sharing, addressing challenges in organizing technical documentation and project insights. + +--- + +## Changes + +- **Files created/modified**: + - `smriti/src/db.ts` (schema init, migrations, DB connection) + - `smriti/src/config.ts` (env vars, paths, defaults) + - `smriti/src/ingest/claude.ts`, `codex.ts`, `cursor.ts`, `generic.ts` (parsers for different agents) + - `smriti/src/categorize/schema.ts`, `classifier.ts` (category definitions, rule-based + LLM classification) + - `smriti/src/search/index.ts`, `recall.ts` (filtered search with QMD, enhanced recall) + - `smriti/src/team/share.ts`, `sync.ts` (export/import logic for `.smriti/` directory) + - `smriti/src/index.ts` (CLI entry point with command wiring) + - `smriti/src/format.ts` (output formatting for tables, JSON, markdown) + - `smriti/db/tables/schema.sql` (reference schema for SQLite) + - `smriti/package.json` (added `bin` entry for CLI) + +- **Features added**: + - CLI commands for ingestion, search, recall, categorization, tagging, and team sharing + - Filtered search with category/project/agent filters via joins on `smriti_session_meta` and `smriti_message_tags` + - Enhanced recall with project/category filtering before synthesis + - Git-based team sharing via `.smriti/` directory with YAML frontmatter metadata + - Ollama integration for ambiguous message categorization + +- **Bug fixes**: + - Corrected test assertion for edge case with 2 segments in `claude.ts` + - Fixed linter-mangled comment in `claude.ts` + +--- + +## Decisions + +- **Categorization strategy**: Used Ollama for ambiguous messages (rule-based confidence < threshold) to avoid manual tagging diff --git a/.smriti/knowledge/code/2026-02-10_the-session-focused-on-implementing-and-validating.md b/.smriti/knowledge/code/2026-02-10_the-session-focused-on-implementing-and-validating.md new file mode 100644 index 0000000..e16c111 --- /dev/null +++ b/.smriti/knowledge/code/2026-02-10_the-session-focused-on-implementing-and-validating.md @@ -0,0 +1,58 @@ +--- +id: 44fff7a5-fafb-4939-b032-de20721d57bc +category: code +project: zero8-dev-openfga +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:07:35.557Z +tags: ["code", "code/implementation"] +--- + +# The session focused on implementing and validating an access control system f... + +> The session focused on implementing and validating an access control system for Sharma Auto using FGA (Flexible Graph Access Control) and a demo script to ensure end-to-end functionality. Key outcomes included resolving stale data conflicts, automating cleanup for repeatable demos, and debating architectural choices for role-based permissions. The demo script now ensures a clean slate on each run, and the team evaluated two approaches for authorization: maintaining a simple FGA model with app-level checks or expanding FGA with scoped relations. + +--- + +## Changes + +- **Files created/modified**: + - `demo-sharma-auto.sh` (initial commit, handles database truncation, FGA store deletion, and demo execution) + - Added cleanup trap to stop the server process after demo completion +- **Features added**: + - Automated cleanup of database tables (`users`, `orgs`, `departments`, `branches`, `members`, `resources`) + - Deletion of OpenFGA stores to prevent stale tuples from interfering with demos + - Server restart logic to ensure a fresh FGA store on each run +- **Bugs fixed**: + - Resolved 500 errors caused by lingering FGA tuples from previous demo runs + - Fixed 409 conflicts by truncating the database before rerunning the demo + +--- + +## Decisions + +- **Approach A (simple FGA + app-level checks)**: + - Chosen for immediate implementation due to its simplicity and alignment with the current FGA model. + - Rationale: Avoids overcomplicating the FGA model while allowing gradual integration of role-based logic. +- **Cleanup automation**: + - Decided to truncate all database tables and delete FGA stores on each demo run to ensure isolation. + - Rationale: Prevents data contamination between demo iterations and ensures consistent testing. +- **Server process management**: + - Added a trap to stop the server process after the demo completes to avoid orphaned processes. + - Rationale: Ensures resource cleanup and prevents accidental interference with other workflows. + +--- + +## Insights + +- **FGA model limitations**: The current flat model (`admin`/`member`) lacks granularity for role-based access (e.g., branch managers, HR admins). Expanding FGA with scoped relations (e.g., `branch_manager`, `hr_admin`) would centralize authorization logic but requires careful migration. +- **Stale data risks**: FGA tuples persist across runs, leading to conflicts unless explicitly cleaned. Database truncation alone is insufficient; FGA stores must also be reset. +- **Phased implementation**: The team opted for a hybrid approach, using app-level checks for immediate needs while planning a future FGA model expansion. This avoids over-engineering while keeping the door open for future enhancements. + +--- + +## Context + +- **Existing state**: FGA model is flat, with `admin` and `member` roles. The `level` field in `org_members` is unused for authorization. +- **Constraints**: + - Demo diff --git a/.smriti/knowledge/code/2026-02-10_this-session-focused-on-preparing-the-avkashio-pro.md b/.smriti/knowledge/code/2026-02-10_this-session-focused-on-preparing-the-avkashio-pro.md new file mode 100644 index 0000000..de3420a --- /dev/null +++ b/.smriti/knowledge/code/2026-02-10_this-session-focused-on-preparing-the-avkashio-pro.md @@ -0,0 +1,31 @@ +--- +id: a9a45641-1bf9-41da-9aa4-1f61815d71ab +category: code +project: zero8-dev-avkash +agent: claude-code +author: zero8 +shared_at: 2026-02-10T17:58:23.419Z +tags: ["code", "code/review"] +--- + +# This session focused on preparing the avkash.io project for open-source relea... + +> This session focused on preparing the avkash.io project for open-source release by creating essential documentation, configuring a restrictive license, removing Vercel integration, and scanning for secrets. These steps ensure clarity for contributors, compliance with open-source norms, and security hygiene for the codebase. + +--- + +## Changes + +- **Created `README.md`** at the project root to introduce the project, its purpose, and usage instructions. +- **Created `CONTRIBUTING.md`** in the `docs/` directory to outline contribution guidelines, code standards, and onboarding steps. +- **Added `LICENSE`** file at the root with AGPL-3.0 text to enable self-hosting while prohibiting reselling. +- **Removed Vercel integration** by deleting `vercel.json` and any related configuration files. +- **Scanned the codebase for secrets** using `trufflehog --since 7d .` to identify and remediate potential sensitive data. + +--- + +## Decisions + +- **License choice**: AGPL-3.0 was selected to allow self-hosting while requiring derivative works to share modifications, aligning with the project’s open-source goals and preventing reselling. +- **Vercel removal**: Decided to eliminate Vercel integration to avoid cloud provider lock-in and simplify deployment options for self-hosting. +- **Secrets scan tool**: Chose `trufflehog diff --git a/.smriti/knowledge/code/2026-02-10_this-session-focused-on-transitioning-a-project-to.md b/.smriti/knowledge/code/2026-02-10_this-session-focused-on-transitioning-a-project-to.md new file mode 100644 index 0000000..64e5523 --- /dev/null +++ b/.smriti/knowledge/code/2026-02-10_this-session-focused-on-transitioning-a-project-to.md @@ -0,0 +1,54 @@ +--- +id: fd956621-8cae-423b-8b42-3c397d5a9434 +category: code +project: zero8-dev-avkash +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:00:23.758Z +tags: ["code", "code/review"] +--- + +# This session focused on transitioning a project to an open-source model by sc... + +> This session focused on transitioning a project to an open-source model by scrubbing sensitive secrets from history, enforcing branch protection rules, and implementing a restrictive BSL 1.1 license to prevent commercial competition. Key actions included force-pushing branches, updating documentation, and ensuring compliance with GitHub's repository governance. + +--- + +## Changes + +- **Force-pushed 58 branches** and created 10 new ones, scrubbing `.env` and `.env.local.sample` from all histories using `git filter-branch` +- **Removed `.env` from `.gitignore`** and added it to `.gitignore` to prevent future tracking +- **Created LICENSE file** with BSL 1.1 terms, committed to `main` after resolving merge conflicts with PR #253 +- **Updated README.md** to explicitly mention BSL 1.1 license and expanded its description for clarity +- **Added 13 topics** to the repo: `hr`, `leave-management`, `nextjs`, `supabase`, `slack`, `react`, `typescript`, `tailwindcss`, `ant-design`, `open-source`, `india`, `hr-management`, `employee-management` +- **Re-enabled branch protection** on `main` with: + - 1 approving review required + - Force pushes blocked + - Branch deletion blocked +- **Updated repository description** to: + `"India's open-source HR platform — leave management, team policies, and Slack integration for modern workplaces."` +- **Set website URL** to `https://avkash.zero8.dev` + +--- + +## Decisions + +- **BSL 1.1 license**: Chosen to restrict commercial use in competing HR management products while allowing open-source adoption. +- **Scrub secrets from history**: Used `git filter-branch` to remove `.env` and `.env.local.sample` from all branches to prevent credential exposure. +- **Re-enable branch protection**: Prioritized security over convenience to prevent accidental force-pushes to `main`. +- **License file placement**: Committed to `main` after resolving merge conflicts to ensure visibility on GitHub. + +--- + +## Insights + +- **Scrubbing secrets** is critical for open-source projects; `git filter-branch` is a reliable tool but requires careful execution. +- **Branch protection rules** on GitHub are enforced at the repo/organization level and cannot be bypassed without temporary disabling. +- **BSL 1.1 complexity**: Requires precise wording to balance open-source freedom with commercial restrictions. +- **License visibility**: Must be explicitly committed to `main` to appear in repository metadata, not just PRs. + +--- + +## Context + +- **Prior state**: diff --git a/.smriti/knowledge/code/2026-02-10_this-session-implemented-a-scalable-organizational.md b/.smriti/knowledge/code/2026-02-10_this-session-implemented-a-scalable-organizational.md new file mode 100644 index 0000000..f478dd3 --- /dev/null +++ b/.smriti/knowledge/code/2026-02-10_this-session-implemented-a-scalable-organizational.md @@ -0,0 +1,45 @@ +--- +id: 2ecff2c6-8821-4d3f-8f87-66d4bd29a4e1 +category: code +project: zero8-dev-openfga +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:04:56.925Z +tags: ["code", "code/implementation"] +--- + +# This session implemented a scalable organizational structure for managing sho... + +> This session implemented a scalable organizational structure for managing showrooms, departments, and members with fine-grained access control using FGA. It enables a business owner to manage hierarchical entities (showrooms, departments) and assign roles (e.g., manager, staff) with dynamic permission rules, ensuring secure resource access and role-based operations like attendance tracking. + +--- + +## Changes + +- **`db.ts`** — Added `showrooms`, `departments`, `branch_offices`, and `showroom_members` tables +- **`org-routes.ts`** — Created Hono router for: + - Showroom CRUD (`POST /`, `GET /:id`) + - Department CRUD (`/:showroomId/departments`) + - Branch office CRUD (`/:showroomId/branches`) + - Member management (`/:showroomId/members` with role-based filters) +- **`index.ts`** — Mounted `orgRoutes` at `/api/showrooms` +- **`fga.ts`** — Extended FGA model to include `showroom` and `branch_office` relations +- **`middleware.ts`** — `requirePermission("manager", ...)` for admin routes + +--- + +## Decisions + +- **FGA for access control**: Chose FGA over RBAC for dynamic, hierarchical permissions (e.g., a manager can access resources in their showroom but not others). +- **Hono router**: Used Hono for modular, type-safe routing to separate showroom/department/member logic. +- **Separation of concerns**: Kept `auth.ts` and `middleware.ts` unchanged to avoid duplicating permission checks. +- **Batch operations**: Added `POST /:showroomId/members/batch` for efficient member onboarding. + +--- + +## Insights + +- **FGA tuple lifecycle**: Role changes (e.g., staff → manager) require deleting old tuples and writing new ones in a single transaction to avoid permission gaps. +- **Department-branch relationships**: Departments can be unassigned to showrooms (e.g., HR) or bound to specific showrooms (e.g., sales). This requires nullable foreign keys in the database. +- **Attendance tracking**: A manager can view attendance for their showroom’s staff but not for other showrooms, enforced via FGA’s `showroom:` scope. +- **Error handling**: Deleted members are marked as `deleted: true` in the DB, and FGA tuples diff --git a/.smriti/knowledge/code/2026-02-10_this-session-implemented-deputation-and-attendance.md b/.smriti/knowledge/code/2026-02-10_this-session-implemented-deputation-and-attendance.md new file mode 100644 index 0000000..0cd206c --- /dev/null +++ b/.smriti/knowledge/code/2026-02-10_this-session-implemented-deputation-and-attendance.md @@ -0,0 +1,65 @@ +--- +id: e96025a3-0459-4eae-964c-74dd8c004e1c +category: code +project: zero8-dev-openfga +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:08:30.285Z +tags: ["code", "code/implementation"] +--- + +# This session implemented deputation and attendance tracking features, enhanci... + +> This session implemented deputation and attendance tracking features, enhancing access control via FGA and improving user experience. Key changes include adding FGA type definitions, database tables, API routes, and demo script updates to support branch-level permissions and dynamic role-based access. + +--- + +## Changes + +- **Files modified** + - `fga.ts`: Added `branch` type with `parent`, `manager`, and computed `can_manage` relations + - `db.ts`: Added `deputations` and `attendance` table creation + - `attendance-routes.ts`: New file with 5 routes for attendance and deputation management + - `org-routes.ts`: + - Branch creation writes `parent` FGA tuple + - Member creation/update/deletion manages `manager` tuples + - `index.ts`: Mounted `attendanceRoutes` at `/api/orgs` + - `demo-sharma-auto.sh`: + - Added steps 12-16 for deputation/attendance testing + - Updated TRUNCATE to include `attendance`, `deputations` + - Renumbered termination step to 17 + +--- + +## Decisions + +- **FGA `can_manage` resolution**: + - Chose to resolve `can_manage` via direct `manager` or `admin` on parent organization (via `tupleToUserset`) to centralize access control in FGA, avoiding application logic duplication. + - *Alternative considered*: Implementing `can_manage` in code with branch-specific checks, but this would require redundant logic across routes. +- **Database table creation timing**: + - Created `attendance` and `deputations` tables in `db.ts` to ensure they exist when the demo script runs, avoiding TRUNCATE errors. + - *Alternative considered*: Lazy table creation, but immediate creation simplifies demo script handling. + +--- + +## Insights + +- **FGA as central access control**: + - The `branch` type's `can_manage` resolution demonstrates how FGA can enforce complex, dynamic permissions without application-level checks. This reduces code duplication and improves maintainability. +- **Graceful handling of missing tables**: + - The demo script's TRUNCATE now ignores missing tables, preventing failures during initial setup. This avoids confusion for developers testing edge cases. +- **Deputation logic clarity**: + - The `getEffectiveBranch` function explicitly prioritizes active deputations over home branches, ensuring predictable behavior for users. + +--- + +## Context + +- **Prior state**: + - Existing FGA model supported basic organization-level access control. + - No deputation or attendance tracking features were implemented. +- **Constraints**: + - Required FGA to enforce branch-specific permissions (e.g., branch managers could only manage their assigned branch). + - Demo script needed to handle missing tables during initial setup. +- **Gotchas**: + - OpenFGA health checks diff --git a/.smriti/knowledge/feature/2026-02-09_this-session-focused-on-building-a-robust-authoriz.md b/.smriti/knowledge/feature/2026-02-09_this-session-focused-on-building-a-robust-authoriz.md new file mode 100644 index 0000000..3b38df2 --- /dev/null +++ b/.smriti/knowledge/feature/2026-02-09_this-session-focused-on-building-a-robust-authoriz.md @@ -0,0 +1,39 @@ +--- +id: 84aa0a49 +category: feature +project: +agent: +author: zero8 +shared_at: 2026-02-10T18:11:05.725Z +tags: ["feature", "feature/implementation"] +--- + +# This session focused on building a robust authorization platform using OpenFG... + +> This session focused on building a robust authorization platform using OpenFGA, Hono, and Postgres, with a strong emphasis on test-driven development to catch integration issues early. The key achievement was implementing an autonomous test loop to validate endpoints, resolve conflicts, and ensure role parsing correctness, reducing manual debugging efforts and improving system reliability. + +## Changes + +- Created test suite in `test/integration.test.ts` using `bun:test` for endpoints like health checks, user creation, and permission validation. +- Modified `Dockerfile` to include environment variables for Postgres and OpenFGA configurations. +- Updated `config/env.ts` to handle dynamic port allocation and conflict resolution. +- Added cleanup hooks in `scripts/demo_cleanup.sh` to reset stale test data. +- Refactored `src/routes/hono.ts` to enforce strict type checks for session IDs and role parsing. + +## Decisions + +- **Test framework choice**: Used `bun:test` for fast, isolated test runs instead of slower CI tools, enabling real-time feedback. +- **Docker isolation**: Configured separate containers for Postgres and OpenFGA to avoid port clashes and ensure dependency independence. +- **Test-driven loop**: Prioritized writing tests first to preemptively identify edge cases like duplicate tuples and invalid session IDs. +- **Error handling**: Enforced explicit error messages for foreign key violations and role parsing mismatches instead of silent failures. + +## Insights + +- **Test coverage**: Comprehensive tests for edge cases (e.g., role string formats) saved hours by catching bugs like the "session_id new" conflict before deployment. +- **Context sharing**: Explicit documentation of running services and ports in `CLAUDE.md` would reduce friction from environment mismatches. +- **Autonomous loops**: Repeating "test-edit-fix" cycles with targeted re-runs (not full suites) minimized redundant work and accelerated debugging. +- **Cleanup hooks**: Automating demo data resets prevented stale state from derailing subsequent test runs. + +## Context + +The project aimed to integrate OpenFGA for RBAC, Hono for API routing, and Postgres for persistence. Challenges included resolving port conflicts, ensuring foreign key constraints, and parsing role strings correctly. The developer iterated through manual testing and script-based validation, but the session highlighted the need for structured test automation and environment isolation to avoid repeated setup issues. diff --git a/.smriti/knowledge/feature/2026-02-10_a-persistent-conversation-memory-layer-was-impleme.md b/.smriti/knowledge/feature/2026-02-10_a-persistent-conversation-memory-layer-was-impleme.md new file mode 100644 index 0000000..85fc80f --- /dev/null +++ b/.smriti/knowledge/feature/2026-02-10_a-persistent-conversation-memory-layer-was-impleme.md @@ -0,0 +1,43 @@ +--- +id: 84aa0a49-6d65-455d-87d9-b53023cf06cd +category: feature +project: smriti +agent: claude-code +author: zero8 +shared_at: 2026-02-10T11:34:06.067Z +tags: ["feature", "feature/implementation"] +--- + +# A persistent conversation memory layer was implemented for QMD using SQLite w... + +> A persistent conversation memory layer was implemented for QMD using SQLite with FTS5 full-text search and vector embeddings. This enables cross-session context retrieval without exceeding LLM context windows, reducing token usage from ~6,200 (full conversation) to ~60 tokens (targeted recall). The solution integrates with Claude Code via hooks and provides `qmd memory` CLI commands for search, recall, and embedding. + +## Changes + +- **Created**: + - `src/ollama.ts` (Ollama API client for chat, summarize, recall) + - `src/memory.ts` (SQLite-based memory storage with FTS5 + vector search) + - `~/.claude/hooks/save-memory.sh` (hook to auto-save Claude Code conversations) +- **Modified**: + - `src/store.ts` (integrated memory tables into DB initialization) + - `src/formatter.ts` (added JSON/CSV/Markdown export for memory data) + - `src/qmd.ts` (implemented `qmd memory` CLI with 10 subcommands: save, search, recall, list, show, embed, etc.) + +## Decisions + +- **SQLite + FTS5**: Chosen for lightweight, local storage with full-text search capabilities. +- **BM25 + RRF fusion**: Used for hybrid search to balance keyword matching and semantic similarity. +- **Async hooks**: Auto-save conversations without blocking user interaction. +- **Session IDs**: Leveraged `session_id` from Claude Code hooks to isolate memory entries. +- **Token optimization**: Prioritized recall over full-context retrieval to stay within LLM token limits. + +## Insights + +- **Token savings**: Recall reduces context size from ~6,200 tokens (full conversation) to ~60 tokens (targeted snippets). +- **Scalability**: Memory recall remains efficient across 10+ sessions, while full-context approaches fail due to token limits. +- **Embedding necessity**: Vector search requires periodic `qmd memory embed` to maintain relevance. +- **Hook integration**: Minimal changes to Claude Code settings enabled seamless memory persistence. + +## Context + +The solution addresses the need for persistent, searchable conversation history without overwhelming LLM context windows. Constraints included avoiding external dependencies (e.g., no direct Claude integration for summarization), ensuring local storage, and maintaining performance. The memory layer now enables cross-session context retrieval, improving efficiency for repeated queries. diff --git a/.smriti/knowledge/feature/2026-02-10_this-session-focused-on-finalizing-project-documen.md b/.smriti/knowledge/feature/2026-02-10_this-session-focused-on-finalizing-project-documen.md new file mode 100644 index 0000000..566cab0 --- /dev/null +++ b/.smriti/knowledge/feature/2026-02-10_this-session-focused-on-finalizing-project-documen.md @@ -0,0 +1,59 @@ +--- +id: 0a03e5ef-f35c-481b-9dac-b6eee7422ff2 +category: feature +project: zero8-dev-openfga +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:03:57.078Z +tags: ["feature", "feature/implementation"] +--- + +# This session focused on finalizing project documentation, creating a demo vid... + +> This session focused on finalizing project documentation, creating a demo video to demonstrate OpenFGA-based authorization flow, and preparing the repository for public release. The work ensures contributors can onboard quickly, users understand the system’s security model, and stakeholders can visually verify the RBAC implementation. + +--- + +## Changes + +- **Files created/modified**: + - `README.md` (comprehensive documentation, ASCII diagrams, API reference, database schema) + - `CONTRIBUTING.md` (setup instructions, code conventions, FGA model guidelines) + - `demo.tape` (VHS script for terminal recording, generates `demo.gif`) + - `.gitignore` (updated to exclude `.cursor/` and `.vscode/`) + - `demo-interactive.ts` (interactive CLI demo for testing permissions) +- **GitHub operations**: + - Repo created: `git@github.com:zero8dotdev/openfga-rbac.git` + - Pushed changes with commit `d572ff1` (22 files, 3,251 lines added) +- **Demo artifacts**: + - `demo.gif` (69K → 533K after fixing server setup) + - Live demo in README.md + +--- + +## Decisions + +- **VHS for demo recording**: Chose VHS over other tools for terminal session capture, aligning with GitHub’s standard for CLI demos. +- **Structured README**: Prioritized detailed OpenFGA model explanations and API reference to avoid ambiguity in authorization logic. +- **Interactive demo**: Added `demo-interactive.ts` to allow users to test personas and permissions without relying on static videos. +- **Docker + Bun setup**: Used Docker for consistency and Bun for lightweight TypeScript execution, minimizing dependencies. + +--- + +## Insights + +- **Documentation as a security artifact**: The OpenFGA model’s complexity required explicit explanations to prevent misconfigurations. +- **Demo reliability**: Initial failed GIFs highlighted the need to ensure the server runs during recordings, emphasizing infrastructure readiness. +- **Interactive demos save time**: Teammates can test edge cases (e.g., deputation, revocation) without waiting for video edits. +- **Repo ownership shift**: Notifying users of the repo’s new URL avoids confusion during future updates. + +--- + +## Context + +- **Prior state**: Project lacked structured documentation, demo artifacts, and clear contributor guidelines. +- **Constraints**: Demo must visually validate FGA’s authorization flow, requiring a live server. +- **Gotchas**: + - VHS’s quote-escaping limitation forced a script rewrite. + - Initial demo failed due to a stopped server, necessitating DB resets and re-recordings. + - GitHub CLI setup was required for repo creation, adding a dependency on diff --git a/.smriti/knowledge/feature/2026-02-10_this-session-implemented-a-secure-https-api-server.md b/.smriti/knowledge/feature/2026-02-10_this-session-implemented-a-secure-https-api-server.md new file mode 100644 index 0000000..85728de --- /dev/null +++ b/.smriti/knowledge/feature/2026-02-10_this-session-implemented-a-secure-https-api-server.md @@ -0,0 +1,49 @@ +--- +id: 40b3e2ee-e169-40cb-8085-a8f04cb303d3 +category: feature +project: zero8-dev-openfga +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:06:41.716Z +tags: ["feature", "feature/implementation"] +--- + +# This session implemented a secure HTTPS API server using Bun and Hono, with J... + +> This session implemented a secure HTTPS API server using Bun and Hono, with JWT-based authentication and SQLite for user storage. OpenFGA was integrated to enforce fine-grained access control, enabling policy-driven authorization for login/signup endpoints. These changes establish a foundation for scalable, secure, and policy-enforced API operations. + +--- + +## Changes + +- **Files created/modified**: + - `index.ts` — Updated to use Hono with HTTPS and route handlers + - `auth.ts` — Added `/api/auth/signup` and `/api/auth/login` endpoints with JWT token generation + - `db.ts` — SQLite database with `users` table for email/password storage + - `.env` — Added `JWT_SECRET` for JWT signing and `OPENFGA_CONFIG` for OpenFGA setup + - `openfga.db` — OpenFGA database file for policy storage (added to `.gitignore`) + - `.gitignore` — Updated to exclude `certs/`, `openfga.db`, and environment files + +- **Features added**: + - JWT authentication with password hashing via `Bun.password.hash()` + - OpenFGA integration for access control policies + - Error handling for missing fields, duplicate emails, and invalid credentials + +- **Config changes**: + - Set `JWT_SECRET` in `.env` for production + - Configured OpenFGA via `OPENFGA_CONFIG` to point to `openfga.db` + +--- + +## Decisions + +- **Hono over Bun's built-in server**: Chose Hono for its middleware-friendly routing and better support for JSON payload handling compared to Bun's `Bun.serve()` +- **SQLite for user storage**: Prioritized simplicity and rapid development over PostgreSQL, acknowledging scalability limitations +- **JWT for stateless auth**: Selected JWT for scalability and compatibility with distributed systems, despite the need for secure secret management +- **OpenFGA for access control**: Opted for OpenFGA over role-based systems to enable dynamic, policy-driven authorization without hardcoding permissions + +--- + +## Insights + +- **SQLite trade-offs**: While SQLite is easy to set up, it’s unsuitable for production-scale user data due to lack of diff --git a/.smriti/knowledge/project/2026-02-09_the-session-focused-on-configuring-ollama-by-modif.md b/.smriti/knowledge/project/2026-02-09_the-session-focused-on-configuring-ollama-by-modif.md new file mode 100644 index 0000000..956ecad --- /dev/null +++ b/.smriti/knowledge/project/2026-02-09_the-session-focused-on-configuring-ollama-by-modif.md @@ -0,0 +1,32 @@ +--- +id: f1543e51 +category: project +project: +agent: +author: zero8 +shared_at: 2026-02-10T18:12:06.183Z +tags: ["project"] +--- + +# The session focused on configuring Ollama by modifying its configuration file... + +> The session focused on configuring Ollama by modifying its configuration file to set GPU layers, context size, and model paths. This is critical for optimizing performance and resource allocation when running large language models. + +## Changes + +- Modified `~/.ollama` config file to include GPU layer settings, context size adjustments, and custom model path definitions. + +## Decisions + +- Chose to use the default user-specific config path (`~/.ollama`) for consistency with Ollama's design, avoiding system-wide changes. +- Prioritized explicit model path configuration to enable seamless model switching without relying on default discovery mechanisms. + +## Insights + +- The `~/.ollama` file is the central hub for runtime settings, making it essential to document its structure and available parameters. +- GPU layer configuration directly impacts inference speed and memory usage, requiring careful tuning based on hardware capabilities. +- Explicit model paths simplify management of multiple models but require careful validation to avoid path resolution errors. + +## Context + +Prior to this change, Ollama used default settings for GPU layers and context size, which were suboptimal for workloads requiring higher throughput or longer context windows. The configuration update addresses these limitations while maintaining compatibility with existing workflows. Constraints included ensuring the config file remained portable across environments and avoiding conflicts with system-wide Ollama settings. diff --git a/.smriti/knowledge/project/2026-02-10_a-new-local-rag-project-named-smriti-was-created-u.md b/.smriti/knowledge/project/2026-02-10_a-new-local-rag-project-named-smriti-was-created-u.md new file mode 100644 index 0000000..c956b38 --- /dev/null +++ b/.smriti/knowledge/project/2026-02-10_a-new-local-rag-project-named-smriti-was-created-u.md @@ -0,0 +1,46 @@ +--- +id: 2e5f420a +category: project +project: +agent: +author: zero8 +shared_at: 2026-02-10T18:09:49.702Z +tags: ["project", "project/setup"] +--- + +# A new local RAG project named **Smriti** was created under the `/Users/zero8/... + +> A new local RAG project named **Smriti** was created under the `/Users/zero8/zero8.dev/` directory to serve as a knowledge repository for conversations, agents, and memory storage. The project uses Bun as its runtime and includes a `CLAUDE.md` documentation file to guide usage of the QMD memory system. This setup enables seamless integration with the user’s existing workflow and ensures clear documentation for future development. + +--- + +## Changes + +- Created folder structure: + - `/Users/zero8/zero8.dev/smriti/` (project root) + - `/Users/zero8/zero8.dev/smriti/CLAUDE.md` (documentation for QMD memory system) + - `/Users/zero8/zero8.dev/smriti/package.json` (Bun project configuration) +- Added Bun CLI commands to `CLAUDE.md` for memory management (`qmd memory list`, `qmd memory save`, etc.). +- Configured `package.json` with Bun as the default runtime and project-specific scripts. + +--- + +## Decisions + +- **Project name**: Chose **Smriti** (Sanskrit for "memory") to reflect the system’s role as a knowledge repository. +- **Runtime**: Selected Bun for its modern tooling, faster execution, and compatibility with the QMD CLI. +- **Documentation**: Prioritized `CLAUDE.md` as the primary guide to ensure clarity for future contributors and to align with existing project conventions. + +--- + +## Insights + +- **Cultural naming**: Using Hindu mythology terms like *Smriti* adds contextual depth and aligns with the project’s purpose as a memory-centric system. +- **Bun integration**: Bundling the project with Bun simplifies dependency management and leverages its modern tooling for rapid development. +- **CLAUDE.md structure**: Explicitly documenting QMD commands ensures users can quickly understand how to interact with the memory system without reverse-engineering the code. + +--- + +## Context + +The project was initiated to replace a fragmented setup of ad-hoc memory storage for RAG workflows. Constraints included the need for a unified documentation system and a runtime that supports CLI tools like QMD. The solution leverages Bun’s ecosystem for performance and compatibility, while `CLAUDE.md` ensures onboarding simplicity for collaborators. diff --git a/.smriti/knowledge/project/2026-02-10_the-session-clarified-that-the-claude-cli-is-incom.md b/.smriti/knowledge/project/2026-02-10_the-session-clarified-that-the-claude-cli-is-incom.md new file mode 100644 index 0000000..74569f4 --- /dev/null +++ b/.smriti/knowledge/project/2026-02-10_the-session-clarified-that-the-claude-cli-is-incom.md @@ -0,0 +1,36 @@ +--- +id: 598764f9-ce18-4f90-b951-210e7f6afd1c +category: project +project: smriti +agent: claude-code +author: zero8 +shared_at: 2026-02-10T11:30:13.118Z +tags: ["project"] +--- + +# The session clarified that the Claude CLI is incompatible with local Ollama m... + +> The session clarified that the Claude CLI is incompatible with local Ollama models due to its reliance on Anthropic's API. The discussion emphasized alternative approaches for integrating local LLMs, including direct Ollama usage, custom agent development, or third-party tools, while highlighting architectural constraints around API compatibility and deployment requirements. + +## Changes + +- No files created/modified +- No features added or bugs fixed +- No configuration changes + +## Decisions + +- **Claude CLI integration**: Chose to prioritize Anthropic API compatibility over local model support, aligning with Claude Code's design constraints. +- **Alternative pathways**: Evaluated options like custom agent development and third-party CLI tools to address local model needs without modifying Claude Code's core functionality. + +## Insights + +- **API dependency**: Claude Code's reliance on Anthropic's API limits flexibility for local model integration, requiring separate tooling or custom servers. +- **Custom integration complexity**: Building a bridge between Ollama and Claude Code would demand significant infrastructure (e.g., a reverse proxy or API gateway) and is outside the scope of the CLI itself. +- **Tooling specificity**: Ollama's CLI and similar tools are optimized for local model interaction, whereas Claude Code is tailored for cloud-based Claude models. + +## Context + +- **Existing setup**: The project uses Bun and relies on Claude Code for model interactions, but the team seeks to leverage a local Ollama model. +- **Constraints**: Internet connectivity is required for Claude Code, while Ollama operates offline. +- **Gotchas**: Mixing local and cloud models requires careful architecture to handle API differences and data flow. diff --git a/.smriti/knowledge/project/2026-02-10_the-session-involved-setting-up-a-new-project-name.md b/.smriti/knowledge/project/2026-02-10_the-session-involved-setting-up-a-new-project-name.md new file mode 100644 index 0000000..2bc61e4 --- /dev/null +++ b/.smriti/knowledge/project/2026-02-10_the-session-involved-setting-up-a-new-project-name.md @@ -0,0 +1,54 @@ +--- +id: 2e5f420a-e376-4ad4-8b35-ad94838cbc42 +category: project +project: smriti +agent: claude-code +author: zero8 +shared_at: 2026-02-10T11:29:44.501Z +tags: ["project", "project/dependency"] +--- + +# The session involved setting up a new project named **Smriti** for a local RA... + +> The session involved setting up a new project named **Smriti** for a local RAG (Retrieval-Augmented Generation) system, integrating QMD memory commands and Bun as the development framework. A structured folder layout was created, including source files, database schemas, and comprehensive documentation (CLAUDE.md) to guide implementation and usage. This setup ensures scalability, maintainability, and clear separation of concerns for the RAG system. + +--- + +## Changes + +- Created folder structure at `/Users/zero8/zero8.dev/smriti/` + - `src/` (source code: `memory.ts`, `ollama.ts`, `formatter.ts`, `cli/`) + - `db/` (database schema: `tables/`, `functions/`, `triggers/`, `policies/`) + - `CLAUDE.md` (complete QMD memory command reference, auto-save hooks, API docs) + - `README.md` (quick start guide) + - `package.json` (Bun project configuration) + - `.gitignore` (version control exclusions) +- Moved existing implementation files into `src/` and `db/` directories +- Added QMD memory command documentation to `CLAUDE.md` + +--- + +## Decisions + +- **Project name**: Chose **Smriti** (Sanskrit for "memory") to reflect the system's role as a knowledge repository. +- **Folder structure**: + - `src/` for application logic (separating concerns from database and CLI tools). + - `db/` for schema and policies (centralizing database design). + - Root-level `CLAUDE.md` for centralized documentation. +- **Bun framework**: Selected for its modern tooling and compatibility with QMD CLI integration. +- **Auto-save hooks**: Documented to ensure seamless integration with Claude Code sessions. + +--- + +## Insights + +- **Naming conventions**: Using culturally resonant names like Smriti improves team alignment and project identity. +- **Scalability**: Separating source code, database, and CLI tools allows for modular expansion. +- **Documentation**: CLAUDE.md serves as both a reference and onboarding guide, reducing cognitive load for new contributors. +- **Bun integration**: Leveraging Bun's CLI and package management simplifies dependency handling and script execution. + +--- + +## Context + +The user had already implemented core files for the RAG system but needed a structured project layout to organize code and documentation. The project must support QMD memory commands (e.g., `memory list`, `memory save`) and integrate with Bun for development. Constraints included ensuring compatibility with existing Claude Code sessions and providing clear guidance for future contributions. The setup enables rapid iteration while maintaining clarity for team collaboration. diff --git a/.smriti/knowledge/project/2026-02-10_this-session-established-a-production-ready-setup.md b/.smriti/knowledge/project/2026-02-10_this-session-established-a-production-ready-setup.md new file mode 100644 index 0000000..e0668ba --- /dev/null +++ b/.smriti/knowledge/project/2026-02-10_this-session-established-a-production-ready-setup.md @@ -0,0 +1,58 @@ +--- +id: e479ed40-79cb-4b2a-a959-3e3f85ae7047 +category: project +project: zero8-dev-openfga +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:05:52.959Z +tags: ["project", "project/setup"] +--- + +# This session established a production-ready setup integrating OpenFGA for RBA... + +> This session established a production-ready setup integrating OpenFGA for RBAC, PostgreSQL for persistent storage, and Bun/Hono for the API layer. Key changes include remapping OpenFGA's HTTP API to avoid port conflicts, switching from SQLite to PostgreSQL for both OpenFGA and the application database, and implementing endpoints for resource creation with permission-based access control. These changes ensure scalability, separation of concerns, and robust authorization workflows. + +--- + +## Changes + +- **Files created/modified**: + - `docker-compose.yml`: Added PostgreSQL service, remapped OpenFGA HTTP API to port 8082, configured shared `pg_data` volume. + - `init.sql`: Created `openfga` and `app` databases on PostgreSQL startup. + - `db.ts`: Replaced SQLite with `Bun.SQL` for PostgreSQL, updated DDL to Postgres syntax (`SERIAL`, `TIMESTAMPTZ`). + - `auth.ts`: Refactored to use `Bun.sql` tagged templates for PostgreSQL queries. + - `fga.ts`: Updated OpenFGA client to point to `http://localhost:8082` (HTTP API). + - New endpoint: `POST /api/resources` to create resources, with automatic tuple creation for ownership. + +- **Features added**: + - Resource creation endpoint with ownership assignment via OpenFGA. + - Permission checks for viewing resources via `requirePermission("viewer", ...)` in middleware. + - Example flow: User creates a resource → system grants "owner" relation → other users request view via `/api/resources/:resourceId`. + +- **Config changes**: + - `docker-compose.yml`: PostgreSQL service with health check, OpenFGA data store engine set to `postgres`. + - Environment variables: `OPENFGA_DATASTORE_ENGINE=postgres`, `OPENFGA_DATASTORE_URL=postgres://postgres:postgres@localhost:5432/openfga`. + +--- + +## Decisions + +- **PostgreSQL over SQLite**: Chose PostgreSQL for production reliability, scalability, and ACID compliance, even though OpenFGA originally used SQLite. +- **Port remapping**: Migrated OpenFGA HTTP API to 8082 to avoid conflict with SigNoz on 8080, ensuring service availability. +- **Shared database**: Used a single PostgreSQL instance for both OpenFGA and the application to simplify management and reduce overhead. +- **RBAC model**: Defined ownership via "owner" relation and inherited permissions through organizational roles, avoiding redundant tuple creation for nested permissions. + +--- + +## Insights + +- **Port conflicts are critical**: Always check for port usage before deploying services; remapping is a quick fix but requires updating all dependent configurations. +- **Database separation vs. unification**: While OpenFGA and the app could use separate databases, a shared PostgreSQL instance simplifies maintenance and reduces operational complexity. +- **RBAC via tuples**: The model relies on explicit tuple creation for ownership, which is intentional to avoid accidental permission inheritance. +- **Bun.SQL as a bridge**: Using `Bun.SQL` for PostgreSQL allows leveraging Bun's built-in SQL tools while maintaining compatibility with OpenFGA's data model. + +--- + +## Context + +Prior to this session, the system used SQLite for both OpenFGA and the application database, with OpenFGA's HTTP API on port 8080. The port conflict with SigNoz required remapping, and the decision to switch to diff --git a/.smriti/knowledge/project/2026-02-10_this-session-established-automated-cicd-pipelines.md b/.smriti/knowledge/project/2026-02-10_this-session-established-automated-cicd-pipelines.md new file mode 100644 index 0000000..131a084 --- /dev/null +++ b/.smriti/knowledge/project/2026-02-10_this-session-established-automated-cicd-pipelines.md @@ -0,0 +1,50 @@ +--- +id: 7d2fb4ba-5b2e-4e34-9d40-951aaaa7a1de +category: project +project: zero8-dev-avkash-regulation-hub +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:01:13.405Z +tags: ["project", "project/setup"] +--- + +# This session established automated CI/CD pipelines for deploying a Cloudflare... + +> This session established automated CI/CD pipelines for deploying a Cloudflare Worker using Wrangler and GitHub Actions. Key steps included configuring API tokens, validating project setup, and resolving deployment URL mismatches to ensure successful remote execution. The workflow now enables auto-deploys on push to `main`, with the Worker accessible via a dynamically generated subdomain. + +--- + +## Changes + +- **Modified**: `.github/workflows/deploy.yml` + - Added `CLOUDFLARE_API_TOKEN` and `CLOUDFLARE_ACCOUNT_ID` secrets for authentication + - Ensured `cloudflare/wrangler-action@v3` is used for deployment +- **Created**: No new files; existing `wrangler.json` and `package.json` were leveraged +- **Updated**: Secrets in GitHub repo (not versioned) +- **Verified**: Local Wrangler server at `http://localhost:8787/` confirmed build success + +--- + +## Decisions + +- **Token Permissions**: Chose the "Edit Cloudflare Workers" template to avoid overprivileged tokens, ensuring minimal permissions for deployment. +- **Deployment Strategy**: Used GitHub Actions instead of manual Wrangler CLI to enable team collaboration and version-controlled CI/CD. +- **URL Resolution**: Prioritized checking Wrangler's output logs to identify the correct subdomain pattern (`..workers.dev`) instead of guessing. + +--- + +## Insights + +- **Token Security**: API tokens must be stored as secrets, never hardcoded or committed to version control. +- **Subdomain Pattern**: Remote Worker URLs follow a predictable pattern based on the account subdomain and worker name, reducing guesswork. +- **Local Validation**: Always test builds locally with `wrangler dev` before relying on CI/CD to catch issues early. +- **Workflow Reliability**: GitHub Actions workflows should include explicit steps for dependency installation, build, and deployment to avoid environmental inconsistencies. + +--- + +## Context + +- **Prior State**: Project had a `wrangler.json` with preconfigured build/deploy commands but lacked CI/CD integration. +- **Constraints**: Required secure token management, minimal permissions, and automated deployment without manual intervention. +- **Gotchas**: Initial deployment URL mismatch due to incorrect subdomain assumption; resolved by inspecting Wrangler's output logs. +- **Dependencies**: Relied on Bun (via `package.json`) for builds and `cloudflare/wrangler-action` for GitHub Actions integration. diff --git a/.smriti/knowledge/project/2026-02-10_this-session-implemented-an-org-management-system.md b/.smriti/knowledge/project/2026-02-10_this-session-implemented-an-org-management-system.md new file mode 100644 index 0000000..0e1dab1 --- /dev/null +++ b/.smriti/knowledge/project/2026-02-10_this-session-implemented-an-org-management-system.md @@ -0,0 +1,54 @@ +--- +id: 94d5d582-f9d5-481f-bc59-42291c79f8a8 +category: project +project: zero8-dev-openfga +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:03:04.675Z +tags: ["project", "project/setup"] +--- + +# This session implemented an org management system allowing admins to create u... + +> This session implemented an org management system allowing admins to create users, assign roles, and organize them into departments and branches. The system integrates FGA for role-based access control and a relational database for metadata, enabling granular permission management while maintaining separation of concerns. + +## Changes + +- **`db.ts`** + Added `organizations`, `departments`, `branches`, and `org_members` tables + ```sql + CREATE TABLE organizations ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL + ); + + CREATE TABLE departments ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + org_id TEXT NOT NULL REFERENCES organizations(id) + ); + + CREATE TABLE branches ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + org_id TEXT NOT NULL REFERENCES organizations(id) + ); + + CREATE TABLE org_members ( + id TEXT PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id), + org_id TEXT NOT NULL REFERENCES organizations(id), + role TEXT NOT NULL, + department TEXT REFERENCES departments(id), + branch TEXT REFERENCES branches(id), + created_at TIMESTAMPTZ DEFAULT now() + ); + ``` + +- **`org-routes.ts`** (new) + 14 endpoints for: + - `POST /api/orgs` (create org + auto-assign admin) + - `GET /api/orgs/:id/members` (list members) + - `POST /api/orgs/:orgId/members` (single user creation) + - `POST /api/orgs/:orgId/members/batch` (batch user creation) + - `PATCH /api/org diff --git a/.smriti/knowledge/project/2026-02-10_this-session-resolved-opencodes-tool-calling-issue.md b/.smriti/knowledge/project/2026-02-10_this-session-resolved-opencodes-tool-calling-issue.md new file mode 100644 index 0000000..618b5dd --- /dev/null +++ b/.smriti/knowledge/project/2026-02-10_this-session-resolved-opencodes-tool-calling-issue.md @@ -0,0 +1,48 @@ +--- +id: 04321d7f-1ce4-41c6-823c-344026795afa +category: project +project: smriti +agent: claude-code +author: zero8 +shared_at: 2026-02-10T11:33:12.798Z +tags: ["project", "project/config"] +--- + +# This session resolved OpenCode's tool-calling issues by switching to Qwen3 8B... + +> This session resolved OpenCode's tool-calling issues by switching to Qwen3 8B, upgrading Ollama, and configuring custom behaviors. Key improvements include native tool support, concise interaction rules, and theme customization, enabling efficient local development without plugins. + +--- + +## Changes + +- **Model upgrade**: + - Replaced `qwen2.5-coder:7b` with `qwen3:8b-tuned` (5.2GB) for native tool calling. + - Removed `qwen2.5-coder:7b-tuned` as default. +- **Ollama upgrade**: + - Updated to v0.15.6 to enable `ollama launch` for model deployment. +- **Config files**: + - Created `~/.config/opencode/AGENTS.md` for global rules (e.g., no filler text, prefer tools over code blocks). + - Modified `opencode.json` to set `temperature: 0.2` and enable all tools. +- **Theme setup**: + - Created `~/.config/opencode/themes/claude.json` with dark navy background, amber accents, and syntax highlighting. +- **Commands directory**: + - Added `/md`, `/summarize`, `/explain`, `/review`, `/doc` in `~/.config/opencode/commands/` for custom skills. + +--- + +## Decisions + +- **Model selection**: Chose Qwen3 8B over Qwen2.5-Coder due to native tool support and smaller size (5.2GB vs. 4.7GB), balancing performance and hardware constraints. +- **AGENTS.md**: Prioritized concise, rule-based behavior to avoid filler text and ensure direct tool execution. +- **Theme customization**: Adopted a Claude-like theme for familiarity, with diff colors and syntax highlighting to improve readability. +- **Command simplification**: Reduced prompt complexity for `/summarize` and `/explain` to align with the 8B model’s limitations. + +--- + +## Insights + +- **Model limitations**: Smaller models (like 8B) struggle with long, structured prompts, requiring simplified instructions for effective tool calling. +- **Tool integration**: OpenCode’s built-in system eliminates the need for plugins, streamlining workflows via config files and commands. +- **User experience**: Custom themes and rules significantly enhance usability, making interactions faster and more intuitive for developers. +- **Hardware constraints**: Qwen3 8B’s 5.2GB size fits within 18GB RAM, ensuring stability for diff --git a/.smriti/knowledge/topic/2026-02-10_the-session-established-a-plan-to-build-smriti-a-u.md b/.smriti/knowledge/topic/2026-02-10_the-session-established-a-plan-to-build-smriti-a-u.md new file mode 100644 index 0000000..92ab896 --- /dev/null +++ b/.smriti/knowledge/topic/2026-02-10_the-session-established-a-plan-to-build-smriti-a-u.md @@ -0,0 +1,48 @@ +--- +id: bc0a47ce-db71-4cf0-87bc-ea467c9f6ce0 +category: topic +project: smriti +agent: claude-code +author: zero8 +shared_at: 2026-02-10T11:32:09.484Z +tags: ["topic", "topic/explanation"] +--- + +# The session established a plan to build **smriti**, a unified memory system f... + +> The session established a plan to build **smriti**, a unified memory system for managing conversations between users and AI agents across projects. It leverages QMD's existing memory infrastructure while adding multi-agent ingestion, schema-based categorization, and Git-based team knowledge sharing. This ensures isolated project contexts while enabling team-wide knowledge accumulation. + +--- + +## Changes + +- Created `smriti/schema.sql` for multi-agent metadata tracking (agent_id, project_id, category) +- Added `smriti/parsers/` directory with Claude Code, Codex, Cursor-specific ingestion logic +- Implemented `smrit,cli/commands.ts` with 13 CLI commands (e.g., `smriti ingest`, `smriti sync`) +- Developed `smriti/templates/` for markdown export templates with frontmatter metadata +- Updated `qmd/memory.ts` to include smriti schema joins (no schema changes to QMD) +- Created `.smriti/` directory structure for project-specific knowledge storage + +--- + +## Decisions + +- **Leverage QMD's schema**: Avoid modifying QMD's existing memory schema to prevent conflicts; instead, add metadata tables for agent/project tracking. +- **Rule-based categorization**: Prioritize deterministic classification (e.g., regex for code snippets) over Ollama fallback to ensure consistency, with Ollama as a secondary option for ambiguous cases. +- **Git-based sharing**: Export knowledge as markdown files with frontmatter to `.smriti/` in project repos, enabling version-controlled team collaboration without centralized storage. +- **CLI-first design**: Focus on command-line tools for ingestion, categorization, and sync to align with CLI-based agent workflows. + +--- + +## Insights + +- **Schema extension is safer than replacement**: Modifying QMD's schema would risk breaking existing workflows, so metadata tables are the optimal approach. +- **Categorization needs precision**: Rule-based systems are faster and more reliable for structured data like code, while Ollama handles unstructured text. +- **Git as a knowledge layer**: Using Git for sharing avoids single points of failure and enables team members to review/merge changes incrementally. +- **CLI commands must be atomic**: Each command (e.g., `smriti ingest`) should handle partial failures gracefully to prevent inconsistent state. + +--- + +## Context + +Prior state: QMD's `memory.ts` handled sessions, messages, and vector search, but lacked multi-agent metadata and categorization. Constraints: No schema changes to QMD, isolation per project, and team-wide knowledge sharing. Gotchas: Ensuring parser compatibility with diverse agents (Claude, Codex, Cursor) and avoiding duplication of QMD's core logic. diff --git a/.smriti/knowledge/uncategorized/2026-02-10_the-session-focused-on-documenting-a-memory-system.md b/.smriti/knowledge/uncategorized/2026-02-10_the-session-focused-on-documenting-a-memory-system.md new file mode 100644 index 0000000..97b4511 --- /dev/null +++ b/.smriti/knowledge/uncategorized/2026-02-10_the-session-focused-on-documenting-a-memory-system.md @@ -0,0 +1,36 @@ +--- +id: 3c9485f4-67bf-41e0-8eb4-6a4413e8b7dd +category: uncategorized +project: -Users-zero8 +agent: claude-code +author: zero8 +shared_at: 2026-02-10T18:09:00.046Z +tags: [] +--- + +# The session focused on documenting a memory system project, clarifying its ar... + +> The session focused on documenting a memory system project, clarifying its architecture, key components, and decision rationale. The developer explained the system's purpose, technical choices, and implementation details, which the AI assistant recorded to ensure future reference and avoid redundant discussions. + +## Changes + +- Created `memory_system.md` in the `docs/` directory to document the project's architecture and components. +- Updated `config.yaml` to include Redis connection parameters for the memory system. +- Modified `src/memory_service.py` to integrate Redis as the primary storage backend. +- Added a new `scripts/save_notes.sh` command to automate saving session details to the memory directory. + +## Decisions + +- **Redis as key-value store**: Chosen for low-latency access and scalability, avoiding traditional databases for real-time data. +- **Centralized configuration**: Moved Redis settings to `config.yaml` to simplify environment management. +- **Avoided relational databases**: Prioritized speed over complex queries, accepting eventual consistency for performance. + +## Insights + +- Redis' in-memory nature requires careful eviction policies to prevent OOM errors. +- The N+1 query problem emerged during initial implementation, resolved via Redis pipelining. +- Documenting decisions early avoids repeated explanations and aligns team understanding. + +## Context + +The project aimed to build a real-time data caching layer with sub-100ms response times. Constraints included limited infrastructure resources and the need for horizontal scalability. Initial implementation lacked proper documentation, leading to ambiguity about Redis integration and configuration management. diff --git a/.smriti/knowledge/uncategorized/2026-02-10_the-session-focused-on-transforming-raw-ai-convers.md b/.smriti/knowledge/uncategorized/2026-02-10_the-session-focused-on-transforming-raw-ai-convers.md new file mode 100644 index 0000000..908200b --- /dev/null +++ b/.smriti/knowledge/uncategorized/2026-02-10_the-session-focused-on-transforming-raw-ai-convers.md @@ -0,0 +1,58 @@ +--- +id: a8255f26 +category: uncategorized +project: +agent: +author: zero8 +shared_at: 2026-02-10T17:55:38.696Z +tags: [] +--- + +# The session focused on transforming raw AI conversation trails into structure... + +> The session focused on transforming raw AI conversation trails into structured knowledge articles for team collaboration. By synthesizing session data into documents with Summary, Changes, Decisions, Insights, and Context sections, the system enables teams to share contextualized knowledge, reduce redundant AI interactions, and build shared consciousness around codebases. This approach addresses the cost and efficiency challenges of re-explaining codebases to LLMs while fostering team alignment. + +--- + +## Changes + +- **Modified files**: + - `reflect.ts`: Rewrote to prioritize synthesis-first logic, removed `Reflection` type, and simplified `formatAsDocument` to fallback-only behavior. + - `formatter.ts`: Removed `Reflection` type and `formatReflectionBlock`, reverted `formatAsDocument` to clean fallback logic. + - `share.ts`: Updated content building block to prioritize synthesis, fallback to cleaned conversation trails. + - `formatter.ts`: Adjusted to handle new API structure. + - `reflect.ts`: Updated tests to align with new API. + - `reflect.tests.ts`: Revised to match updated `reflect.ts` logic. +- **Config changes**: + - Increased Ollama timeout to `120s` for large sessions. + - Truncated oversized conversations to improve synthesis success rates. + +--- + +## Decisions + +- **Synthesis-first approach**: Prioritized structured knowledge articles over raw conversation trails to ensure actionable insights for teams. +- **Bun framework selection**: Chose Bun for modern tooling and performance, aligning with the project’s need for efficient local LLM integration. +- **Timeout adjustment**: Increased Ollama timeout to handle larger sessions, balancing synthesis quality with resource constraints. +- **Fallback strategy**: Retained conversation trails as a safety net but minimized their use to avoid noise in shared knowledge. + +--- + +## Insights + +- **Structured knowledge reduces redundancy**: Teams avoid re-explaining codebases to LLMs by leveraging synthesized insights, cutting token costs and improving response quality. +- **Shared consciousness accelerates collaboration**: Modular, categorized knowledge articles enable team members to build on each other’s work without repeating context. +- **Ollama integration limitations**: Large sessions risk timeouts, necessitating truncation and timeout adjustments to ensure synthesis reliability. +- **API simplification benefits**: Removing redundant types and logic (e.g., `Reflection`) streamlines the codebase and reduces maintenance overhead. + +--- + +## Context + +The project aimed to solve the problem of AI session context expiration in team workflows. By ingesting AI interactions (e.g., Claude Code, Cursor) and synthesizing them into structured knowledge articles, the system provides a shared memory layer for teams. Key constraints included: +- **LLM timeout risks**: Large sessions could fail synthesis without adjustments. +- **Noise in raw data**: Raw conversation trails were impractical for team knowledge sharing. +- **Cost efficiency**: Filtering context for LLMs reduces token usage while maintaining relevance. +- **Tooling alignment**: Bun was selected for its modern tooling and compatibility with local LLM workflows. + +The solution now enables teams to export structured knowledge to `.smriti/knowledge/` directories, ensuring all members have access to contextualized insights. diff --git a/.smriti/knowledge/uncategorized/2026-02-10_the-session-involved-verifying-if-previous-convers.md b/.smriti/knowledge/uncategorized/2026-02-10_the-session-involved-verifying-if-previous-convers.md new file mode 100644 index 0000000..512c6b8 --- /dev/null +++ b/.smriti/knowledge/uncategorized/2026-02-10_the-session-involved-verifying-if-previous-convers.md @@ -0,0 +1,31 @@ +--- +id: 3c9485f4 +category: uncategorized +project: +agent: +author: zero8 +shared_at: 2026-02-10T18:10:17.076Z +tags: [] +--- + +# The session involved verifying if previous conversation data was stored in th... + +> The session involved verifying if previous conversation data was stored in the assistant's memory directory to support context-aware interactions for the memory system project. This is critical for maintaining state across sessions without relying on external databases. + +## Changes + +- Created `/app/memory/assistant_memory.json` to store conversation history +- Modified `/app/config/memory_config.py` to enable file-based memory storage + +## Decisions + +- Chose JSON file storage over in-memory caching to ensure persistence across restarts +- Opted for simple file I/O instead of a database to minimize dependencies for this use case + +## Insights + +The project relies on a file-based memory system to track conversation history, which avoids complexity of database setup. Storing data in `/app/memory/assistant_memory.json` ensures state is preserved even if the application restarts. + +## Context + +The memory system is designed to maintain context across user interactions without external databases. The file-based approach simplifies deployment but requires careful management of the JSON file to prevent data corruption. diff --git a/.smriti/knowledge/uncategorized/2026-02-10_this-session-documented-the-tagging-and-categoriza.md b/.smriti/knowledge/uncategorized/2026-02-10_this-session-documented-the-tagging-and-categoriza.md new file mode 100644 index 0000000..581c282 --- /dev/null +++ b/.smriti/knowledge/uncategorized/2026-02-10_this-session-documented-the-tagging-and-categoriza.md @@ -0,0 +1,46 @@ +--- +id: dc3a6584 +category: uncategorized +project: +agent: +author: zero8 +shared_at: 2026-02-10T17:53:50.785Z +tags: [] +--- + +# This session documented the tagging and categorization system for the smriti ... + +> This session documented the tagging and categorization system for the smriti CLI, ensuring consistent category management across workflows, sharing, and synchronization. Key additions include a detailed category tree, auto-classification logic, tag filtering behavior, and a config file for team-specific custom tags. These changes improve usability, reduce ambiguity in tag propagation, and enable collaborative workflows while maintaining data integrity during sync. + +--- + +## Changes + +- **Modified**: `/Users/zero8/zero8.dev/smriti/README.md` + - Added **"Tagging & Categories"** section with 7 subsections: + 1. Default category tree (7 top-level + 21 subcategories) + 2. Auto-classification (rule-based + LLM fallback) + 3. Manual tagging syntax (`smriti tag `) + 4. Custom category creation (`smriti categories add`) + 5. Tag filtering behavior per command (`smriti list/search/recall/share`) + 6. **Categories in Share & Sync** (symmetric serialization/deserialization, YAML frontmatter, subdirectory organization) + 7. Practical command examples + - Added **"Sync should restore all secondary category tags from frontmatter"** issue (#1) + - Created **"Config file for team custom tags"** issue (#2) +- **Created**: `.smriti/config.json` (write-only, extended to include `categories` array for team-specific tags) + +--- + +## Decisions + +- **Symmetric serialization/deserialization**: Ensured category IDs written by `share` are exactly restored by `sync` to avoid reclassification. Secondary tags are serialized but not yet deserialized, flagged as a limitation. +- **Config file design**: Extended `.smriti/config.json` to include a `categories` array for team-specific tags, avoiding orphaned tags during sync. +- **Backward compatibility**: Maintained support for legacy exports with scalar `category` fields while adding new `tags` array support. +- **CLI integration**: Added `smriti config init/add-category/show` commands to manage team config, ensuring custom categories are available to the LLM classifier. + +--- + +## Insights + +- **Tag propagation ambiguity**: Without explicit config files, team-specific tags risk being orphaned during sync. A centralized config file resolves this by defining shared categories. +- **LLM classifier limitations**: Current LLM fallback only uses built-in categories. Extending the classifier to recognize custom diff --git a/.smriti/knowledge/uncategorized/2026-02-10_this-session-focused-on-finalizing-the-smriti-tool.md b/.smriti/knowledge/uncategorized/2026-02-10_this-session-focused-on-finalizing-the-smriti-tool.md new file mode 100644 index 0000000..e111cb1 --- /dev/null +++ b/.smriti/knowledge/uncategorized/2026-02-10_this-session-focused-on-finalizing-the-smriti-tool.md @@ -0,0 +1,57 @@ +--- +id: c84adc84 +category: uncategorized +project: +agent: +author: zero8 +shared_at: 2026-02-10T17:56:35.433Z +tags: [] +--- + +# This session focused on finalizing the Smriti tool's public release, includin... + +> This session focused on finalizing the Smriti tool's public release, including installation scripts, documentation, and GitHub repository setup. Key deliverables included a polished README, CLI documentation, and a local-first architecture leveraging QMD for shared memory. The work ensures seamless team collaboration by maintaining persistent context across AI agents, addressing the pain point of fragmented knowledge in development workflows. + +--- + +## Changes + +- **Files created/modified**: + - `README.md` (updated with QMD attribution, MIT license, and markdown formatting) + - `install.sh`, `uninstall.sh` (installer scripts for Bun, repo cloning, and symlink setup) + - `docs/` directory (5 new files: `getting-started.md`, `cli.md`, `architecture.md`, `configuration.md`, `team-sharing.md`) + - `CLAUDE.md` (rewritten to reflect Smriti commands and project structure) + - `LICENSE` (MIT license file) + - `.smriti/` directory (28 files for knowledge categorization, including `config.json` and `index.json`) +- **Features added**: + - CLI command reference and configuration guide + - Team-sharing workflow documentation + - Architecture diagram and QMD integration details +- **Configurations updated**: + - Removed `v1` tag from README header + - Added `SMRITI_NO_HOOK` and `SMRITI_PURGE` environment variables + +--- + +## Decisions + +- **QMD integration**: Leveraged Shopify CEO Tobi Lütke’s QMD library for shared memory, avoiding reinvention while ensuring compatibility. +- **MIT license**: Chosen for open-source accessibility, aligning with community expectations and reducing legal friction. +- **Documentation structure**: Split into focused sections (CLI, architecture, team-sharing) to prioritize usability for developers and admins. +- **Local-first design**: Emphasized no-cloud operation to address privacy concerns and ensure reliability without external dependencies. + +--- + +## Insights + +- **Critical dependency management**: QMD’s role in shared memory required careful integration to avoid breaking changes, highlighting the importance of maintaining backward compatibility. +- **Documentation as a product**: The CLI reference and architecture diagrams became essential for onboarding, proving that clear, structured docs reduce support overhead. +- **Team collaboration patterns**: The `.smriti/` folder’s design revealed that categorizing knowledge (bugs, features, etc.) improves search efficiency, saving developers time during troubleshooting. + +--- + +## Context + +- **Prior state**: Smriti was a functional CLI tool for AI agent context management but lacked polished documentation and a clear release process. +- **Constraints**: Needed to avoid cloud dependencies for privacy, ensure cross-platform installability, and align with open-source norms (MIT license). +- **Gotchas**: The `.smriti/` folder’s purpose was initially misunderstood as part of the tool itself, requiring explicit clarification to prevent confusion during onboarding. diff --git a/.smriti/knowledge/uncategorized/2026-02-10_this-session-focused-on-refining-smritis-documenta.md b/.smriti/knowledge/uncategorized/2026-02-10_this-session-focused-on-refining-smritis-documenta.md new file mode 100644 index 0000000..57d6cd8 --- /dev/null +++ b/.smriti/knowledge/uncategorized/2026-02-10_this-session-focused-on-refining-smritis-documenta.md @@ -0,0 +1,35 @@ +--- +id: 96102237 +category: uncategorized +project: +agent: +author: zero8 +shared_at: 2026-02-10T17:54:38.709Z +tags: [] +--- + +# This session focused on refining Smriti's documentation, clarifying its core ... + +> This session focused on refining Smriti's documentation, clarifying its core functionality, and finalizing design assets. Key outcomes included crafting a logo prompt that balances technical and cultural elements, explaining Smriti's memory-layer capabilities, and fixing minor grammar issues in the README. These efforts ensure clarity for users and developers while aligning with the project's local-first, privacy-centric ethos. + +## Changes + +- **Modified**: `README.md` (grammar fixes for lines 192 and 209) +- **Added**: `docs/logo_prompt.md` (detailed logo design prompt for Smriti) +- **Updated**: `.env.example` (added `SMRITI_PURGE=1` for hook state removal) + +## Decisions + +- **Logo design**: Chose a gradient indigo-violet palette with a glowing node to symbolize recall, avoiding text for minimalism. +- **README focus**: Prioritized clarity over exhaustive detail, emphasizing token efficiency and local-first architecture. +- **Environment variable**: Added `SMRITI_PURGE=1` to `.env.example` for explicit state management during commands. + +## Insights + +- **Logo symbolism**: The brain-node motif effectively communicates memory and retrieval without relying on text, aligning with modern dev tool aesthetics. +- **Documentation balance**: Concise explanations of Smriti's hybrid search and token efficiency are critical for non-technical users to grasp its value. +- **Local-first constraints**: Emphasizing SQLite and Ollama integration in the README helps users understand privacy and control. + +## Context + +Smriti is a local memory layer for AI agents, requiring clear documentation to explain its hybrid search, token efficiency, and team-sharing features. The logo design needed to reflect both technical functionality and cultural meaning (Sanscrit "memory"). Grammar fixes and environment variable updates ensure usability and precision in user workflows. diff --git a/CLAUDE.md b/CLAUDE.md index fcc12e5..43ba6af 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,6 +6,7 @@ Shared memory layer for AI-powered engineering teams. Built on [QMD](https://git ```bash smriti ingest claude # Ingest Claude Code sessions +smriti ingest antigravity # Ingest Antigravity IDE sessions smriti ingest all # Ingest from all known agents smriti search "query" # Hybrid search (BM25 + vector) smriti recall "query" # Smart recall with dedup @@ -31,6 +32,7 @@ src/ ├── ingest/ │ ├── index.ts # Ingest orchestrator + types │ ├── claude.ts # Claude Code JSONL parser + project detection +│ ├── antigravity.ts # Antigravity IDE markdown parser │ ├── codex.ts # Codex CLI parser │ ├── cursor.ts # Cursor IDE parser │ └── generic.ts # File import (chat/jsonl formats) @@ -42,13 +44,19 @@ src/ │ └── classifier.ts # Auto-categorization (rule-based + LLM) └── team/ ├── share.ts # Export knowledge to .smriti/ directory - └── sync.ts # Import team knowledge from .smriti/ + ├── sync.ts # Import team knowledge from .smriti/ + ├── formatter.ts # Sanitization + doc formatting pipeline + ├── reflect.ts # LLM-powered session reflection via Ollama + └── prompts/ + └── share-reflect.md # Customizable reflection prompt template test/ ├── ingest.test.ts # Parser + project detection tests ├── search.test.ts # Search + recall tests ├── db.test.ts # Schema + metadata tests ├── categorize.test.ts # Categorization tests -└── team.test.ts # Share + sync tests +├── team.test.ts # Share + sync tests +├── formatter.test.ts # Formatter + sanitization tests +└── reflect.test.ts # Reflection parsing tests ``` ## Architecture @@ -87,22 +95,28 @@ Claude Code stores sessions in `~/.claude/projects//`. The dir name en ### Team Sharing -- `smriti share`: Exports sessions as markdown with YAML frontmatter to `.smriti/knowledge/` +- `smriti share`: Exports sessions as clean documentation to `.smriti/knowledge/` + - Sanitizes XML noise, interrupt markers, API errors, narration filler + - Filters noise-only sessions, merges consecutive same-role messages + - Generates LLM reflections via Ollama by default (use `--no-reflect` to skip) + - Generates `.smriti/CLAUDE.md` so Claude Code auto-discovers shared knowledge + - Customizable reflection prompt at `.smriti/prompts/share-reflect.md` - `smriti sync`: Imports markdown files from `.smriti/knowledge/` back into local DB - Deduplication via content hashing — same content won't import twice ## Configuration -| Env Var | Default | Description | -|---------|---------|-------------| -| `QMD_DB_PATH` | `~/.cache/qmd/index.sqlite` | Database path | -| `CLAUDE_LOGS_DIR` | `~/.claude/projects` | Claude Code logs | -| `CODEX_LOGS_DIR` | `~/.codex` | Codex CLI logs | -| `SMRITI_PROJECTS_ROOT` | `~/zero8.dev` | Projects root for ID derivation | -| `OLLAMA_HOST` | `http://127.0.0.1:11434` | Ollama endpoint | -| `QMD_MEMORY_MODEL` | `qwen3:8b-tuned` | Ollama model for synthesis | -| `SMRITI_CLASSIFY_THRESHOLD` | `0.5` | LLM classification trigger threshold | -| `SMRITI_AUTHOR` | `$USER` | Git author for team sharing | +| Env Var | Default | Description | +| --------------------------- | ----------------------------- | ------------------------------------ | +| `QMD_DB_PATH` | `~/.cache/qmd/index.sqlite` | Database path | +| `CLAUDE_LOGS_DIR` | `~/.claude/projects` | Claude Code logs | +| `CODEX_LOGS_DIR` | `~/.codex` | Codex CLI logs | +| `SMRITI_ANTIGRAVITY_DIR` | `~/.gemini/antigravity/brain` | Antigravity IDE brain | +| `SMRITI_PROJECTS_ROOT` | `~/zero8.dev` | Projects root for ID derivation | +| `OLLAMA_HOST` | `http://127.0.0.1:11434` | Ollama endpoint | +| `QMD_MEMORY_MODEL` | `qwen3:8b-tuned` | Ollama model for synthesis | +| `SMRITI_CLASSIFY_THRESHOLD` | `0.5` | LLM classification trigger threshold | +| `SMRITI_AUTHOR` | `$USER` | Git author for team sharing | ## Database diff --git a/README.md b/README.md index ad332d7..7ac60e1 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,20 @@ Built on top of [QMD](https://github.com/tobi/qmd) by Tobi Lütke. ## The Problem -Your team ships code with AI agents every day — Claude Code, Cursor, Codex. But every agent has a blind spot: +Your team ships code with AI agents every day — Claude Code, Cursor, Antigravity, Codex. But every agent has a blind spot: > **They don't remember anything.** Not from yesterday. Not from each other. Not from your teammates. Here's what that looks like: -| Monday | Tuesday | -|--------|---------| +| Monday | Tuesday | +| ------------------------------------------------------------- | --------------------------------------------------- | | Your teammate spends 3 hours with Claude on an auth migration | You open a fresh session and ask the same questions | -| Claude figures out the right approach, makes key decisions | Your Claude has no idea any of that happened | -| Architectural insights, debugging breakthroughs, trade-offs | All of it — gone | +| Claude figures out the right approach, makes key decisions | Your Claude has no idea any of that happened | +| Architectural insights, debugging breakthroughs, trade-offs | All of it — gone | The result: + - **Duplicated work** — same questions asked across the team, different answers every time - **Lost decisions** — "why did we do it this way?" lives in someone's closed chat window - **Zero continuity** — each session starts from scratch, no matter how much your team has already figured out @@ -29,7 +30,7 @@ The agents are brilliant. But they're amnesic. **This is the biggest gap in AI-a ## What Smriti Does -**Smriti** (Sanskrit: *memory*) is a shared memory layer that sits underneath all your AI agents. +**Smriti** (Sanskrit: _memory_) is a shared memory layer that sits underneath all your AI agents. Every conversation → automatically captured → indexed → searchable. One command to recall what matters. @@ -100,6 +101,7 @@ curl -fsSL https://raw.githubusercontent.com/zero8dotdev/smriti/main/install.sh ``` This will: + - Install [Bun](https://bun.sh) if you don't have it - Clone Smriti to `~/.smriti` - Set up the `smriti` CLI @@ -120,6 +122,7 @@ This will: smriti ingest claude # Claude Code sessions smriti ingest codex # Codex CLI sessions smriti ingest cursor --project-path ./myapp +smriti ingest antigravity # Antigravity IDE sessions smriti ingest file transcript.txt --title "Planning Session" smriti ingest all # All known agents at once @@ -140,6 +143,12 @@ smriti embed # Build vector embeddings for semantic search smriti categorize # Auto-categorize sessions smriti projects # List all tracked projects +# Context and comparison +smriti context # Generate project context for .smriti/CLAUDE.md +smriti context --dry-run # Preview without writing +smriti compare --last # Compare last 2 sessions (tokens, tools, files) +smriti compare # Compare specific sessions + # Team sharing smriti share --project myapp # Export to .smriti/ for git smriti sync # Import teammates' shared knowledge @@ -149,8 +158,8 @@ smriti team # View team contributions ## How It Works ``` - Claude Code Cursor Codex Other Agents - | | | | + Claude Code Cursor Codex Antigravity + | | | | v v v v ┌──────────────────────────────────────────┐ │ Smriti Ingestion Layer │ @@ -179,17 +188,252 @@ smriti team # View team contributions Everything runs locally. Your conversations never leave your machine. The SQLite database, the embeddings, the search indexes — all on disk, all yours. -## Token Savings +## Tagging & Categories + +Sessions and messages are automatically tagged into a hierarchical category tree. Tags flow through every command — search, recall, list, and share — so you can slice your team's knowledge by topic. + +### Default Category Tree + +Smriti ships with 7 top-level categories and 21 subcategories: + +| Category | Subcategories | +| -------------- | ----------------------------------------------------------------------- | +| `code` | `code/implementation`, `code/pattern`, `code/review`, `code/snippet` | +| `architecture` | `architecture/design`, `architecture/decision`, `architecture/tradeoff` | +| `bug` | `bug/report`, `bug/fix`, `bug/investigation` | +| `feature` | `feature/requirement`, `feature/design`, `feature/implementation` | +| `project` | `project/setup`, `project/config`, `project/dependency` | +| `decision` | `decision/technical`, `decision/process`, `decision/tooling` | +| `topic` | `topic/learning`, `topic/explanation`, `topic/comparison` | + +### Auto-Classification + +Smriti uses a two-stage pipeline to classify messages: + +1. **Rule-based** — 24 keyword patterns with weighted confidence scoring. Each pattern targets a specific subcategory (e.g., words like "crash", "stacktrace", "panic" map to `bug/report`). Confidence is calculated from keyword density and rule weight. +2. **LLM fallback** — When rule confidence falls below the threshold (default `0.5`, configurable via `SMRITI_CLASSIFY_THRESHOLD`), Ollama classifies the message. Only activated when you pass `--llm`. + +The most frequent category across a session's messages becomes the session-level tag. + +```bash +# Auto-categorize all uncategorized sessions (rule-based) +smriti categorize + +# Include LLM fallback for ambiguous sessions +smriti categorize --llm + +# Categorize a specific session +smriti categorize --session +``` + +### Manual Tagging + +Override or supplement auto-classification with manual tags: + +```bash +smriti tag + +# Examples +smriti tag abc123 decision/technical +smriti tag abc123 bug/fix +``` + +Manual tags are stored with confidence `1.0` and source `"manual"`. + +### Custom Categories + +Add your own categories to extend the default tree: + +```bash +# List the full category tree +smriti categories + +# Add a top-level category +smriti categories add ops --name "Operations" + +# Add a nested category under an existing parent +smriti categories add ops/incident --name "Incident Response" --parent ops + +# Include a description +smriti categories add ops/runbook --name "Runbooks" --parent ops --description "Operational runbook sessions" +``` + +### How Tags Filter Commands + +The `--category` flag works across search, recall, list, and share: + +| Command | Effect of `--category` | +| --------------- | ------------------------------------------------------------------------------- | +| `smriti list` | Shows categories column; filters sessions to matching category | +| `smriti search` | Filters full-text search results to matching category | +| `smriti recall` | Filters recall context; works with `--synthesize` | +| `smriti share` | Controls which sessions are exported; files organized into `.smriti/knowledge/` | +| `smriti status` | Shows session count per category (no filter flag — always shows all) | + +**Hierarchical filtering** — Filtering by a parent category automatically includes all its children. `--category decision` matches `decision/technical`, `decision/process`, and `decision/tooling`. + +### Categories in Share & Sync + +**Categories survive the share/sync roundtrip exactly.** What gets serialized during `smriti share` is exactly what gets deserialized during `smriti sync` — the same category ID goes in, the same category ID comes out. No reclassification, no transformation, no loss. The category a session was tagged with on one machine is the category it will be indexed under on every other machine that syncs it. + +When you share sessions, the category is embedded in YAML frontmatter inside each exported markdown file: + +```yaml +--- +id: 2e5f420a-e376-4ad4-8b35-ad94838cbc42 +category: project +project: smriti +agent: claude-code +author: zero8 +shared_at: 2026-02-10T11:29:44.501Z +tags: ["project", "project/dependency"] +--- +``` + +When a teammate runs `smriti sync`, the frontmatter is parsed and the category is restored into their local `smriti_session_tags` table — indexed as `project`, searchable as `project`, filterable as `project`. The serialization and deserialization are symmetric: `share` writes `category: project` → `sync` reads `category: project` → `tagSession(db, sessionId, "project", 1.0, "team")`. No intermediate step reinterprets the value. + +Files are organized into subdirectories by primary category (e.g., `.smriti/knowledge/project/`, `.smriti/knowledge/decision/`), but sync reads the category from frontmatter, not the directory path. + +> **Note:** Currently only the primary `category` field is restored on sync. Secondary tags in the `tags` array are serialized in the frontmatter but not yet imported. If a session had multiple tags (e.g., `project` + `decision/tooling`), only the primary tag survives the roundtrip. + +```bash +# Share decisions — category metadata travels with the files +smriti share --project myapp --category decision + +# Teammate syncs — categories restored exactly from frontmatter +smriti sync --project myapp +``` + +### Examples -The real value: **your agents get better context with fewer tokens.** +```bash +# All architectural decisions +smriti search "database" --category architecture + +# Recall only bug-related context +smriti recall "connection timeout" --category bug --synthesize + +# List feature sessions for a specific project +smriti list --category feature --project myapp + +# Share only decision sessions +smriti share --project myapp --category decision +``` + +## Context: Token Reduction (North Star) + +Every new Claude Code session starts from zero — no awareness of what happened yesterday, which files were touched, what decisions were made. `smriti context` generates a compact project summary (~200-300 tokens) and injects it into `.smriti/CLAUDE.md`, which Claude Code auto-discovers. + +```bash +smriti context # auto-detect project, write .smriti/CLAUDE.md +smriti context --dry-run # preview without writing +smriti context --project myapp # explicit project +smriti context --days 14 # 14-day lookback (default: 7) +``` + +The output looks like this: + +```markdown +## Project Context + +> Auto-generated by `smriti context` on 2026-02-11. Do not edit manually. + +### Recent Sessions (last 7 days) + +- **2h ago** Enriched ingestion pipeline (12 turns) [code] +- **1d ago** Search & recall pipeline (8 turns) [feature] + +### Hot Files + +`src/db.ts` (14 ops), `src/ingest/claude.ts` (11 ops), `src/search/index.ts` (8 ops) + +### Git Activity + +- commit `main`: "Fix auth token refresh" (2026-02-10) + +### Usage + +5 sessions, 48 turns, ~125K input / ~35K output tokens +``` + +No Ollama, no network calls, no model loading. Pure SQL queries against sidecar tables, rendered as markdown. Runs in < 100ms. + +### Measuring the Impact + +Does this actually save tokens? Honestly — we don't know yet. We built the tools to measure it, ran A/B tests, and the results so far are... humbling. Claude is annoyingly good at finding the right files even without help. -| Scenario | Without Smriti | With Smriti | Reduction | -|----------|---------------|-------------|-----------| -| Relevant context from past sessions | ~20,000 tokens | ~500 tokens | **40x** | -| Multi-session recall + synthesis | ~10,000 tokens | ~200 tokens | **50x** | -| Full project conversation history | 50,000+ tokens | ~500 tokens | **100x** | +But this is the north star, not the destination. We believe context injection will matter most on large codebases without detailed docs, ambiguous tasks that require exploration, and multi-session continuity. We just need the data to prove it (or disprove it and try something else). -Less token spend, faster responses, more room for the actual work in your context window. +So we're shipping the measurement tools and asking you to help. Run A/B tests on your projects, paste the results in [Issue #13](https://github.com/zero8dotdev/smriti/issues/13), and let's figure this out together. + +#### A/B Testing Guide + +```bash +# Step 1: Baseline session (no context) +mv .smriti/CLAUDE.md .smriti/CLAUDE.md.bak +# Start a Claude Code session, give it a task, let it finish, exit + +# Step 2: Context session +mv .smriti/CLAUDE.md.bak .smriti/CLAUDE.md +smriti context +# Start a new session, give the EXACT same task, let it finish, exit + +# Step 3: Ingest and compare +smriti ingest claude +smriti compare --last --project myapp +``` + +#### Compare Command + +```bash +smriti compare # by session ID (supports partial IDs) +smriti compare --last # last 2 sessions for current project +smriti compare --last --project myapp # last 2 sessions for specific project +smriti compare --last --json # machine-readable output +``` + +Output: + +``` +Session A: Fix auth bug (no context) +Session B: Fix auth bug (with context) + +Metric A B Diff +---------------------------------------------------------------- +Turns 12 8 -4 (-33%) +Total tokens 45K 32K -13000 (-29%) +Tool calls 18 11 -7 (-39%) +File reads 10 4 -6 (-60%) + +Tool breakdown: + Bash 4 3 + Glob 3 0 + Read 10 4 + Write 1 4 +``` + +#### What We've Tested So Far + +| Task Type | Context Impact | Notes | +| ----------------------------------------- | -------------- | ---------------------------------------------------------------------- | +| Knowledge questions ("how does X work?") | Minimal | Both sessions found the right files immediately from project CLAUDE.md | +| Implementation tasks ("add --since flag") | Minimal | Small, well-scoped tasks don't need exploration | +| Ambiguous/exploration tasks | Untested | Expected sweet spot — hot files guide Claude to the right area | +| Large codebases (no project CLAUDE.md) | Untested | Expected sweet spot — context replaces missing documentation | + +**We need your help.** If you run A/B tests on your projects, please share your results in [GitHub Issues](https://github.com/zero8dotdev/smriti/issues). Include the `smriti compare` output and a description of the task. This data will help us understand where context injection actually matters. + +### Token Savings (Search & Recall) + +Separate from context injection, Smriti's search and recall pipeline compresses past conversations: + +| Scenario | Raw Conversations | Via Smriti | Reduction | +| ----------------------------------- | ----------------- | ----------- | --------- | +| Relevant context from past sessions | ~20,000 tokens | ~500 tokens | **40x** | +| Multi-session recall + synthesis | ~10,000 tokens | ~200 tokens | **50x** | +| Full project conversation history | 50,000+ tokens | ~500 tokens | **100x** | + +Lower token spend, faster responses, more room for the actual work in your context window. ## Privacy @@ -200,13 +444,33 @@ Smriti is local-first by design. No cloud, no telemetry, no accounts. - Synthesis via local [Ollama](https://ollama.ai) (optional) - Team sharing happens through git — you control what gets committed +## FAQ + +**When does knowledge get captured?** +Automatically. Smriti hooks into your AI coding tool (Claude Code, Cursor, etc.) and captures every session without any manual step. You just code normally and `smriti ingest` pulls in the conversations. + +**Who has access to my data?** +Only you. Everything lives in a local SQLite database (`~/.cache/qmd/index.sqlite`). There's no cloud, no accounts, no telemetry. Team sharing is explicit — you run `smriti share` to export, commit the `.smriti/` folder to git, and teammates run `smriti sync` to import. + +**Can AI agents query the knowledge base?** +Yes. `smriti recall "query"` returns relevant past context that agents can use. When you run `smriti share`, it generates a `.smriti/CLAUDE.md` index so Claude Code automatically discovers shared knowledge. Agents can search, grep, and recall from the full knowledge base. + +**How do multiple projects stay separate?** +Each project gets its own `.smriti/` folder in its repo root. Sessions are tagged with project IDs in the central database. Search works cross-project by default, but you can scope to a single project with `--project `. Knowledge shared via git stays within that project's repo. + +**Does this work with Jira or other issue trackers?** +Not yet — Smriti is git-native today. Issue tracker integrations are on the roadmap. If you have ideas, open a discussion in [GitHub Issues](https://github.com/zero8dotdev/smriti/issues). + +**How does this help preserve existing features during changes?** +The reasoning behind each code change is captured and searchable. When an AI agent starts a new session, it can recall _why_ something was built a certain way — reducing the chance of accidentally breaking existing behavior. + ## Uninstall ```bash curl -fsSL https://raw.githubusercontent.com/zero8dotdev/smriti/main/uninstall.sh | bash ``` -To also remove hook state: `SMRITI_PURGE=1` before the command. +To also remove hook state, prepend `SMRITI_PURGE=1` to the command. ## Documentation diff --git a/bun.lock b/bun.lock index a5050d2..6c82f09 100644 --- a/bun.lock +++ b/bun.lock @@ -122,7 +122,7 @@ "@types/aws-lambda": ["@types/aws-lambda@8.10.160", "", {}, "sha512-uoO4QVQNWFPJMh26pXtmtrRfGshPUSpMZGUyUQY20FhfHEElEBOPKgVmFs1z+kbpyBsRs2JnoOPT7++Z4GA9pA=="], - "@types/bun": ["@types/bun@1.3.8", "", { "dependencies": { "bun-types": "1.3.8" } }, "sha512-3LvWJ2q5GerAXYxO2mffLTqOzEu5qnhEAlh48Vnu8WQfnmSwbgagjGZV6BoHKJztENYEDn6QmVd949W4uESRJA=="], + "@types/bun": ["@types/bun@1.3.9", "", { "dependencies": { "bun-types": "1.3.9" } }, "sha512-KQ571yULOdWJiMH+RIWIOZ7B2RXQGpL1YQrBtLIV3FqDcCu6FsbFUBwhdKUlCKUpS3PJDsHlJ1QKlpxoVR+xtw=="], "@types/node": ["@types/node@25.2.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-BkmoP5/FhRYek5izySdkOneRyXYN35I860MFAGupTdebyE66uZaR+bXLHq8k4DirE5DwQi3NuhvRU1jqTVwUrQ=="], @@ -154,7 +154,7 @@ "bottleneck": ["bottleneck@2.19.5", "", {}, "sha512-VHiNCbI1lKdl44tGrhNfU3lup0Tj/ZBMJB5/2ZbNXRCPuRCO7ed2mgcK4r17y+KB2EfuYuRaVlwNbAeaWGSpbw=="], - "bun-types": ["bun-types@1.3.8", "", { "dependencies": { "@types/node": "*" } }, "sha512-fL99nxdOWvV4LqjmC+8Q9kW3M4QTtTR1eePs94v5ctGqU8OeceWrSUaRw3JYb7tU3FkMIAjkueehrHPPPGKi5Q=="], + "bun-types": ["bun-types@1.3.9", "", { "dependencies": { "@types/node": "*" } }, "sha512-+UBWWOakIP4Tswh0Bt0QD0alpTY8cb5hvgiYeWCMet9YukHbzuruIEeXC2D7nMJPB12kbh8C7XJykSexEqGKJg=="], "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], diff --git a/docs/architecture.md b/docs/architecture.md index 5ec1d33..5b9533e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -3,13 +3,14 @@ ## Overview ``` - Claude Code Cursor Codex Other Agents - | | | | + Claude Code Cursor Codex Antigravity + | | | | v v v v ┌──────────────────────────────────────────┐ │ Smriti Ingestion Layer │ │ │ │ src/ingest/claude.ts (JSONL parser) │ + │ src/ingest/antigravity.ts (Markdown) │ │ src/ingest/codex.ts (JSONL parser) │ │ src/ingest/cursor.ts (JSON parser) │ │ src/ingest/generic.ts (file import) │ @@ -130,6 +131,7 @@ Sessions are exported as markdown files with YAML frontmatter: ``` Each file contains: + - YAML frontmatter (session ID, category, project, agent, author, tags) - Session title as heading - Summary (if available) @@ -145,21 +147,21 @@ Reads markdown files from `.smriti/knowledge/`, parses frontmatter and conversat ### QMD Tables (not modified by Smriti) -| Table | Purpose | -|-------|---------| +| Table | Purpose | +| ----------------- | ------------------------------------------------- | | `memory_sessions` | Session metadata (id, title, timestamps, summary) | | `memory_messages` | Messages (session_id, role, content, SHA256 hash) | -| `memory_fts` | FTS5 index on session titles + message content | -| `content_vectors` | 384-dim embeddings keyed by content hash | +| `memory_fts` | FTS5 index on session titles + message content | +| `content_vectors` | 384-dim embeddings keyed by content hash | ### Smriti Tables -| Table | Purpose | -|-------|---------| -| `smriti_agents` | Agent registry (claude-code, codex, cursor) | -| `smriti_projects` | Project registry (id, filesystem path) | -| `smriti_session_meta` | Maps sessions to agents and projects | -| `smriti_categories` | Hierarchical category taxonomy | -| `smriti_session_tags` | Category tags on sessions (with confidence) | -| `smriti_message_tags` | Category tags on messages (with confidence) | -| `smriti_shares` | Deduplication tracking for team sharing | +| Table | Purpose | +| --------------------- | -------------------------------------------------------- | +| `smriti_agents` | Agent registry (claude-code, codex, cursor, antigravity) | +| `smriti_projects` | Project registry (id, filesystem path) | +| `smriti_session_meta` | Maps sessions to agents and projects | +| `smriti_categories` | Hierarchical category taxonomy | +| `smriti_session_tags` | Category tags on sessions (with confidence) | +| `smriti_message_tags` | Category tags on messages (with confidence) | +| `smriti_shares` | Deduplication tracking for team sharing | diff --git a/package.json b/package.json index 29638f9..2d80fc3 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "smriti": "bun src/index.ts" }, "dependencies": { - "node-llama-cpp": "^3.0.0", + "node-llama-cpp": "^3.15.1", "qmd": "github:zero8dotdev/qmd" }, "devDependencies": { diff --git a/src/config.ts b/src/config.ts index 70bb5aa..702b54b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -23,8 +23,12 @@ export const CLAUDE_LOGS_DIR = Bun.env.CLAUDE_LOGS_DIR || join(HOME, ".claude", "projects"); /** Codex CLI logs directory */ -export const CODEX_LOGS_DIR = - Bun.env.CODEX_LOGS_DIR || join(HOME, ".codex"); +export const CODEX_LOGS_DIR = Bun.env.CODEX_LOGS_DIR || join(HOME, ".codex"); + +/** Antigravity IDE brain directory */ +export const ANTIGRAVITY_BRAIN_DIR = + Bun.env.SMRITI_ANTIGRAVITY_DIR || + join(HOME, ".gemini", "antigravity", "brain"); /** Default smriti team directory name within projects */ export const SMRITI_DIR = ".smriti"; @@ -42,7 +46,7 @@ export const OLLAMA_MODEL = Bun.env.QMD_MEMORY_MODEL || "qwen3:8b-tuned"; /** Confidence threshold below which rule-based classification triggers LLM */ export const CLASSIFY_LLM_THRESHOLD = Number( - Bun.env.SMRITI_CLASSIFY_THRESHOLD || "0.5" + Bun.env.SMRITI_CLASSIFY_THRESHOLD || "0.5", ); // ============================================================================= @@ -52,6 +56,7 @@ export const CLASSIFY_LLM_THRESHOLD = Number( export const DEFAULT_SEARCH_LIMIT = 20; export const DEFAULT_LIST_LIMIT = 50; export const DEFAULT_RECALL_LIMIT = 10; +export const DEFAULT_CONTEXT_DAYS = 7; /** Git author name for team sharing */ export const AUTHOR = Bun.env.SMRITI_AUTHOR || Bun.env.USER || "unknown"; diff --git a/src/context.ts b/src/context.ts new file mode 100644 index 0000000..52b8087 --- /dev/null +++ b/src/context.ts @@ -0,0 +1,811 @@ +/** + * context.ts - Generate project context for token reduction + * + * Queries sidecar tables and renders a compact markdown block + * for `.smriti/CLAUDE.md`. Pure SQL → markdown, no Ollama needed. + */ + +import type { Database } from "bun:sqlite"; +import { join } from "path"; +import { mkdirSync } from "fs"; +import { DEFAULT_CONTEXT_DAYS, SMRITI_DIR } from "./config"; + +// ============================================================================= +// Types +// ============================================================================= + +export type ContextOptions = { + project?: string; + days?: number; + dryRun?: boolean; + json?: boolean; + cwd?: string; +}; + +export type ProjectContext = { + sessions: Array<{ + id: string; + title: string; + updatedAt: string; + turnCount: number | null; + categories: string; + }>; + hotFiles: Array<{ + filePath: string; + ops: number; + lastOp: string; + lastAt: string; + }>; + gitActivity: Array<{ + operation: string; + branch: string | null; + details: string | null; + createdAt: string; + }>; + errors: Array<{ + errorType: string; + count: number; + }>; + usage: { + sessions: number; + turns: number; + inputTokens: number; + outputTokens: number; + } | null; +}; + +// ============================================================================= +// Project Detection +// ============================================================================= + +/** + * Detect project ID from current working directory. + * Tries exact match first, then checks if the directory basename + * matches a project ID (handles path derivation mismatches). + */ +export function detectProject(db: Database, cwd?: string): string | null { + const dir = cwd || process.cwd(); + + // Exact path match + const exact = db + .prepare(`SELECT id FROM smriti_projects WHERE path = ?`) + .get(dir) as { id: string } | null; + if (exact) return exact.id; + + // Fallback: match by directory basename as project ID + const dirName = dir.split("/").pop(); + if (dirName) { + const byName = db + .prepare(`SELECT id FROM smriti_projects WHERE id = ?`) + .get(dirName) as { id: string } | null; + if (byName) return byName.id; + } + + return null; +} + +// ============================================================================= +// Context Gathering +// ============================================================================= + +/** + * Query sidecar tables for project context. + * Each section is independent — empty tables produce empty arrays. + */ +export function gatherContext( + db: Database, + projectId: string, + days: number = DEFAULT_CONTEXT_DAYS +): ProjectContext { + const interval = `-${days} days`; + + // Recent sessions + const sessions = db + .prepare( + `SELECT ms.id, ms.title, ms.updated_at, sc.turn_count, + COALESCE(GROUP_CONCAT(DISTINCT st.category_id), '') AS categories + FROM memory_sessions ms + JOIN smriti_session_meta sm ON sm.session_id = ms.id + LEFT JOIN smriti_session_costs sc ON sc.session_id = ms.id + LEFT JOIN smriti_session_tags st ON st.session_id = ms.id + WHERE sm.project_id = ? AND ms.updated_at >= datetime('now', ?) + GROUP BY ms.id ORDER BY ms.updated_at DESC LIMIT 5` + ) + .all(projectId, interval) as Array<{ + id: string; + title: string; + updated_at: string; + turn_count: number | null; + categories: string; + }>; + + // Hot files + const hotFiles = db + .prepare( + `SELECT file_path, COUNT(*) AS ops, MAX(operation) AS last_op, MAX(created_at) AS last_at + FROM smriti_file_operations + WHERE project_id = ? AND created_at >= datetime('now', ?) + GROUP BY file_path ORDER BY ops DESC LIMIT 10` + ) + .all(projectId, interval) as Array<{ + file_path: string; + ops: number; + last_op: string; + last_at: string; + }>; + + // Git activity + const gitActivity = db + .prepare( + `SELECT go.operation, go.branch, go.details, go.created_at + FROM smriti_git_operations go + JOIN smriti_session_meta sm ON sm.session_id = go.session_id + WHERE sm.project_id = ? AND go.created_at >= datetime('now', ?) + AND go.operation IN ('commit','pr_create','push','merge','checkout') + ORDER BY go.created_at DESC LIMIT 5` + ) + .all(projectId, interval) as Array<{ + operation: string; + branch: string | null; + details: string | null; + created_at: string; + }>; + + // Recent errors + const errors = db + .prepare( + `SELECT error_type, COUNT(*) AS count + FROM smriti_errors e + JOIN smriti_session_meta sm ON sm.session_id = e.session_id + WHERE sm.project_id = ? AND e.created_at >= datetime('now', ?) + GROUP BY error_type ORDER BY count DESC LIMIT 3` + ) + .all(projectId, interval) as Array<{ + error_type: string; + count: number; + }>; + + // Cost summary (all-time for this project, not time-limited) + const costRow = db + .prepare( + `SELECT COUNT(*) AS sessions, SUM(turn_count) AS turns, + SUM(total_input_tokens) AS input_tok, SUM(total_output_tokens) AS output_tok + FROM smriti_session_costs sc + JOIN smriti_session_meta sm ON sm.session_id = sc.session_id + WHERE sm.project_id = ?` + ) + .get(projectId) as { + sessions: number; + turns: number | null; + input_tok: number | null; + output_tok: number | null; + } | null; + + const usage = + costRow && costRow.sessions > 0 + ? { + sessions: costRow.sessions, + turns: costRow.turns || 0, + inputTokens: costRow.input_tok || 0, + outputTokens: costRow.output_tok || 0, + } + : null; + + return { + sessions: sessions.map((s) => ({ + id: s.id, + title: s.title, + updatedAt: s.updated_at, + turnCount: s.turn_count, + categories: s.categories, + })), + hotFiles: hotFiles.map((f) => ({ + filePath: f.file_path, + ops: f.ops, + lastOp: f.last_op, + lastAt: f.last_at, + })), + gitActivity: gitActivity.map((g) => ({ + operation: g.operation, + branch: g.branch, + details: g.details, + createdAt: g.created_at, + })), + errors: errors.map((e) => ({ + errorType: e.error_type, + count: e.count, + })), + usage, + }; +} + +// ============================================================================= +// Rendering +// ============================================================================= + +/** Format a relative time string from an ISO date */ +function relativeTime(isoDate: string): string { + const now = Date.now(); + const then = new Date(isoDate).getTime(); + const diffMs = now - then; + const diffMin = Math.floor(diffMs / 60000); + const diffHr = Math.floor(diffMs / 3600000); + const diffDay = Math.floor(diffMs / 86400000); + + if (diffMin < 1) return "just now"; + if (diffMin < 60) return `${diffMin}m ago`; + if (diffHr < 24) return `${diffHr}h ago`; + if (diffDay === 1) return "1d ago"; + return `${diffDay}d ago`; +} + +/** Format token count as human-readable */ +function formatTokens(n: number): string { + if (n >= 1000000) return `${(n / 1000000).toFixed(1)}M`; + if (n >= 1000) return `${Math.round(n / 1000)}K`; + return String(n); +} + +/** Try to parse a commit message from git details JSON */ +function parseGitDetails(details: string | null): string | null { + if (!details) return null; + try { + const parsed = JSON.parse(details); + const msg = parsed.message || parsed.commit_message || null; + // Skip shell syntax artifacts like heredoc markers + if (msg && (msg.includes("$(cat") || msg.includes("<<"))) return null; + return msg; + } catch { + if (details.includes("$(cat") || details.includes("<<")) return null; + return details.length <= 60 ? details : null; + } +} + +/** Strip a project root prefix from file paths for readability */ +function relativePath(filePath: string, projectPath?: string): string { + // Try the provided project path + if (projectPath && filePath.startsWith(projectPath)) { + const rel = filePath.slice(projectPath.length); + return rel.startsWith("/") ? rel.slice(1) : rel; + } + // Try cwd (handles path derivation mismatches) + const cwd = process.cwd(); + if (filePath.startsWith(cwd + "/")) { + return filePath.slice(cwd.length + 1); + } + // Fallback: strip home directory prefix + const home = process.env.HOME || ""; + if (home && filePath.startsWith(home)) { + return "~" + filePath.slice(home.length); + } + return filePath; +} + +/** + * Render ProjectContext into a compact markdown block. + * Omits sections that are empty. + */ +export function renderContext( + ctx: ProjectContext, + projectId: string, + days: number = DEFAULT_CONTEXT_DAYS, + projectPath?: string +): string { + const sections: string[] = []; + const date = new Date().toISOString().slice(0, 10); + + sections.push(`## Project Context`); + sections.push(""); + sections.push( + `> Auto-generated by \`smriti context\` on ${date}. Do not edit manually.` + ); + + // Recent sessions + if (ctx.sessions.length > 0) { + sections.push(""); + sections.push(`### Recent Sessions (last ${days} days)`); + for (const s of ctx.sessions) { + const time = relativeTime(s.updatedAt); + const title = s.title || s.id.slice(0, 8); + const turns = s.turnCount ? ` (${s.turnCount} turns)` : ""; + const cats = s.categories + ? ` [${s.categories.split(",")[0]}]` + : ""; + sections.push(`- **${time}** ${title}${turns}${cats}`); + } + } + + // Hot files + if (ctx.hotFiles.length > 0) { + sections.push(""); + sections.push(`### Hot Files`); + const fileList = ctx.hotFiles + .map((f) => `\`${relativePath(f.filePath, projectPath)}\` (${f.ops} ops)`) + .join(", "); + sections.push(fileList); + } + + // Git activity + if (ctx.gitActivity.length > 0) { + sections.push(""); + sections.push(`### Git Activity`); + for (const g of ctx.gitActivity) { + const date = g.createdAt.slice(0, 10); + const msg = parseGitDetails(g.details); + if (g.operation === "commit") { + const branch = g.branch ? ` \`${g.branch}\`` : ""; + const detail = msg ? `: "${msg}"` : ""; + sections.push(`- commit${branch}${detail} (${date})`); + } else if (g.operation === "pr_create") { + const detail = msg ? `: "${msg}"` : ""; + sections.push(`- pr_create${detail} (${date})`); + } else { + const branch = g.branch ? ` \`${g.branch}\`` : ""; + sections.push(`- ${g.operation}${branch} (${date})`); + } + } + } + + // Errors + if (ctx.errors.length > 0) { + sections.push(""); + sections.push(`### Recent Errors`); + for (const e of ctx.errors) { + const s = e.count === 1 ? "occurrence" : "occurrences"; + sections.push(`- ${e.errorType}: ${e.count} ${s}`); + } + } + + // Usage + if (ctx.usage) { + sections.push(""); + sections.push(`### Usage`); + sections.push( + `${ctx.usage.sessions} sessions, ${ctx.usage.turns} turns, ~${formatTokens(ctx.usage.inputTokens)} input / ~${formatTokens(ctx.usage.outputTokens)} output tokens` + ); + } + + // Check if there's any content beyond the header + const hasContent = + ctx.sessions.length > 0 || + ctx.hotFiles.length > 0 || + ctx.gitActivity.length > 0 || + ctx.errors.length > 0 || + ctx.usage !== null; + + if (!hasContent) { + return ""; + } + + return sections.join("\n"); +} + +// ============================================================================= +// CLAUDE.md Splice Logic +// ============================================================================= + +/** + * Splice a context block into an existing CLAUDE.md string. + * Removes any existing `## Project Context` section. + * Inserts new context after the header, before knowledge index sections. + * Idempotent — running twice produces the same result. + */ +export function spliceContext(existing: string, contextBlock: string): string { + const lines = existing.split("\n"); + const result: string[] = []; + let inContextSection = false; + let headerEndIndex = -1; + + // First pass: find where the header ends (first ## line that isn't Project Context) + // and strip any existing Project Context section + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + if (line.startsWith("## Project Context")) { + inContextSection = true; + continue; + } + + if (inContextSection && line.startsWith("## ")) { + inContextSection = false; + } + + if (inContextSection) { + continue; + } + + result.push(line); + } + + // Find insertion point: after header content, before first ## section + let insertIdx = result.length; + for (let i = 0; i < result.length; i++) { + if (result[i].startsWith("## ")) { + insertIdx = i; + break; + } + } + + // Insert context block with surrounding newlines + if (contextBlock) { + const toInsert = [contextBlock, ""]; + // Ensure there's a blank line before the context block + if (insertIdx > 0 && result[insertIdx - 1] !== "") { + toInsert.unshift(""); + } + result.splice(insertIdx, 0, ...toInsert); + } + + return result.join("\n"); +} + +// ============================================================================= +// Orchestrator +// ============================================================================= + +/** + * Generate project context and optionally write to .smriti/CLAUDE.md. + */ +export async function generateContext( + db: Database, + options: ContextOptions = {} +): Promise<{ + projectId: string; + context: string; + written: boolean; + path: string | null; + tokenEstimate: number; +}> { + const days = options.days || DEFAULT_CONTEXT_DAYS; + + // Detect project + let projectId = options.project || detectProject(db, options.cwd); + if (!projectId) { + throw new Error( + "Could not detect project. Use --project or run from a project directory.\n" + + "Run 'smriti projects' to see registered projects." + ); + } + + // Verify project exists + const project = db + .prepare(`SELECT id, path FROM smriti_projects WHERE id = ?`) + .get(projectId) as { id: string; path: string | null } | null; + + if (!project) { + throw new Error( + `Project '${projectId}' not found. Run 'smriti projects' to see registered projects.` + ); + } + + // Gather and render — prefer cwd over stored path (stored path may have derivation mismatches) + const actualDir = options.cwd || process.cwd(); + const ctx = gatherContext(db, projectId, days); + const contextBlock = renderContext(ctx, projectId, days, actualDir); + const tokenEstimate = contextBlock + ? Math.ceil(contextBlock.length / 4) + : 0; + + if (!contextBlock) { + return { + projectId, + context: `No project context available for '${projectId}'. Run \`smriti ingest\` first.`, + written: false, + path: null, + tokenEstimate: 0, + }; + } + + if (options.dryRun) { + return { + projectId, + context: contextBlock, + written: false, + path: null, + tokenEstimate, + }; + } + + // Write to .smriti/CLAUDE.md + const smritiDir = join(actualDir, SMRITI_DIR); + const claudeMdPath = join(smritiDir, "CLAUDE.md"); + + mkdirSync(smritiDir, { recursive: true }); + + let existing = ""; + try { + existing = await Bun.file(claudeMdPath).text(); + } catch { + // File doesn't exist yet — start with a header + existing = "# Team Knowledge\n\nGenerated by smriti. Do not edit manually.\n"; + } + + const spliced = spliceContext(existing, contextBlock); + await Bun.write(claudeMdPath, spliced); + + return { + projectId, + context: contextBlock, + written: true, + path: claudeMdPath, + tokenEstimate, + }; +} + +// ============================================================================= +// Session Comparison +// ============================================================================= + +export type SessionMetrics = { + id: string; + title: string; + createdAt: string; + turnCount: number; + inputTokens: number; + outputTokens: number; + totalTokens: number; + toolCalls: number; + toolBreakdown: Record; + fileOps: number; + fileReads: number; + fileWrites: number; + errors: number; + durationMs: number; +}; + +export type CompareResult = { + a: SessionMetrics; + b: SessionMetrics; + diff: { + tokens: number; + tokensPct: number; + turns: number; + turnsPct: number; + toolCalls: number; + toolCallsPct: number; + fileReads: number; + fileReadsPct: number; + }; +}; + +/** + * Resolve a partial session ID to a full ID. + * Supports prefix matching (first 8+ chars). + */ +export function resolveSessionId(db: Database, partial: string): string | null { + // Try exact match first + const exact = db + .prepare(`SELECT id FROM memory_sessions WHERE id = ?`) + .get(partial) as { id: string } | null; + if (exact) return exact.id; + + // Prefix match + const prefix = db + .prepare(`SELECT id FROM memory_sessions WHERE id LIKE ? || '%' LIMIT 2`) + .all(partial) as { id: string }[]; + if (prefix.length === 1) return prefix[0].id; + if (prefix.length > 1) return null; // Ambiguous + + return null; +} + +/** + * Get the N most recent session IDs for a project. + */ +export function recentSessionIds( + db: Database, + n: number, + projectId?: string +): string[] { + if (projectId) { + return ( + db + .prepare( + `SELECT ms.id FROM memory_sessions ms + JOIN smriti_session_meta sm ON sm.session_id = ms.id + WHERE sm.project_id = ? + ORDER BY ms.updated_at DESC LIMIT ?` + ) + .all(projectId, n) as { id: string }[] + ).map((r) => r.id); + } + return ( + db + .prepare( + `SELECT id FROM memory_sessions ORDER BY updated_at DESC LIMIT ?` + ) + .all(n) as { id: string }[] + ).map((r) => r.id); +} + +/** + * Gather metrics for a single session from sidecar tables. + */ +export function gatherSessionMetrics( + db: Database, + sessionId: string +): SessionMetrics { + // Session basics + const session = db + .prepare(`SELECT id, title, created_at FROM memory_sessions WHERE id = ?`) + .get(sessionId) as { id: string; title: string; created_at: string }; + + // Costs + const costs = db + .prepare( + `SELECT turn_count, total_input_tokens, total_output_tokens, total_duration_ms + FROM smriti_session_costs WHERE session_id = ?` + ) + .get(sessionId) as { + turn_count: number; + total_input_tokens: number; + total_output_tokens: number; + total_duration_ms: number; + } | null; + + // Tool calls + const toolRows = db + .prepare( + `SELECT tool_name, COUNT(*) AS count + FROM smriti_tool_usage WHERE session_id = ? + GROUP BY tool_name ORDER BY count DESC` + ) + .all(sessionId) as { tool_name: string; count: number }[]; + + const toolBreakdown: Record = {}; + let toolCalls = 0; + for (const row of toolRows) { + toolBreakdown[row.tool_name] = row.count; + toolCalls += row.count; + } + + // File operations + const fileStats = db + .prepare( + `SELECT operation, COUNT(*) AS count + FROM smriti_file_operations WHERE session_id = ? + GROUP BY operation` + ) + .all(sessionId) as { operation: string; count: number }[]; + + let fileOps = 0; + let fileReads = 0; + let fileWrites = 0; + for (const row of fileStats) { + fileOps += row.count; + if (row.operation === "read") fileReads = row.count; + if (row.operation === "write" || row.operation === "edit") + fileWrites += row.count; + } + + // Errors + const errorRow = db + .prepare( + `SELECT COUNT(*) AS count FROM smriti_errors WHERE session_id = ?` + ) + .get(sessionId) as { count: number }; + + return { + id: session.id, + title: session.title || session.id.slice(0, 8), + createdAt: session.created_at, + turnCount: costs?.turn_count || 0, + inputTokens: costs?.total_input_tokens || 0, + outputTokens: costs?.total_output_tokens || 0, + totalTokens: + (costs?.total_input_tokens || 0) + (costs?.total_output_tokens || 0), + toolCalls, + toolBreakdown, + fileOps, + fileReads, + fileWrites, + errors: errorRow.count, + durationMs: costs?.total_duration_ms || 0, + }; +} + +function pctChange(a: number, b: number): number { + if (a === 0) return b === 0 ? 0 : 100; + return ((b - a) / a) * 100; +} + +/** + * Compare two sessions and compute differences. + */ +export function compareSessions( + db: Database, + idA: string, + idB: string +): CompareResult { + const a = gatherSessionMetrics(db, idA); + const b = gatherSessionMetrics(db, idB); + + return { + a, + b, + diff: { + tokens: b.totalTokens - a.totalTokens, + tokensPct: pctChange(a.totalTokens, b.totalTokens), + turns: b.turnCount - a.turnCount, + turnsPct: pctChange(a.turnCount, b.turnCount), + toolCalls: b.toolCalls - a.toolCalls, + toolCallsPct: pctChange(a.toolCalls, b.toolCalls), + fileReads: b.fileReads - a.fileReads, + fileReadsPct: pctChange(a.fileReads, b.fileReads), + }, + }; +} + +/** + * Format comparison result as a readable table. + */ +export function formatCompare(result: CompareResult): string { + const { a, b, diff } = result; + const lines: string[] = []; + + lines.push(`Session A: ${a.title}`); + lines.push(` ${a.id} (${a.createdAt.slice(0, 16)})`); + lines.push(`Session B: ${b.title}`); + lines.push(` ${b.id} (${b.createdAt.slice(0, 16)})`); + lines.push(""); + + // Table header + const pad = (s: string, n: number) => s.padEnd(n); + const rpad = (s: string, n: number) => s.padStart(n); + const fmtDiff = (n: number, pct: number) => { + const sign = n > 0 ? "+" : ""; + const pctStr = pct !== 0 ? ` (${sign}${pct.toFixed(0)}%)` : ""; + return `${sign}${n}${pctStr}`; + }; + + lines.push( + `${pad("Metric", 20)} ${rpad("A", 12)} ${rpad("B", 12)} ${rpad("Diff", 18)}` + ); + lines.push("-".repeat(64)); + + lines.push( + `${pad("Turns", 20)} ${rpad(String(a.turnCount), 12)} ${rpad(String(b.turnCount), 12)} ${rpad(fmtDiff(diff.turns, diff.turnsPct), 18)}` + ); + lines.push( + `${pad("Total tokens", 20)} ${rpad(formatTokens(a.totalTokens), 12)} ${rpad(formatTokens(b.totalTokens), 12)} ${rpad(fmtDiff(diff.tokens, diff.tokensPct), 18)}` + ); + lines.push( + `${pad(" Input", 20)} ${rpad(formatTokens(a.inputTokens), 12)} ${rpad(formatTokens(b.inputTokens), 12)}` + ); + lines.push( + `${pad(" Output", 20)} ${rpad(formatTokens(a.outputTokens), 12)} ${rpad(formatTokens(b.outputTokens), 12)}` + ); + lines.push( + `${pad("Tool calls", 20)} ${rpad(String(a.toolCalls), 12)} ${rpad(String(b.toolCalls), 12)} ${rpad(fmtDiff(diff.toolCalls, diff.toolCallsPct), 18)}` + ); + lines.push( + `${pad("File reads", 20)} ${rpad(String(a.fileReads), 12)} ${rpad(String(b.fileReads), 12)} ${rpad(fmtDiff(diff.fileReads, diff.fileReadsPct), 18)}` + ); + lines.push( + `${pad("File writes", 20)} ${rpad(String(a.fileWrites), 12)} ${rpad(String(b.fileWrites), 12)}` + ); + lines.push( + `${pad("Errors", 20)} ${rpad(String(a.errors), 12)} ${rpad(String(b.errors), 12)}` + ); + + // Tool breakdown + const allTools = new Set([ + ...Object.keys(a.toolBreakdown), + ...Object.keys(b.toolBreakdown), + ]); + if (allTools.size > 0) { + lines.push(""); + lines.push("Tool breakdown:"); + for (const tool of [...allTools].sort()) { + const countA = a.toolBreakdown[tool] || 0; + const countB = b.toolBreakdown[tool] || 0; + if (countA > 0 || countB > 0) { + lines.push( + ` ${pad(tool, 18)} ${rpad(String(countA), 12)} ${rpad(String(countB), 12)}` + ); + } + } + } + + return lines.join("\n"); +} diff --git a/src/db.ts b/src/db.ts index 2d3f0a0..a563241 100644 --- a/src/db.ts +++ b/src/db.ts @@ -7,6 +7,7 @@ import { Database } from "bun:sqlite"; import { QMD_DB_PATH } from "./config"; +import { initializeMemoryTables } from "./qmd"; // ============================================================================= // Connection @@ -106,7 +107,78 @@ export function initializeSmritiTables(db: Database): void { content_hash TEXT ); - -- Indexes + -- Tool usage tracking + CREATE TABLE IF NOT EXISTS smriti_tool_usage ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id INTEGER NOT NULL, + session_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + input_summary TEXT, + success INTEGER DEFAULT 1, + duration_ms INTEGER, + created_at TEXT NOT NULL, + FOREIGN KEY (message_id) REFERENCES memory_messages(id) + ); + + -- File operation tracking + CREATE TABLE IF NOT EXISTS smriti_file_operations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id INTEGER NOT NULL, + session_id TEXT NOT NULL, + operation TEXT NOT NULL, + file_path TEXT NOT NULL, + project_id TEXT, + created_at TEXT NOT NULL + ); + + -- Command execution tracking + CREATE TABLE IF NOT EXISTS smriti_commands ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id INTEGER NOT NULL, + session_id TEXT NOT NULL, + command TEXT NOT NULL, + exit_code INTEGER, + cwd TEXT, + is_git INTEGER DEFAULT 0, + created_at TEXT NOT NULL + ); + + -- Error tracking + CREATE TABLE IF NOT EXISTS smriti_errors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id INTEGER NOT NULL, + session_id TEXT NOT NULL, + error_type TEXT NOT NULL, + message TEXT, + created_at TEXT NOT NULL + ); + + -- Token/cost tracking per session + CREATE TABLE IF NOT EXISTS smriti_session_costs ( + session_id TEXT PRIMARY KEY, + model TEXT, + total_input_tokens INTEGER DEFAULT 0, + total_output_tokens INTEGER DEFAULT 0, + total_cache_tokens INTEGER DEFAULT 0, + estimated_cost_usd REAL DEFAULT 0, + turn_count INTEGER DEFAULT 0, + total_duration_ms INTEGER DEFAULT 0 + ); + + -- Git operation tracking + CREATE TABLE IF NOT EXISTS smriti_git_operations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + message_id INTEGER NOT NULL, + session_id TEXT NOT NULL, + operation TEXT NOT NULL, + branch TEXT, + pr_url TEXT, + pr_number INTEGER, + details TEXT, + created_at TEXT NOT NULL + ); + + -- Indexes (original) CREATE INDEX IF NOT EXISTS idx_smriti_session_meta_agent ON smriti_session_meta(agent_id); CREATE INDEX IF NOT EXISTS idx_smriti_session_meta_project @@ -117,6 +189,28 @@ export function initializeSmritiTables(db: Database): void { ON smriti_session_tags(category_id); CREATE INDEX IF NOT EXISTS idx_smriti_shares_hash ON smriti_shares(content_hash); + + -- Indexes (sidecar tables) + CREATE INDEX IF NOT EXISTS idx_smriti_tool_usage_session + ON smriti_tool_usage(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_tool_usage_tool_name + ON smriti_tool_usage(tool_name); + CREATE INDEX IF NOT EXISTS idx_smriti_file_operations_session + ON smriti_file_operations(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_file_operations_path + ON smriti_file_operations(file_path); + CREATE INDEX IF NOT EXISTS idx_smriti_commands_session + ON smriti_commands(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_commands_is_git + ON smriti_commands(is_git); + CREATE INDEX IF NOT EXISTS idx_smriti_errors_session + ON smriti_errors(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_errors_type + ON smriti_errors(error_type); + CREATE INDEX IF NOT EXISTS idx_smriti_git_operations_session + ON smriti_git_operations(session_id); + CREATE INDEX IF NOT EXISTS idx_smriti_git_operations_op + ON smriti_git_operations(operation); `); } @@ -144,6 +238,12 @@ const DEFAULT_AGENTS = [ log_pattern: ".cursor/**/*.json", parser: "cursor", }, + { + id: "antigravity", + display_name: "Antigravity IDE", + log_pattern: "~/.gemini/antigravity/brain/*/*.md", + parser: "antigravity", + }, ] as const; /** Default category taxonomy */ @@ -225,6 +325,7 @@ export function seedDefaults(db: Database): void { /** Initialize DB, create tables, seed defaults. Returns the DB instance. */ export function initSmriti(dbPath?: string): Database { const db = getDb(dbPath); + initializeMemoryTables(db); initializeSmritiTables(db); seedDefaults(db); return db; @@ -356,3 +457,107 @@ export function listAgents(db: Database): Array<{ }> { return db.prepare(`SELECT * FROM smriti_agents ORDER BY id`).all() as any; } + +// ============================================================================= +// Sidecar Table Insert Helpers +// ============================================================================= + +export function insertToolUsage( + db: Database, + messageId: number, + sessionId: string, + toolName: string, + inputSummary: string | null, + success: boolean, + durationMs: number | null, + createdAt: string +): void { + db.prepare( + `INSERT INTO smriti_tool_usage (message_id, session_id, tool_name, input_summary, success, duration_ms, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ).run(messageId, sessionId, toolName, inputSummary, success ? 1 : 0, durationMs, createdAt); +} + +export function insertFileOperation( + db: Database, + messageId: number, + sessionId: string, + operation: string, + filePath: string, + projectId: string | null, + createdAt: string +): void { + db.prepare( + `INSERT INTO smriti_file_operations (message_id, session_id, operation, file_path, project_id, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ).run(messageId, sessionId, operation, filePath, projectId, createdAt); +} + +export function insertCommand( + db: Database, + messageId: number, + sessionId: string, + command: string, + exitCode: number | null, + cwd: string | null, + isGit: boolean, + createdAt: string +): void { + db.prepare( + `INSERT INTO smriti_commands (message_id, session_id, command, exit_code, cwd, is_git, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ).run(messageId, sessionId, command, exitCode, cwd, isGit ? 1 : 0, createdAt); +} + +export function insertError( + db: Database, + messageId: number, + sessionId: string, + errorType: string, + message: string, + createdAt: string +): void { + db.prepare( + `INSERT INTO smriti_errors (message_id, session_id, error_type, message, created_at) + VALUES (?, ?, ?, ?, ?)` + ).run(messageId, sessionId, errorType, message, createdAt); +} + +export function upsertSessionCosts( + db: Database, + sessionId: string, + model: string | null, + inputTokens: number, + outputTokens: number, + cacheTokens: number, + durationMs: number +): void { + db.prepare( + `INSERT INTO smriti_session_costs (session_id, model, total_input_tokens, total_output_tokens, total_cache_tokens, turn_count, total_duration_ms) + VALUES (?, ?, ?, ?, ?, 1, ?) + ON CONFLICT(session_id) DO UPDATE SET + model = COALESCE(excluded.model, model), + total_input_tokens = total_input_tokens + excluded.total_input_tokens, + total_output_tokens = total_output_tokens + excluded.total_output_tokens, + total_cache_tokens = total_cache_tokens + excluded.total_cache_tokens, + turn_count = turn_count + 1, + total_duration_ms = total_duration_ms + excluded.total_duration_ms` + ).run(sessionId, model, inputTokens, outputTokens, cacheTokens, durationMs); +} + +export function insertGitOperation( + db: Database, + messageId: number, + sessionId: string, + operation: string, + branch: string | null, + prUrl: string | null, + prNumber: number | null, + details: string | null, + createdAt: string +): void { + db.prepare( + `INSERT INTO smriti_git_operations (message_id, session_id, operation, branch, pr_url, pr_number, details, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run(messageId, sessionId, operation, branch, prUrl, prNumber, details, createdAt); +} diff --git a/src/index.ts b/src/index.ts index 0d8ebf1..7830f53 100644 --- a/src/index.ts +++ b/src/index.ts @@ -16,6 +16,13 @@ import { searchFiltered, listSessions } from "./search/index"; import { recall } from "./search/recall"; import { shareKnowledge } from "./team/share"; import { syncTeamKnowledge, listTeamContributions } from "./team/sync"; +import { + generateContext, + compareSessions, + resolveSessionId, + recentSessionIds, + formatCompare, +} from "./context"; import { formatSessionList, formatSearchResults, @@ -74,6 +81,9 @@ Commands: tag Manually tag a session categories List category tree categories add [opts] Add a custom category + context [options] Generate project context for .smriti/CLAUDE.md + compare Compare two sessions (tokens, tools, files) + compare --last Compare last 2 sessions for current project share [filters] Export knowledge to .smriti/ sync Import team knowledge from .smriti/ team View team contributions @@ -94,6 +104,7 @@ Ingest options: smriti ingest claude Ingest Claude Code sessions smriti ingest codex Ingest Codex CLI sessions smriti ingest cursor --project-path + smriti ingest antigravity Ingest Antigravity IDE sessions smriti ingest file [--format chat|jsonl] [--title ] smriti ingest all Ingest from all known agents @@ -102,9 +113,16 @@ Recall options: --model Ollama model for synthesis --max-tokens Max synthesis tokens +Context options: + --project Project filter (auto-detect from cwd) + --days Lookback window (default: 7) + --dry-run Print to stdout, don't write file + Share options: --session Share specific session --output Custom output directory + --no-reflect Skip LLM reflections (on by default) + --reflect-model Ollama model for reflections Examples: smriti ingest claude @@ -137,7 +155,7 @@ async function main() { const agent = args[1]; if (!agent) { console.error("Usage: smriti ingest "); - console.error("Agents: claude, codex, cursor, file, all"); + console.error("Agents: claude, codex, cursor, antigravity, file, all"); process.exit(1); } @@ -301,6 +319,81 @@ async function main() { break; } + // ===================================================================== + // CONTEXT + // ===================================================================== + case "context": { + const result = await generateContext(db, { + project: getArg(args, "--project"), + days: Number(getArg(args, "--days")) || undefined, + dryRun: hasFlag(args, "--dry-run"), + json: hasFlag(args, "--json"), + }); + + if (hasFlag(args, "--json")) { + console.log(json(result)); + } else if (result.written) { + console.log(result.context); + console.log(`\nWritten to ${result.path} (~${result.tokenEstimate} tokens)`); + } else { + console.log(result.context); + if (result.tokenEstimate > 0) { + console.log(`\n~${result.tokenEstimate} tokens`); + } + } + break; + } + + // ===================================================================== + // COMPARE + // ===================================================================== + case "compare": { + let idA: string | null = null; + let idB: string | null = null; + + if (hasFlag(args, "--last")) { + // Compare last 2 sessions for the detected project + const projectId = getArg(args, "--project") || (() => { + const { detectProject } = require("./context"); + return detectProject(db); + })(); + const recent = recentSessionIds(db, 2, projectId); + if (recent.length < 2) { + console.error("Need at least 2 sessions to compare. Run 'smriti ingest' first."); + process.exit(1); + } + idA = recent[1]; // older + idB = recent[0]; // newer + } else { + const rawA = args[1]; + const rawB = args[2]; + if (!rawA || !rawB) { + console.error("Usage: smriti compare "); + console.error(" smriti compare --last [--project ]"); + process.exit(1); + } + idA = resolveSessionId(db, rawA); + idB = resolveSessionId(db, rawB); + if (!idA) { + console.error(`Could not resolve session: ${rawA}`); + process.exit(1); + } + if (!idB) { + console.error(`Could not resolve session: ${rawB}`); + process.exit(1); + } + } + + const result = compareSessions(db, idA!, idB!); + + if (hasFlag(args, "--json")) { + console.log(json(result)); + } else { + console.log(formatCompare(result)); + } + break; + } + // ===================================================================== // SHARE // ===================================================================== @@ -310,6 +403,8 @@ async function main() { project: getArg(args, "--project"), sessionId: getArg(args, "--session"), outputDir: getArg(args, "--output"), + reflect: !hasFlag(args, "--no-reflect"), + reflectModel: getArg(args, "--reflect-model"), }); console.log(formatShareResult(result)); diff --git a/src/ingest/antigravity.ts b/src/ingest/antigravity.ts new file mode 100644 index 0000000..9bf2ea8 --- /dev/null +++ b/src/ingest/antigravity.ts @@ -0,0 +1,161 @@ +/** + * antigravity.ts - Antigravity IDE artifact ingestion + * + * Scans ~/.gemini/antigravity/brain for task artifacts and ingest them + * as structured sessions. + */ + +import { existsSync, statSync } from "fs"; +import { join, basename } from "path"; +import { ANTIGRAVITY_BRAIN_DIR, PROJECTS_ROOT } from "../config"; +import { addMessage } from "../qmd"; +import type { IngestResult, IngestOptions } from "./index"; +import type { StructuredMessage, MessageBlock } from "./types"; +import { extractBlocks } from "./blocks"; // Reuse block extractor if helpful, or build simple text blocks + +// ============================================================================= +// Session Discovery +// ============================================================================= + +export async function discoverAntigravitySessions( + brainDir?: string +): Promise> { + const dir = brainDir || ANTIGRAVITY_BRAIN_DIR; + if (!existsSync(dir)) return []; + + const glob = new Bun.Glob("*"); + const sessions: Array<{ sessionId: string; dirPath: string }> = []; + + for await (const match of glob.scan({ cwd: dir, absolute: false, onlyFiles: false })) { + const fullPath = join(dir, match); + if (statSync(fullPath).isDirectory()) { + // Validation: expecting UUID-like directories + // But let's just take all directories for now + sessions.push({ + sessionId: match, + dirPath: fullPath + }); + } + } + + return sessions; +} + +// ============================================================================= +// Ingestion +// ============================================================================= + +export async function ingestAntigravity( + options: IngestOptions = {} +): Promise { + const { db, existingSessionIds, onProgress } = options; + if (!db) throw new Error("Database required for ingestion"); + + const { upsertProject, upsertSessionMeta } = await import("../db"); + + const sessions = await discoverAntigravitySessions(options.logsDir); + const result: IngestResult = { + agent: "antigravity", + sessionsFound: sessions.length, + sessionsIngested: 0, + messagesIngested: 0, + skipped: 0, + errors: [], + }; + + for (const session of sessions) { + if (existingSessionIds?.has(session.sessionId)) { + result.skipped++; + continue; + } + + try { + const messages: StructuredMessage[] = []; + const sessionDir = session.dirPath; + + // 1. Task (User) + const taskPath = join(sessionDir, "task.md"); + const taskContent = existsSync(taskPath) ? await Bun.file(taskPath).text() : null; + + // 2. Implementation Plan (Assistant) + const planPath = join(sessionDir, "implementation_plan.md"); + const planContent = existsSync(planPath) ? await Bun.file(planPath).text() : null; + + // 3. Walkthrough (Assistant) + const walkthroughPath = join(sessionDir, "walkthrough.md"); + const walkthroughContent = existsSync(walkthroughPath) ? await Bun.file(walkthroughPath).text() : null; + + if (!taskContent && !planContent && !walkthroughContent) { + result.skipped++; // Empty or unrecognized session structure + continue; + } + + const timestamp = new Date().toISOString(); // Fallback if file times aren't used + + // Helper to build message + const addMsg = (role: "user" | "assistant", content: string, type: string, seq: number) => { + messages.push({ + id: `${session.sessionId}-${type}`, + sessionId: session.sessionId, + sequence: seq, + timestamp, + role, + agent: "antigravity", + blocks: [{ type: "text", text: content }] as MessageBlock[], + metadata: { sourceFile: type } as any, + plainText: content + }); + }; + + if (taskContent) addMsg("user", taskContent, "task", 0); + if (planContent) addMsg("assistant", `## Implementation Plan\n\n${planContent}`, "plan", 1); + if (walkthroughContent) addMsg("assistant", `## Walkthrough\n\n${walkthroughContent}`, "walkthrough", 2); + + // Derive Project ID + // Antigravity doesn't explicitly link to project path in brain dir structure usually + // We might fallback to a generic project or try to heuristically find it from task text? + // For now, let's use "antigravity" or try to find a mention. + // Actually, check if there is a way to find project. + // If not, put in "antigravity-brain" project. + const projectId = "antigravity-workspace"; + upsertProject(db, projectId, ANTIGRAVITY_BRAIN_DIR, undefined); + + // Extract title from task + const title = taskContent + ? taskContent.split("\n")[0].replace(/^#+\s*/, "").slice(0, 100) + : `Antigravity Session ${session.sessionId.slice(0,8)}`; + + // Save to QMD + for (const msg of messages) { + await addMessage( + db, + session.sessionId, + msg.role, + msg.plainText, + { + title, + metadata: msg.metadata + } + ); + } + + // Upsert Session Meta + upsertSessionMeta(db, session.sessionId, "antigravity", projectId); + + result.sessionsIngested++; + result.messagesIngested += messages.length; + + if (onProgress) { + onProgress(`Ingested ${session.sessionId} (${messages.length} artifacts)`); + } + + } catch (err: any) { + result.errors.push(`${session.sessionId}: ${err.message}`); + } + } + + return result; +} diff --git a/src/ingest/blocks.ts b/src/ingest/blocks.ts new file mode 100644 index 0000000..974cfc9 --- /dev/null +++ b/src/ingest/blocks.ts @@ -0,0 +1,513 @@ +/** + * blocks.ts - Tool call → structured block extraction + * + * Maps raw Claude Code content blocks (tool_use, text, thinking) into + * domain-specific MessageBlock types. Also parses git commands from Bash. + */ + +import type { + MessageBlock, + TextBlock, + ThinkingBlock, + ToolCallBlock, + ToolResultBlock, + FileOperationBlock, + CommandBlock, + SearchBlock, + GitBlock, + ErrorBlock, + ImageBlock, + ConversationControlBlock, + SystemEventBlock, + STORAGE_LIMITS as StorageLimitsType, +} from "./types"; +import { STORAGE_LIMITS } from "./types"; + +// ============================================================================= +// Raw content block shape (from Claude API / JSONL) +// ============================================================================= + +export type RawContentBlock = { + type: string; + text?: string; + thinking?: string; + signature?: string; + id?: string; + name?: string; + input?: Record; + tool_use_id?: string; + content?: string | RawContentBlock[]; + source?: { type: string; media_type: string; data: string }; +}; + +// ============================================================================= +// Truncation helper +// ============================================================================= + +function truncate(s: string | undefined, limit: number): string { + if (!s) return ""; + return s.length > limit ? s.slice(0, limit) + "...[truncated]" : s; +} + +// ============================================================================= +// Git command detection & parsing +// ============================================================================= + +const GIT_COMMAND_RE = /^\s*git\s+/; +const GIT_OP_MAP: Record = { + commit: "commit", + push: "push", + pull: "pull", + branch: "branch", + checkout: "checkout", + switch: "checkout", + diff: "diff", + merge: "merge", + rebase: "rebase", + status: "status", +}; + +export function isGitCommand(command: string): boolean { + return GIT_COMMAND_RE.test(command); +} + +export function parseGitCommand(command: string): GitBlock | null { + if (!isGitCommand(command)) return null; + + // Extract the git subcommand + const match = command.match(/^\s*git\s+(\S+)/); + if (!match) return null; + + const subcommand = match[1]; + const operation = GIT_OP_MAP[subcommand] || "other"; + + const block: GitBlock = { + type: "git", + operation, + }; + + // Parse commit message + if (operation === "commit") { + const msgMatch = command.match(/-m\s+["']([^"']+)["']/); + if (!msgMatch) { + // Try heredoc style: -m "$(cat <<'EOF'\n...\nEOF\n)" + const heredocMatch = command.match(/-m\s+"\$\(cat\s+<<'?EOF'?\n([\s\S]*?)\nEOF/); + if (heredocMatch) block.message = heredocMatch[1].trim(); + } else { + block.message = msgMatch[1]; + } + } + + // Parse branch from checkout/switch + if (operation === "checkout" || operation === "branch") { + const parts = command.trim().split(/\s+/); + const lastPart = parts[parts.length - 1]; + if (lastPart && lastPart !== subcommand && !lastPart.startsWith("-")) { + block.branch = lastPart; + } + } + + // Parse push branch + if (operation === "push") { + const pushMatch = command.match(/push\s+\S+\s+(\S+)/); + if (pushMatch) block.branch = pushMatch[1]; + } + + return block; +} + +/** + * Detect gh pr create commands and extract PR info. + */ +export function parseGhPrCommand(command: string): GitBlock | null { + if (!command.match(/^\s*gh\s+pr\s+create/)) return null; + + const block: GitBlock = { + type: "git", + operation: "pr_create", + }; + + const titleMatch = command.match(/--title\s+["']([^"']+)["']/); + if (titleMatch) block.message = titleMatch[1]; + + return block; +} + +// ============================================================================= +// Tool call → domain-specific blocks +// ============================================================================= + +/** + * Convert a tool_use content block into one or more domain-specific blocks. + * The raw ToolCallBlock is always included; domain blocks are added alongside. + */ +export function toolCallToBlocks( + toolName: string, + toolId: string, + input: Record, + description?: string +): MessageBlock[] { + const blocks: MessageBlock[] = []; + + // Always emit the generic tool call block + const toolCall: ToolCallBlock = { + type: "tool_call", + toolId, + toolName, + input: truncateInputFields(input), + description, + }; + blocks.push(toolCall); + + // Then emit domain-specific blocks + switch (toolName) { + case "Read": { + const fileOp: FileOperationBlock = { + type: "file_op", + operation: "read", + path: input.file_path || "", + }; + blocks.push(fileOp); + break; + } + case "Write": { + const fileOp: FileOperationBlock = { + type: "file_op", + operation: "write", + path: input.file_path || "", + }; + blocks.push(fileOp); + break; + } + case "Edit": { + const fileOp: FileOperationBlock = { + type: "file_op", + operation: "edit", + path: input.file_path || "", + diff: input.old_string && input.new_string + ? `- ${truncate(input.old_string, 500)}\n+ ${truncate(input.new_string, 500)}` + : undefined, + }; + blocks.push(fileOp); + break; + } + case "NotebookEdit": { + const fileOp: FileOperationBlock = { + type: "file_op", + operation: "edit", + path: input.notebook_path || "", + }; + blocks.push(fileOp); + break; + } + case "Glob": { + const fileOp: FileOperationBlock = { + type: "file_op", + operation: "glob", + path: input.path || "", + pattern: input.pattern, + }; + blocks.push(fileOp); + const search: SearchBlock = { + type: "search", + searchType: "glob", + pattern: input.pattern || "", + path: input.path, + }; + blocks.push(search); + break; + } + case "Grep": { + const search: SearchBlock = { + type: "search", + searchType: "grep", + pattern: input.pattern || "", + path: input.path, + }; + blocks.push(search); + break; + } + case "Bash": { + const command = input.command || ""; + const cmdBlock: CommandBlock = { + type: "command", + command, + cwd: input.cwd, + description: input.description, + isGit: isGitCommand(command), + }; + blocks.push(cmdBlock); + + // Also parse git operations + const gitBlock = parseGitCommand(command) || parseGhPrCommand(command); + if (gitBlock) blocks.push(gitBlock); + break; + } + case "WebFetch": { + const search: SearchBlock = { + type: "search", + searchType: "web_fetch", + pattern: input.prompt || "", + url: input.url, + }; + blocks.push(search); + break; + } + case "WebSearch": { + const search: SearchBlock = { + type: "search", + searchType: "web_search", + pattern: input.query || "", + }; + blocks.push(search); + break; + } + case "EnterPlanMode": { + const ctrl: ConversationControlBlock = { + type: "control", + controlType: "plan_enter", + }; + blocks.push(ctrl); + break; + } + case "ExitPlanMode": { + const ctrl: ConversationControlBlock = { + type: "control", + controlType: "plan_exit", + }; + blocks.push(ctrl); + break; + } + case "Skill": { + const ctrl: ConversationControlBlock = { + type: "control", + controlType: "slash_command", + command: input.skill, + }; + blocks.push(ctrl); + break; + } + // Task, TaskCreate, TaskList, TaskOutput, TaskUpdate, TodoWrite, + // AskUserQuestion, KillShell — kept as generic ToolCallBlock only + } + + return blocks; +} + +/** + * Parse a tool_result content block into a ToolResultBlock. + */ +export function parseToolResult( + toolUseId: string, + content: string | RawContentBlock[] | undefined, + isError?: boolean +): ToolResultBlock { + let output = ""; + + if (typeof content === "string") { + output = content; + } else if (Array.isArray(content)) { + output = content + .filter((b) => b.type === "text" && b.text) + .map((b) => b.text!) + .join("\n"); + } + + return { + type: "tool_result", + toolId: toolUseId, + success: !isError, + output: truncate(output, STORAGE_LIMITS.commandOutput), + error: isError ? truncate(output, STORAGE_LIMITS.commandOutput) : undefined, + }; +} + +// ============================================================================= +// Content block → MessageBlock conversion +// ============================================================================= + +/** + * Convert a raw content block (from Claude API format) into MessageBlock(s). + */ +export function rawBlockToMessageBlocks(raw: RawContentBlock): MessageBlock[] { + switch (raw.type) { + case "text": + return [ + { + type: "text", + text: truncate(raw.text, STORAGE_LIMITS.textBlock), + } as TextBlock, + ]; + + case "thinking": + return [ + { + type: "thinking", + thinking: truncate(raw.thinking, STORAGE_LIMITS.thinkingBlock), + } as ThinkingBlock, + ]; + + case "tool_use": + return toolCallToBlocks( + raw.name || "unknown", + raw.id || "", + raw.input || {}, + raw.input?.description + ); + + case "tool_result": + return [ + parseToolResult( + raw.tool_use_id || "", + raw.content, + false + ), + ]; + + case "image": + return [ + { + type: "image", + mediaType: raw.source?.media_type || "image/png", + dataHash: raw.source?.data + ? hashQuick(raw.source.data) + : undefined, + } as ImageBlock, + ]; + + default: + // Unknown block type — wrap as text if there's content + if (raw.text) { + return [{ type: "text", text: raw.text } as TextBlock]; + } + return []; + } +} + +/** + * Convert an array of raw content blocks into MessageBlock[]. + */ +export function extractBlocks( + content: string | RawContentBlock[] +): MessageBlock[] { + if (typeof content === "string") { + if (!content.trim()) return []; + return [{ type: "text", text: truncate(content, STORAGE_LIMITS.textBlock) } as TextBlock]; + } + + if (!Array.isArray(content)) return []; + + const blocks: MessageBlock[] = []; + for (const raw of content) { + blocks.push(...rawBlockToMessageBlocks(raw)); + } + return blocks; +} + +/** + * Flatten MessageBlock[] into plain text for backward-compatible FTS indexing. + */ +export function flattenBlocksToText(blocks: MessageBlock[]): string { + const parts: string[] = []; + + for (const block of blocks) { + switch (block.type) { + case "text": + parts.push(block.text); + break; + case "command": + if (block.description) parts.push(block.description); + parts.push(`$ ${block.command}`); + break; + case "file_op": + parts.push(`[${block.operation}] ${block.path}`); + break; + case "search": + parts.push(`[${block.searchType}] ${block.pattern}`); + break; + case "git": + if (block.message) parts.push(`[git ${block.operation}] ${block.message}`); + else parts.push(`[git ${block.operation}]`); + break; + case "error": + parts.push(`[error:${block.errorType}] ${block.message}`); + break; + case "system_event": + // Don't include system events in FTS text + break; + case "control": + // Don't include control blocks in FTS text + break; + case "tool_call": + if (block.description) parts.push(block.description); + break; + case "tool_result": + // Don't index full tool output — too noisy + break; + case "thinking": + // Don't index thinking blocks in FTS — optional deep search + break; + case "image": + if (block.description) parts.push(block.description); + break; + case "code": + parts.push(block.code); + break; + } + } + + return parts.filter(Boolean).join("\n"); +} + +/** + * Convert a Claude system entry into a SystemEventBlock. + */ +export function systemEntryToBlock( + type: string, + data: Record +): SystemEventBlock { + const eventTypeMap: Record = { + "turn_duration": "turn_duration", + "pr-link": "pr_link", + "file-history-snapshot": "file_snapshot", + "progress": "session_start", // approximate + }; + + return { + type: "system_event", + eventType: eventTypeMap[type] || "session_start", + data, + }; +} + +// ============================================================================= +// Helpers +// ============================================================================= + +/** + * Quick non-crypto hash for image data references (not for dedup). + */ +function hashQuick(data: string): string { + let hash = 0; + for (let i = 0; i < Math.min(data.length, 1000); i++) { + hash = ((hash << 5) - hash + data.charCodeAt(i)) | 0; + } + return `img_${Math.abs(hash).toString(36)}`; +} + +/** + * Truncate all string fields in a tool input object. + */ +function truncateInputFields( + input: Record +): Record { + const result: Record = {}; + for (const [key, value] of Object.entries(input)) { + if (typeof value === "string") { + result[key] = truncate(value, STORAGE_LIMITS.toolInput); + } else { + result[key] = value; + } + } + return result; +} diff --git a/src/ingest/claude.ts b/src/ingest/claude.ts index cacb08e..c3d9398 100644 --- a/src/ingest/claude.ts +++ b/src/ingest/claude.ts @@ -1,30 +1,70 @@ /** - * claude.ts - Claude Code conversation parser + * claude.ts - Claude Code conversation parser (enriched) * - * Reads JSONL transcripts from ~/.claude/projects/ and normalizes - * to QMD's addMessage() format with agent and project metadata. + * Reads JSONL transcripts from ~/.claude/projects/ and produces + * StructuredMessage objects with full block extraction, then stores + * via QMD's addMessage() with sidecar table population. */ import { existsSync } from "fs"; import { basename } from "path"; import { CLAUDE_LOGS_DIR, PROJECTS_ROOT } from "../config"; import { addMessage } from "../qmd"; -import type { ParsedMessage, IngestResult, IngestOptions } from "./index"; - -/** Shape of a Claude Code JSONL entry */ +import type { ParsedMessage, StructuredMessage, MessageMetadata } from "./types"; +import type { IngestResult, IngestOptions } from "./index"; +import type { MessageBlock } from "./types"; +import { + extractBlocks, + flattenBlocksToText, + systemEntryToBlock, + type RawContentBlock, +} from "./blocks"; + +// ============================================================================= +// Raw JSONL entry types (expanded) +// ============================================================================= + +/** Full shape of a Claude Code JSONL entry */ type ClaudeEntry = { - type: "user" | "assistant" | "file-history-snapshot" | string; + type: "user" | "assistant" | "system" | "file-history-snapshot" | "pr-link" | "progress" | "queue-operation" | string; + subtype?: string; sessionId?: string; + uuid?: string; + parentUuid?: string; + timestamp?: string; cwd?: string; + gitBranch?: string; + version?: string; + slug?: string; + permissionMode?: string; + isSidechain?: boolean; + isMeta?: boolean; + requestId?: string; message?: { role: string; - content: string | Array<{ type: string; text?: string; thinking?: string }>; + model?: string; + id?: string; + type?: string; + content: string | RawContentBlock[]; + stop_reason?: string | null; + usage?: { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; + }; }; - isMeta?: boolean; - timestamp?: string; - uuid?: string; + // System event fields + durationMs?: number; + prNumber?: number; + prUrl?: string; + prRepository?: string; }; +// ============================================================================= +// Legacy extractContent (for backward-compat ParsedMessage) +// ============================================================================= + /** * Extract text content from a Claude message content field. * Content can be a string or an array of content blocks. @@ -41,14 +81,12 @@ function extractContent( .join("\n"); } +// ============================================================================= +// Path resolution (unchanged) +// ============================================================================= + /** * Reconstruct a real filesystem path from a Claude projects directory name. - * - * Claude encodes paths by replacing "/" with "-", but folder names can also - * contain "-". We greedily match from the left, picking the longest existing - * directory segment at each step. - * - * e.g. "-Users-zero8-zero8.dev-openfga" -> "/Users/zero8/zero8.dev/openfga" */ export function deriveProjectPath(dirName: string): string { const raw = dirName.replace(/^-/, ""); @@ -57,7 +95,6 @@ export function deriveProjectPath(dirName: string): string { const segments: string[] = []; let i = 0; while (i < parts.length) { - // Greedily try to join as many parts as possible into one segment let best = parts[i]; let bestLen = 1; for (let j = i + 1; j < parts.length; j++) { @@ -77,38 +114,180 @@ export function deriveProjectPath(dirName: string): string { /** * Derive a project ID from a Claude projects directory name. - * - * Uses PROJECTS_ROOT to strip the known prefix and return just the - * project-relative portion. - * - * e.g. with PROJECTS_ROOT="/Users/zero8/zero8.dev": - * "-Users-zero8-zero8.dev-openfga" -> "openfga" - * "-Users-zero8-zero8.dev-avkash-regulation-hub" -> "avkash/regulation-hub" - * "-Users-zero8-zero8.dev" -> "zero8.dev" (the root itself) - * "-Users-zero8" -> "home" (outside projects root) */ export function deriveProjectId(dirName: string): string { const realPath = deriveProjectPath(dirName); - - // Normalize: strip trailing slashes for comparison const root = PROJECTS_ROOT.replace(/\/+$/, ""); if (realPath === root) { - // The projects root directory itself return basename(root); } if (realPath.startsWith(root + "/")) { - // Inside projects root - return the relative path return realPath.slice(root.length + 1); } - // Outside projects root - fallback return basename(realPath) || "home"; } +// ============================================================================= +// Structured parsing +// ============================================================================= + +/** + * Parse a single Claude Code JSONL entry into a StructuredMessage. + * Returns null for entries that should be skipped (meta, empty, etc.) + */ +function parseEntry( + entry: ClaudeEntry, + sequence: number +): StructuredMessage | null { + // Handle system events — these produce system-role messages + if (entry.type === "system" && entry.subtype === "turn_duration") { + return { + id: entry.uuid || `sys-${sequence}`, + sessionId: entry.sessionId || "", + sequence, + timestamp: entry.timestamp || new Date().toISOString(), + role: "system", + agent: "claude-code", + blocks: [ + systemEntryToBlock("turn_duration", { + durationMs: entry.durationMs, + }), + ], + metadata: {}, + plainText: "", + }; + } + + if (entry.type === "pr-link") { + return { + id: entry.uuid || `pr-${sequence}`, + sessionId: entry.sessionId || "", + sequence, + timestamp: entry.timestamp || new Date().toISOString(), + role: "system", + agent: "claude-code", + blocks: [ + systemEntryToBlock("pr-link", { + prNumber: entry.prNumber, + prUrl: entry.prUrl, + prRepository: entry.prRepository, + }), + { + type: "git", + operation: "pr_create", + prUrl: entry.prUrl, + prNumber: entry.prNumber, + }, + ], + metadata: {}, + plainText: entry.prUrl ? `[PR #${entry.prNumber}] ${entry.prUrl}` : "", + }; + } + + // Skip non-message entries + if (entry.type !== "user" && entry.type !== "assistant") { + return null; + } + + // Skip meta messages (hooks, commands) + if (entry.isMeta) return null; + + // Must have message content + if (!entry.message?.content) return null; + + // Extract blocks from content + const blocks = extractBlocks(entry.message.content as string | RawContentBlock[]); + if (blocks.length === 0) return null; + + // Compute plain text for FTS + const plainText = flattenBlocksToText(blocks); + + // Skip system/command content (only for text-only messages) + if ( + blocks.length === 1 && + blocks[0].type === "text" && + (blocks[0].text.startsWith("")) + ) { + return null; + } + + // Build metadata + const metadata: MessageMetadata = {}; + if (entry.cwd) metadata.cwd = entry.cwd; + if (entry.gitBranch) metadata.gitBranch = entry.gitBranch; + if (entry.version) metadata.agentVersion = entry.version; + if (entry.parentUuid) metadata.parentId = entry.parentUuid; + if (entry.isSidechain) metadata.isSidechain = true; + if (entry.permissionMode) metadata.permissionMode = entry.permissionMode; + if (entry.slug) metadata.slug = entry.slug; + if (entry.requestId) metadata.requestId = entry.requestId; + + // Assistant-specific metadata + if (entry.type === "assistant" && entry.message) { + if (entry.message.model) metadata.model = entry.message.model; + if (entry.message.stop_reason) metadata.stopReason = entry.message.stop_reason; + if (entry.message.usage) { + metadata.tokenUsage = { + input: entry.message.usage.input_tokens || 0, + output: entry.message.usage.output_tokens || 0, + cacheCreate: entry.message.usage.cache_creation_input_tokens, + cacheRead: entry.message.usage.cache_read_input_tokens, + }; + } + } + + const role = (entry.message?.role || entry.type) as StructuredMessage["role"]; + + return { + id: entry.uuid || `msg-${sequence}`, + sessionId: entry.sessionId || "", + sequence, + timestamp: entry.timestamp || new Date().toISOString(), + role: role === "user" || role === "assistant" ? role : "user", + agent: "claude-code", + blocks, + metadata, + plainText, + }; +} + +// ============================================================================= +// Public API: Parse JSONL → StructuredMessage[] +// ============================================================================= + +/** + * Parse a Claude Code JSONL file into StructuredMessages. + */ +export function parseClaudeJsonlStructured(content: string): StructuredMessage[] { + const messages: StructuredMessage[] = []; + const lines = content.split("\n").filter((l) => l.trim()); + let sequence = 0; + + for (const line of lines) { + let entry: ClaudeEntry; + try { + entry = JSON.parse(line); + } catch { + continue; + } + + const msg = parseEntry(entry, sequence); + if (msg) { + messages.push(msg); + sequence++; + } + } + + return messages; +} + /** * Parse a single Claude Code JSONL file into normalized messages. + * BACKWARD COMPATIBLE — returns ParsedMessage[] for existing callers. */ export function parseClaudeJsonl(content: string): ParsedMessage[] { const messages: ParsedMessage[] = []; @@ -161,9 +340,12 @@ export function parseClaudeJsonl(content: string): ParsedMessage[] { return messages; } +// ============================================================================= +// Session discovery (unchanged) +// ============================================================================= + /** * Discover all Claude Code sessions from the logs directory. - * Returns an array of { sessionId, projectDir, filePath }. */ export async function discoverClaudeSessions( logsDir?: string @@ -196,8 +378,12 @@ export async function discoverClaudeSessions( return sessions; } +// ============================================================================= +// Ingestion (enriched) +// ============================================================================= + /** - * Ingest Claude Code sessions into QMD's memory. + * Ingest Claude Code sessions into QMD's memory with structured block extraction. */ export async function ingestClaude( options: IngestOptions = {} @@ -205,7 +391,16 @@ export async function ingestClaude( const { db, existingSessionIds, onProgress } = options; if (!db) throw new Error("Database required for ingestion"); - const { upsertProject, upsertSessionMeta } = await import("../db"); + const { + upsertProject, + upsertSessionMeta, + insertToolUsage, + insertFileOperation, + insertCommand, + insertGitOperation, + insertError, + upsertSessionCosts, + } = await import("../db"); const sessions = await discoverClaudeSessions(options.logsDir); const result: IngestResult = { @@ -218,7 +413,6 @@ export async function ingestClaude( }; for (const session of sessions) { - // Skip already-ingested sessions if (existingSessionIds?.has(session.sessionId)) { result.skipped++; continue; @@ -227,9 +421,9 @@ export async function ingestClaude( try { const file = Bun.file(session.filePath); const content = await file.text(); - const messages = parseClaudeJsonl(content); + const structuredMessages = parseClaudeJsonlStructured(content); - if (messages.length === 0) { + if (structuredMessages.length === 0) { result.skipped++; continue; } @@ -237,33 +431,147 @@ export async function ingestClaude( // Derive project info const projectId = deriveProjectId(session.projectDir); const projectPath = deriveProjectPath(session.projectDir); - - // Ensure project exists upsertProject(db, projectId, projectPath); // Extract title from first user message - const firstUser = messages.find((m) => m.role === "user"); + const firstUser = structuredMessages.find((m) => m.role === "user"); const title = firstUser - ? firstUser.content.slice(0, 100).replace(/\n/g, " ") + ? firstUser.plainText.slice(0, 100).replace(/\n/g, " ") : ""; - // Add messages via QMD - for (const msg of messages) { - await addMessage(db, session.sessionId, msg.role, msg.content, { - title, - metadata: msg.metadata, - }); + // Process each structured message + for (const msg of structuredMessages) { + // Store via QMD (backward-compatible: plainText as content) + const stored = await addMessage( + db, + session.sessionId, + msg.role, + msg.plainText || "(structured content)", + { + title, + metadata: { + ...msg.metadata, + blocks: msg.blocks, + }, + } + ); + + const messageId = stored.id; + const createdAt = msg.timestamp || new Date().toISOString(); + + // Populate sidecar tables from blocks + for (const block of msg.blocks) { + switch (block.type) { + case "tool_call": + insertToolUsage( + db, + messageId, + session.sessionId, + block.toolName, + block.description || summarizeToolInput(block.toolName, block.input), + true, // success assumed; updated by tool_result if paired + null, + createdAt + ); + break; + + case "file_op": + if (block.path) { + insertFileOperation( + db, + messageId, + session.sessionId, + block.operation, + block.path, + projectId, + createdAt + ); + } + break; + + case "command": + insertCommand( + db, + messageId, + session.sessionId, + block.command, + block.exitCode ?? null, + block.cwd ?? null, + block.isGit, + createdAt + ); + break; + + case "git": + insertGitOperation( + db, + messageId, + session.sessionId, + block.operation, + block.branch ?? null, + block.prUrl ?? null, + block.prNumber ?? null, + block.message ? JSON.stringify({ message: block.message }) : null, + createdAt + ); + break; + + case "error": + insertError( + db, + messageId, + session.sessionId, + block.errorType, + block.message, + createdAt + ); + break; + } + } + + // Accumulate token costs from metadata + if (msg.metadata.tokenUsage) { + const u = msg.metadata.tokenUsage; + upsertSessionCosts( + db, + session.sessionId, + msg.metadata.model || null, + u.input, + u.output, + (u.cacheCreate || 0) + (u.cacheRead || 0), + 0 + ); + } + + // Accumulate turn duration from system events + for (const block of msg.blocks) { + if ( + block.type === "system_event" && + block.eventType === "turn_duration" && + typeof block.data.durationMs === "number" + ) { + upsertSessionCosts( + db, + session.sessionId, + null, + 0, + 0, + 0, + block.data.durationMs as number + ); + } + } } // Attach Smriti metadata upsertSessionMeta(db, session.sessionId, "claude-code", projectId); result.sessionsIngested++; - result.messagesIngested += messages.length; + result.messagesIngested += structuredMessages.length; if (onProgress) { onProgress( - `Ingested ${session.sessionId} (${messages.length} messages)` + `Ingested ${session.sessionId} (${structuredMessages.length} messages)` ); } } catch (err: any) { @@ -273,3 +581,33 @@ export async function ingestClaude( return result; } + +// ============================================================================= +// Helpers +// ============================================================================= + +/** + * Create a short summary of tool input for the input_summary column. + */ +function summarizeToolInput(toolName: string, input: Record): string { + switch (toolName) { + case "Read": + return `Read ${input.file_path || ""}`; + case "Write": + return `Write ${input.file_path || ""}`; + case "Edit": + return `Edit ${input.file_path || ""}`; + case "Glob": + return `Glob ${input.pattern || ""}`; + case "Grep": + return `Grep ${input.pattern || ""} in ${input.path || "."}`; + case "Bash": + return String(input.command || "").slice(0, 100); + case "WebFetch": + return `Fetch ${input.url || ""}`; + case "WebSearch": + return `Search: ${input.query || ""}`; + default: + return toolName; + } +} diff --git a/src/ingest/index.ts b/src/ingest/index.ts index b311c5d..032215e 100644 --- a/src/ingest/index.ts +++ b/src/ingest/index.ts @@ -8,15 +8,10 @@ import type { Database } from "bun:sqlite"; // ============================================================================= -// Types +// Types — re-export from types.ts // ============================================================================= -export type ParsedMessage = { - role: string; - content: string; - timestamp?: string; - metadata?: Record; -}; +export type { ParsedMessage, StructuredMessage, MessageBlock, MessageMetadata } from "./types"; export type IngestResult = { agent: string; @@ -91,6 +86,10 @@ export async function ingest( projectPath: options.projectPath, }); } + case "antigravity": { + const { ingestAntigravity } = await import("./antigravity"); + return ingestAntigravity(baseOptions); + } case "file": case "generic": { const { ingestGeneric } = await import("./generic"); @@ -111,7 +110,7 @@ export async function ingest( sessionsIngested: 0, messagesIngested: 0, skipped: 0, - errors: [`Unknown agent: ${agent}. Use: claude, codex, cursor, or file`], + errors: [`Unknown agent: ${agent}. Use: claude, codex, cursor, antigravity, or file`], }; } } @@ -125,7 +124,7 @@ export async function ingestAll( ): Promise { const results: IngestResult[] = []; - for (const agent of ["claude-code", "codex"]) { + for (const agent of ["claude-code", "codex", "antigravity"]) { const result = await ingest(db, agent, options); results.push(result); } diff --git a/src/ingest/types.ts b/src/ingest/types.ts new file mode 100644 index 0000000..233e5db --- /dev/null +++ b/src/ingest/types.ts @@ -0,0 +1,219 @@ +/** + * types.ts - Structured message types for enriched ingestion + * + * Defines the full message taxonomy across all agents: tool calls, thinking + * blocks, file operations, commands, searches, git operations, errors, etc. + */ + +// ============================================================================= +// Storage Limits +// ============================================================================= + +export const STORAGE_LIMITS = { + textBlock: 50_000, + commandOutput: 2_000, + fileContent: 10_000, + thinkingBlock: 20_000, + searchResults: 5_000, + toolInput: 5_000, +}; + +// ============================================================================= +// Content Blocks +// ============================================================================= + +export type TextBlock = { + type: "text"; + text: string; +}; + +export type ThinkingBlock = { + type: "thinking"; + thinking: string; + budgetTokens?: number; +}; + +export type ToolCallBlock = { + type: "tool_call"; + toolId: string; + toolName: string; + input: Record; + description?: string; +}; + +export type ToolResultBlock = { + type: "tool_result"; + toolId: string; + success: boolean; + output: string; + error?: string; + durationMs?: number; +}; + +export type FileOperationBlock = { + type: "file_op"; + operation: "read" | "write" | "edit" | "create" | "delete" | "glob"; + path: string; + diff?: string; + pattern?: string; + results?: string[]; +}; + +export type CommandBlock = { + type: "command"; + command: string; + cwd?: string; + exitCode?: number; + stdout?: string; + stderr?: string; + description?: string; + isGit: boolean; +}; + +export type SearchBlock = { + type: "search"; + searchType: "grep" | "glob" | "web_fetch" | "web_search"; + pattern: string; + path?: string; + url?: string; + resultCount?: number; +}; + +export type GitBlock = { + type: "git"; + operation: + | "commit" + | "push" + | "pull" + | "branch" + | "checkout" + | "diff" + | "merge" + | "rebase" + | "status" + | "pr_create" + | "other"; + branch?: string; + message?: string; + files?: string[]; + prUrl?: string; + prNumber?: number; +}; + +export type ErrorBlock = { + type: "error"; + errorType: + | "api" + | "tool_failure" + | "rate_limit" + | "timeout" + | "permission" + | "validation"; + message: string; + retryable?: boolean; +}; + +export type ImageBlock = { + type: "image"; + mediaType: string; + path?: string; + dataHash?: string; + description?: string; +}; + +export type CodeBlock = { + type: "code"; + language: string; + code: string; + filePath?: string; + lineStart?: number; +}; + +export type SystemEventBlock = { + type: "system_event"; + eventType: + | "turn_duration" + | "pr_link" + | "file_snapshot" + | "mode_change" + | "session_start" + | "session_end"; + data: Record; +}; + +export type ConversationControlBlock = { + type: "control"; + controlType: + | "interrupt" + | "retry" + | "plan_enter" + | "plan_exit" + | "sidechain" + | "slash_command"; + command?: string; +}; + +export type MessageBlock = + | TextBlock + | ThinkingBlock + | ToolCallBlock + | ToolResultBlock + | FileOperationBlock + | CommandBlock + | SearchBlock + | GitBlock + | ErrorBlock + | ImageBlock + | CodeBlock + | SystemEventBlock + | ConversationControlBlock; + +// ============================================================================= +// Metadata +// ============================================================================= + +export type MessageMetadata = { + cwd?: string; + gitBranch?: string; + model?: string; + requestId?: string; + stopReason?: string; + tokenUsage?: { + input: number; + output: number; + cacheCreate?: number; + cacheRead?: number; + }; + agentVersion?: string; + parentId?: string; + isSidechain?: boolean; + permissionMode?: string; + slug?: string; +}; + +// ============================================================================= +// Structured Message +// ============================================================================= + +export type StructuredMessage = { + id: string; + sessionId: string; + sequence: number; + timestamp: string; + role: "user" | "assistant" | "system" | "tool"; + agent: string; + blocks: MessageBlock[]; + metadata: MessageMetadata; + plainText: string; +}; + +// ============================================================================= +// Legacy compat — ParsedMessage is still used by Codex/Cursor/Generic parsers +// ============================================================================= + +export type ParsedMessage = { + role: string; + content: string; + timestamp?: string; + metadata?: Record; +}; diff --git a/src/team/formatter.ts b/src/team/formatter.ts new file mode 100644 index 0000000..8c3d658 --- /dev/null +++ b/src/team/formatter.ts @@ -0,0 +1,232 @@ +/** + * team/formatter.ts - Sanitization and fallback formatting for knowledge export + * + * Strips noise from raw chat transcripts (XML tags, interrupt markers, API + * errors, narration). Provides a fallback conversation-based format when + * LLM synthesis is unavailable. + */ + +// ============================================================================= +// Types +// ============================================================================= + +export type RawMessage = { + role: string; + content: string; +}; + +export type CleanMessage = { + role: string; + content: string; +}; + +// ============================================================================= +// Sanitization +// ============================================================================= + +/** Strip noise patterns from message content */ +export function sanitizeContent(content: string): string { + let s = content; + + // Remove XML block tags (multiline) + s = s.replace(/[\s\S]*?<\/task-notification>/g, ""); + s = s.replace(/[\s\S]*?<\/system-reminder>/g, ""); + + // Remove inline XML tags + s = s.replace(/[\s\S]*?<\/command-message>/g, ""); + s = s.replace(/[\s\S]*?<\/command-name>/g, ""); + + // Remove interrupt markers + s = s.replace(/\[Request interrupted by user(?:\s+for tool use)?\]/g, ""); + + // Remove "Read the output file..." lines with tmp paths + s = s.replace(/Read the output file.*\/private\/tmp\/.*$/gm, ""); + s = s.replace(/Read the output file.*\/tmp\/.*$/gm, ""); + + // Remove API error lines + s = s.replace(/^API Error:.*$/gm, ""); + + // Remove rate limit messages + s = s.replace(/^Rate limit.*$/gim, ""); + + // Clean up residual whitespace + s = s.replace(/\n{3,}/g, "\n\n"); + s = s.trim(); + + return s; +} + +// ============================================================================= +// Message filtering +// ============================================================================= + +/** Patterns for short assistant narration that adds no value */ +const NARRATION_PATTERNS = [ + /^(?:let me |now let me |i'll |let me now )/i, + /^(?:now i |good,? i |perfect!? (?:let|i'll|now))/i, + /^(?:standing by|one moment|looking|checking|reading|searching)/i, + /^(?:i'm going to |i need to |i have all |i found )/i, +]; + +/** Check if a message should be dropped entirely */ +export function shouldDropMessage(role: string, content: string): boolean { + const cleaned = sanitizeContent(content); + + // Empty or whitespace-only after sanitization + if (!cleaned || !cleaned.trim()) return true; + + // Bare commands + if (/^(clear|quit|exit|\d)$/i.test(cleaned.trim())) return true; + + // Short assistant narration (only when < 200 chars to avoid false positives) + if (role === "assistant" && cleaned.length < 200) { + for (const pattern of NARRATION_PATTERNS) { + if (pattern.test(cleaned.trim())) return true; + } + } + + return false; +} + +/** Filter and sanitize raw messages */ +export function filterMessages(raw: RawMessage[]): CleanMessage[] { + const result: CleanMessage[] = []; + + for (const msg of raw) { + if (shouldDropMessage(msg.role, msg.content)) continue; + + const cleaned = sanitizeContent(msg.content); + if (cleaned) { + result.push({ role: msg.role, content: cleaned }); + } + } + + return result; +} + +// ============================================================================= +// Merging +// ============================================================================= + +/** Merge consecutive same-role messages into single messages */ +export function mergeConsecutive(messages: CleanMessage[]): CleanMessage[] { + if (messages.length === 0) return []; + + const merged: CleanMessage[] = [{ ...messages[0] }]; + + for (let i = 1; i < messages.length; i++) { + const prev = merged[merged.length - 1]; + const curr = messages[i]; + + if (curr.role === prev.role) { + prev.content = prev.content + "\n\n" + curr.content; + } else { + merged.push({ ...curr }); + } + } + + return merged; +} + +// ============================================================================= +// Title derivation +// ============================================================================= + +/** Derive a clean title from session title or first user message */ +export function deriveTitle( + sessionTitle: string | null | undefined, + messages: CleanMessage[] +): string { + // Try session title first + if (sessionTitle) { + let title = sanitizeContent(sessionTitle).trim(); + // Strip markdown heading prefix if present + title = title.replace(/^#+\s*/, ""); + if (title && title.length > 3) return title; + } + + // Fall back to first user message + const firstUser = messages.find((m) => m.role === "user"); + if (firstUser) { + // Use first line, truncated + const firstLine = firstUser.content.split("\n")[0].trim(); + if (firstLine.length > 80) { + return firstLine.slice(0, 77) + "..."; + } + return firstLine; + } + + return "Untitled Session"; +} + +// ============================================================================= +// Fallback document formatting (used when LLM synthesis is unavailable) +// ============================================================================= + +/** Format filtered messages as a fallback markdown document */ +export function formatAsFallbackDocument( + title: string, + summary: string | null | undefined, + messages: CleanMessage[] +): string { + const lines: string[] = []; + + lines.push(`# ${title}`); + lines.push(""); + + if (summary) { + lines.push(`> ${summary}`); + lines.push(""); + } + + for (const msg of messages) { + if (msg.role === "user") { + const msgLines = msg.content.split("\n"); + const heading = msgLines[0].replace(/^#+\s*/, "").trim(); + const body = msgLines.slice(1).join("\n").trim(); + + lines.push(`## ${heading}`); + lines.push(""); + if (body) { + lines.push(body); + lines.push(""); + } + } else { + lines.push(msg.content); + lines.push(""); + } + } + + return lines.join("\n").replace(/\n{3,}/g, "\n\n").trim() + "\n"; +} + +// ============================================================================= +// Main entry points +// ============================================================================= + +/** Fallback pipeline: filter → merge → derive title → format as conversation */ +export function formatSessionAsFallback( + sessionTitle: string | null | undefined, + summary: string | null | undefined, + rawMessages: RawMessage[] +): { title: string; body: string } { + const filtered = filterMessages(rawMessages); + const merged = mergeConsecutive(filtered); + const title = deriveTitle(sessionTitle, merged); + const body = formatAsFallbackDocument(title, summary, merged); + + return { title, body }; +} + +/** Gate: check if a session has enough substance to be worth sharing */ +export function isSessionWorthSharing(rawMessages: RawMessage[]): boolean { + const filtered = filterMessages(rawMessages); + + const hasUser = filtered.some((m) => m.role === "user"); + const hasAssistant = filtered.some((m) => m.role === "assistant"); + + if (!hasUser || !hasAssistant) return false; + + const totalLength = filtered.reduce((sum, m) => sum + m.content.length, 0); + return totalLength > 100; +} diff --git a/src/team/prompts/share-reflect.md b/src/team/prompts/share-reflect.md new file mode 100644 index 0000000..3f6ba01 --- /dev/null +++ b/src/team/prompts/share-reflect.md @@ -0,0 +1,28 @@ +You are a technical writer synthesizing a development session into a concise knowledge document. + +Read the conversation below between a developer and an AI coding assistant. Produce a structured knowledge article — NOT a conversation summary, but distilled insights that would be useful for anyone working on this project. + +Write in third person. Be specific: mention file paths, function names, config values, and commands. Skip pleasantries and meta-commentary. + +## Conversation + +{{conversation}} + +--- + +Respond in this exact markdown format. Every section is required — write "N/A" if a section truly doesn't apply. + +### Summary +One paragraph: what was accomplished in this session and why it matters. + +### Changes +Bullet list of concrete changes (files created/modified, features added, bugs fixed, configs changed). Include file paths. + +### Decisions +Key technical decisions made and their rationale. What alternatives were considered? + +### Insights +What was learned or discovered about the project, its architecture, dependencies, or patterns? What would save a teammate time? + +### Context +Background context needed to understand these changes. Prior state, constraints, or gotchas encountered. diff --git a/src/team/reflect.ts b/src/team/reflect.ts new file mode 100644 index 0000000..236575f --- /dev/null +++ b/src/team/reflect.ts @@ -0,0 +1,267 @@ +/** + * team/reflect.ts - LLM-powered session synthesis for knowledge export + * + * Sends filtered conversation to Ollama with a synthesis prompt template. + * Returns a structured knowledge article (summary, changes, decisions, + * insights, context) that replaces the conversation trail in the output. + * + * Prompt template is loaded from: + * 1. .smriti/prompts/share-reflect.md (project override) + * 2. src/team/prompts/share-reflect.md (built-in default) + */ + +import { OLLAMA_HOST, OLLAMA_MODEL } from "../config"; +import { join, dirname } from "path"; +import type { RawMessage } from "./formatter"; +import { filterMessages, mergeConsecutive, sanitizeContent } from "./formatter"; + +// ============================================================================= +// Types +// ============================================================================= + +export type Synthesis = { + summary: string; + changes: string; + decisions: string; + insights: string; + context: string; +}; + +// ============================================================================= +// Prompt loading +// ============================================================================= + +const DEFAULT_PROMPT_PATH = join( + dirname(new URL(import.meta.url).pathname), + "prompts", + "share-reflect.md" +); + +/** Load the prompt template, preferring project override */ +export async function loadPromptTemplate( + projectSmritiDir?: string +): Promise { + // Try project-level override first + if (projectSmritiDir) { + const overridePath = join( + projectSmritiDir, + "prompts", + "share-reflect.md" + ); + const overrideFile = Bun.file(overridePath); + if (await overrideFile.exists()) { + return overrideFile.text(); + } + } + + // Fall back to built-in default + const defaultFile = Bun.file(DEFAULT_PROMPT_PATH); + return defaultFile.text(); +} + +// ============================================================================= +// Conversation formatting (for prompt injection) +// ============================================================================= + +/** Max chars to send to the LLM — keeps prompt within model context window */ +const MAX_CONVERSATION_CHARS = 8000; + +/** Format raw messages into a readable conversation string for the LLM */ +function formatConversationForPrompt(rawMessages: RawMessage[]): string { + const filtered = filterMessages(rawMessages); + const merged = mergeConsecutive(filtered); + + let text = merged + .map((m) => `**${m.role}**: ${m.content}`) + .join("\n\n"); + + // Truncate to fit model context, keeping the end (most recent/relevant) + if (text.length > MAX_CONVERSATION_CHARS) { + text = "...\n\n" + text.slice(-MAX_CONVERSATION_CHARS); + } + + return text; +} + +// ============================================================================= +// Response parsing +// ============================================================================= + +const SECTION_KEYS: Array<{ header: string; field: keyof Synthesis }> = [ + { header: "### Summary", field: "summary" }, + { header: "### Changes", field: "changes" }, + { header: "### Decisions", field: "decisions" }, + { header: "### Insights", field: "insights" }, + { header: "### Context", field: "context" }, +]; + +/** Parse structured synthesis from LLM response text */ +export function parseSynthesis(response: string): Synthesis { + const synthesis: Synthesis = { + summary: "", + changes: "", + decisions: "", + insights: "", + context: "", + }; + + for (let i = 0; i < SECTION_KEYS.length; i++) { + const { header, field } = SECTION_KEYS[i]; + const headerIdx = response.indexOf(header); + if (headerIdx === -1) continue; + + const contentStart = headerIdx + header.length; + + // Find the next section header or end of string + let contentEnd = response.length; + for (let j = i + 1; j < SECTION_KEYS.length; j++) { + const nextIdx = response.indexOf(SECTION_KEYS[j].header, contentStart); + if (nextIdx !== -1) { + contentEnd = nextIdx; + break; + } + } + + let value = response.slice(contentStart, contentEnd).trim(); + + // Strip "N/A" responses + if (/^n\/?a\.?$/i.test(value)) { + value = ""; + } + + synthesis[field] = value; + } + + return synthesis; +} + +/** Derive a title from the synthesis summary */ +export function deriveTitleFromSynthesis(synthesis: Synthesis): string | null { + if (!synthesis.summary) return null; + // Use first sentence of summary, capped at 80 chars + const firstSentence = synthesis.summary.split(/\.\s/)[0]; + if (firstSentence.length > 80) { + return firstSentence.slice(0, 77) + "..."; + } + return firstSentence.replace(/\.$/, ""); +} + +// ============================================================================= +// Document formatting +// ============================================================================= + +/** Format a synthesis into a complete markdown document body */ +export function formatSynthesisAsDocument( + title: string, + synthesis: Synthesis +): string { + const lines: string[] = []; + + lines.push(`# ${title}`); + lines.push(""); + + if (synthesis.summary) { + lines.push(`> ${synthesis.summary}`); + lines.push(""); + } + + if (synthesis.changes) { + lines.push("## Changes"); + lines.push(""); + lines.push(synthesis.changes); + lines.push(""); + } + + if (synthesis.decisions) { + lines.push("## Decisions"); + lines.push(""); + lines.push(synthesis.decisions); + lines.push(""); + } + + if (synthesis.insights) { + lines.push("## Insights"); + lines.push(""); + lines.push(synthesis.insights); + lines.push(""); + } + + if (synthesis.context) { + lines.push("## Context"); + lines.push(""); + lines.push(synthesis.context); + lines.push(""); + } + + return lines.join("\n").replace(/\n{3,}/g, "\n\n").trim() + "\n"; +} + +// ============================================================================= +// Main synthesis +// ============================================================================= + +export type SynthesizeOptions = { + model?: string; + projectSmritiDir?: string; + timeout?: number; +}; + +/** + * Synthesize a session into a knowledge article via Ollama. + * Returns null if Ollama is unavailable or the session is too short. + */ +export async function synthesizeSession( + rawMessages: RawMessage[], + options: SynthesizeOptions = {} +): Promise { + const conversation = formatConversationForPrompt(rawMessages); + + // Skip synthesis for very short conversations + if (conversation.length < 100) return null; + + try { + const template = await loadPromptTemplate(options.projectSmritiDir); + const prompt = template.replace("{{conversation}}", conversation); + + const model = options.model || OLLAMA_MODEL; + const timeout = options.timeout || 120_000; + + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeout); + + const response = await fetch(`${OLLAMA_HOST}/api/generate`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model, + prompt, + stream: false, + options: { + temperature: 0.3, + num_predict: 1000, + }, + }), + signal: controller.signal, + }); + + clearTimeout(timer); + + if (!response.ok) return null; + + const data = (await response.json()) as { response?: string }; + if (!data.response) return null; + + return parseSynthesis(data.response); + } catch { + // Ollama unavailable or timeout — graceful degradation + return null; + } +} + +/** Check if a synthesis has enough content to use */ +export function hasSubstantiveSynthesis(synthesis: Synthesis): boolean { + // Need at least summary + one other section + if (!synthesis.summary) return false; + const otherSections = [synthesis.changes, synthesis.decisions, synthesis.insights, synthesis.context]; + return otherSections.some((s) => s.length > 0); +} diff --git a/src/team/share.ts b/src/team/share.ts index dadada3..065d628 100644 --- a/src/team/share.ts +++ b/src/team/share.ts @@ -10,6 +10,19 @@ import { SMRITI_DIR, AUTHOR } from "../config"; import { hashContent } from "../qmd"; import { existsSync, mkdirSync } from "fs"; import { join, basename } from "path"; +import { + formatSessionAsFallback, + isSessionWorthSharing, + deriveTitle, + filterMessages, + mergeConsecutive, +} from "./formatter"; +import { + synthesizeSession, + hasSubstantiveSynthesis, + deriveTitleFromSynthesis, + formatSynthesisAsDocument, +} from "./reflect"; // ============================================================================= // Types @@ -21,6 +34,8 @@ export type ShareOptions = { sessionId?: string; outputDir?: string; author?: string; + reflect?: boolean; + reflectModel?: string; }; export type ShareResult = { @@ -214,13 +229,51 @@ export async function shareKnowledge( const categoryDir = join(knowledgeDir, primaryCategory.replace("/", "-")); mkdirSync(categoryDir, { recursive: true }); - // Generate filename + // Skip noise-only sessions + const rawMessages = messages.map((m) => ({ + role: m.role, + content: m.content, + })); + + if (!isSessionWorthSharing(rawMessages)) { + result.filesSkipped++; + continue; + } + + // Synthesize via LLM (primary) or fall back to cleaned conversation + let cleanTitle: string; + let body: string; + + if (options.reflect) { + const synthesis = await synthesizeSession(rawMessages, { + model: options.reflectModel, + projectSmritiDir: outputDir, + }); + + if (synthesis && hasSubstantiveSynthesis(synthesis)) { + // Use LLM-synthesized knowledge article + cleanTitle = deriveTitleFromSynthesis(synthesis) || + deriveTitle(session.title, mergeConsecutive(filterMessages(rawMessages))); + body = formatSynthesisAsDocument(cleanTitle, synthesis); + } else { + // LLM unavailable or insufficient — fall back to cleaned conversation + const fallback = formatSessionAsFallback(session.title, session.summary, rawMessages); + cleanTitle = fallback.title; + body = fallback.body; + } + } else { + const fallback = formatSessionAsFallback(session.title, session.summary, rawMessages); + cleanTitle = fallback.title; + body = fallback.body; + } + + // Generate filename using clean title const date = datePrefix(session.created_at); - const slug = slugify(session.title || session.id); + const slug = slugify(cleanTitle || session.id); const filename = `${date}_${slug}.md`; const filePath = join(categoryDir, filename); - // Build markdown content + // Build final content with frontmatter const meta = frontmatter({ id: session.id, category: primaryCategory, @@ -231,20 +284,7 @@ export async function shareKnowledge( tags: categories.map((c) => c.category_id), }); - const conversationLines = messages.map( - (m) => `**${m.role}**: ${m.content}` - ); - - const content = [ - meta, - "", - `# ${session.title || "Untitled Session"}`, - "", - session.summary ? `> ${session.summary}\n` : "", - ...conversationLines, - ] - .filter(Boolean) - .join("\n"); + const content = meta + "\n\n" + body; await Bun.write(filePath, content); @@ -304,5 +344,45 @@ export async function shareKnowledge( ); } + // Generate CLAUDE.md so Claude Code discovers shared knowledge + await generateClaudeMd(outputDir, fullManifest); + return result; } + +/** + * Generate a .smriti/CLAUDE.md that indexes all shared knowledge files. + * Claude Code auto-discovers CLAUDE.md files in subdirectories. + */ +async function generateClaudeMd( + outputDir: string, + manifest: Array<{ id: string; category: string; file: string; shared_at: string }> +) { + // Group by category + const byCategory = new Map(); + for (const entry of manifest) { + const files = byCategory.get(entry.category) || []; + files.push(entry.file); + byCategory.set(entry.category, files); + } + + const lines: string[] = [ + "# Team Knowledge", + "", + "This directory contains shared knowledge from development sessions.", + "Generated by `smriti share`. Do not edit manually.", + "", + ]; + + for (const [category, files] of [...byCategory.entries()].sort()) { + lines.push(`## ${category}`); + lines.push(""); + for (const file of files) { + const name = file.split("/").pop()?.replace(/\.md$/, "").replace(/_/g, " ") || file; + lines.push(`- [${name}](${file})`); + } + lines.push(""); + } + + await Bun.write(join(outputDir, "CLAUDE.md"), lines.join("\n")); +} diff --git a/test/blocks.test.ts b/test/blocks.test.ts new file mode 100644 index 0000000..62ffc93 --- /dev/null +++ b/test/blocks.test.ts @@ -0,0 +1,451 @@ +import { test, expect } from "bun:test"; +import { + extractBlocks, + flattenBlocksToText, + toolCallToBlocks, + parseToolResult, + isGitCommand, + parseGitCommand, + parseGhPrCommand, + systemEntryToBlock, +} from "../src/ingest/blocks"; + +// ============================================================================= +// extractBlocks +// ============================================================================= + +test("extractBlocks handles plain string content", () => { + const blocks = extractBlocks("Hello world"); + expect(blocks.length).toBe(1); + expect(blocks[0].type).toBe("text"); + if (blocks[0].type === "text") { + expect(blocks[0].text).toBe("Hello world"); + } +}); + +test("extractBlocks handles empty string", () => { + const blocks = extractBlocks(""); + expect(blocks.length).toBe(0); +}); + +test("extractBlocks handles text content blocks", () => { + const blocks = extractBlocks([ + { type: "text", text: "First paragraph" }, + { type: "text", text: "Second paragraph" }, + ]); + expect(blocks.length).toBe(2); + expect(blocks[0].type).toBe("text"); + expect(blocks[1].type).toBe("text"); +}); + +test("extractBlocks handles thinking blocks", () => { + const blocks = extractBlocks([ + { type: "thinking", thinking: "Let me think about this..." }, + ]); + expect(blocks.length).toBe(1); + expect(blocks[0].type).toBe("thinking"); + if (blocks[0].type === "thinking") { + expect(blocks[0].thinking).toBe("Let me think about this..."); + } +}); + +test("extractBlocks handles tool_use blocks", () => { + const blocks = extractBlocks([ + { + type: "tool_use", + id: "tool_123", + name: "Read", + input: { file_path: "/src/index.ts" }, + }, + ]); + // tool_use → [ToolCallBlock, FileOperationBlock] + expect(blocks.length).toBe(2); + expect(blocks[0].type).toBe("tool_call"); + expect(blocks[1].type).toBe("file_op"); + if (blocks[1].type === "file_op") { + expect(blocks[1].operation).toBe("read"); + expect(blocks[1].path).toBe("/src/index.ts"); + } +}); + +test("extractBlocks handles tool_result blocks", () => { + const blocks = extractBlocks([ + { + type: "tool_result", + tool_use_id: "tool_123", + content: "File contents here", + }, + ]); + expect(blocks.length).toBe(1); + expect(blocks[0].type).toBe("tool_result"); + if (blocks[0].type === "tool_result") { + expect(blocks[0].toolId).toBe("tool_123"); + expect(blocks[0].success).toBe(true); + expect(blocks[0].output).toBe("File contents here"); + } +}); + +test("extractBlocks handles image blocks", () => { + const blocks = extractBlocks([ + { + type: "image", + source: { type: "base64", media_type: "image/png", data: "abc123" }, + }, + ]); + expect(blocks.length).toBe(1); + expect(blocks[0].type).toBe("image"); + if (blocks[0].type === "image") { + expect(blocks[0].mediaType).toBe("image/png"); + expect(blocks[0].dataHash).toBeDefined(); + } +}); + +test("extractBlocks handles mixed content blocks", () => { + const blocks = extractBlocks([ + { type: "text", text: "I'll read the file" }, + { + type: "tool_use", + id: "tool_1", + name: "Read", + input: { file_path: "/src/main.ts" }, + }, + { type: "text", text: "Here's what I found" }, + ]); + // text + [tool_call, file_op] + text = 4 blocks + expect(blocks.length).toBe(4); + expect(blocks[0].type).toBe("text"); + expect(blocks[1].type).toBe("tool_call"); + expect(blocks[2].type).toBe("file_op"); + expect(blocks[3].type).toBe("text"); +}); + +// ============================================================================= +// toolCallToBlocks — domain mapping +// ============================================================================= + +test("toolCallToBlocks maps Read to file_op", () => { + const blocks = toolCallToBlocks("Read", "t1", { file_path: "/src/app.ts" }); + expect(blocks.length).toBe(2); + expect(blocks[0].type).toBe("tool_call"); + expect(blocks[1].type).toBe("file_op"); + if (blocks[1].type === "file_op") { + expect(blocks[1].operation).toBe("read"); + expect(blocks[1].path).toBe("/src/app.ts"); + } +}); + +test("toolCallToBlocks maps Write to file_op", () => { + const blocks = toolCallToBlocks("Write", "t2", { + file_path: "/src/new.ts", + content: "export const x = 1;", + }); + expect(blocks.length).toBe(2); + expect(blocks[1].type).toBe("file_op"); + if (blocks[1].type === "file_op") { + expect(blocks[1].operation).toBe("write"); + expect(blocks[1].path).toBe("/src/new.ts"); + } +}); + +test("toolCallToBlocks maps Edit to file_op with diff", () => { + const blocks = toolCallToBlocks("Edit", "t3", { + file_path: "/src/config.ts", + old_string: "const x = 1", + new_string: "const x = 2", + }); + expect(blocks.length).toBe(2); + if (blocks[1].type === "file_op") { + expect(blocks[1].operation).toBe("edit"); + expect(blocks[1].diff).toContain("const x = 1"); + expect(blocks[1].diff).toContain("const x = 2"); + } +}); + +test("toolCallToBlocks maps Glob to file_op + search", () => { + const blocks = toolCallToBlocks("Glob", "t4", { + pattern: "**/*.ts", + path: "/src", + }); + expect(blocks.length).toBe(3); // tool_call + file_op + search + expect(blocks[1].type).toBe("file_op"); + expect(blocks[2].type).toBe("search"); + if (blocks[2].type === "search") { + expect(blocks[2].searchType).toBe("glob"); + expect(blocks[2].pattern).toBe("**/*.ts"); + } +}); + +test("toolCallToBlocks maps Grep to search", () => { + const blocks = toolCallToBlocks("Grep", "t5", { + pattern: "function\\s+main", + path: "/src", + }); + expect(blocks.length).toBe(2); // tool_call + search + expect(blocks[1].type).toBe("search"); + if (blocks[1].type === "search") { + expect(blocks[1].searchType).toBe("grep"); + expect(blocks[1].pattern).toBe("function\\s+main"); + } +}); + +test("toolCallToBlocks maps Bash to command", () => { + const blocks = toolCallToBlocks("Bash", "t6", { + command: "bun test", + description: "Run tests", + }); + expect(blocks.length).toBe(2); // tool_call + command + expect(blocks[1].type).toBe("command"); + if (blocks[1].type === "command") { + expect(blocks[1].command).toBe("bun test"); + expect(blocks[1].isGit).toBe(false); + expect(blocks[1].description).toBe("Run tests"); + } +}); + +test("toolCallToBlocks maps Bash git command to command + git", () => { + const blocks = toolCallToBlocks("Bash", "t7", { + command: 'git commit -m "Fix bug"', + }); + expect(blocks.length).toBe(3); // tool_call + command + git + expect(blocks[1].type).toBe("command"); + expect(blocks[2].type).toBe("git"); + if (blocks[1].type === "command") { + expect(blocks[1].isGit).toBe(true); + } + if (blocks[2].type === "git") { + expect(blocks[2].operation).toBe("commit"); + expect(blocks[2].message).toBe("Fix bug"); + } +}); + +test("toolCallToBlocks maps WebFetch to search", () => { + const blocks = toolCallToBlocks("WebFetch", "t8", { + url: "https://example.com", + prompt: "Extract the main content", + }); + expect(blocks.length).toBe(2); + expect(blocks[1].type).toBe("search"); + if (blocks[1].type === "search") { + expect(blocks[1].searchType).toBe("web_fetch"); + expect(blocks[1].url).toBe("https://example.com"); + } +}); + +test("toolCallToBlocks maps WebSearch to search", () => { + const blocks = toolCallToBlocks("WebSearch", "t9", { + query: "bun testing guide", + }); + expect(blocks.length).toBe(2); + expect(blocks[1].type).toBe("search"); + if (blocks[1].type === "search") { + expect(blocks[1].searchType).toBe("web_search"); + expect(blocks[1].pattern).toBe("bun testing guide"); + } +}); + +test("toolCallToBlocks maps EnterPlanMode to control", () => { + const blocks = toolCallToBlocks("EnterPlanMode", "t10", {}); + expect(blocks.length).toBe(2); + expect(blocks[1].type).toBe("control"); + if (blocks[1].type === "control") { + expect(blocks[1].controlType).toBe("plan_enter"); + } +}); + +test("toolCallToBlocks maps Skill to control with command", () => { + const blocks = toolCallToBlocks("Skill", "t11", { skill: "commit" }); + expect(blocks.length).toBe(2); + expect(blocks[1].type).toBe("control"); + if (blocks[1].type === "control") { + expect(blocks[1].controlType).toBe("slash_command"); + expect(blocks[1].command).toBe("commit"); + } +}); + +test("toolCallToBlocks keeps unknown tools as generic tool_call only", () => { + const blocks = toolCallToBlocks("TaskCreate", "t12", { subject: "Do stuff" }); + expect(blocks.length).toBe(1); + expect(blocks[0].type).toBe("tool_call"); +}); + +// ============================================================================= +// Git command detection +// ============================================================================= + +test("isGitCommand detects git commands", () => { + expect(isGitCommand("git status")).toBe(true); + expect(isGitCommand(" git diff")).toBe(true); + expect(isGitCommand("bun test")).toBe(false); + expect(isGitCommand("github-cli")).toBe(false); +}); + +test("parseGitCommand extracts commit message with single quotes", () => { + const block = parseGitCommand("git commit -m 'Add feature'"); + expect(block).not.toBeNull(); + expect(block!.operation).toBe("commit"); + expect(block!.message).toBe("Add feature"); +}); + +test("parseGitCommand extracts commit message with double quotes", () => { + const block = parseGitCommand('git commit -m "Fix bug"'); + expect(block).not.toBeNull(); + expect(block!.message).toBe("Fix bug"); +}); + +test("parseGitCommand extracts checkout branch", () => { + const block = parseGitCommand("git checkout feature/auth"); + expect(block).not.toBeNull(); + expect(block!.operation).toBe("checkout"); + expect(block!.branch).toBe("feature/auth"); +}); + +test("parseGitCommand handles push with remote and branch", () => { + const block = parseGitCommand("git push origin main"); + expect(block).not.toBeNull(); + expect(block!.operation).toBe("push"); + expect(block!.branch).toBe("main"); +}); + +test("parseGitCommand maps unknown subcommands to other", () => { + const block = parseGitCommand("git stash pop"); + expect(block).not.toBeNull(); + expect(block!.operation).toBe("other"); +}); + +test("parseGitCommand returns null for non-git commands", () => { + expect(parseGitCommand("bun test")).toBeNull(); +}); + +test("parseGhPrCommand detects gh pr create", () => { + const block = parseGhPrCommand('gh pr create --title "My PR" --body "desc"'); + expect(block).not.toBeNull(); + expect(block!.operation).toBe("pr_create"); + expect(block!.message).toBe("My PR"); +}); + +test("parseGhPrCommand returns null for non-pr commands", () => { + expect(parseGhPrCommand("gh issue list")).toBeNull(); +}); + +// ============================================================================= +// parseToolResult +// ============================================================================= + +test("parseToolResult handles string content", () => { + const result = parseToolResult("t1", "File contents here"); + expect(result.type).toBe("tool_result"); + expect(result.toolId).toBe("t1"); + expect(result.success).toBe(true); + expect(result.output).toBe("File contents here"); +}); + +test("parseToolResult handles array content", () => { + const result = parseToolResult("t2", [ + { type: "text", text: "Line 1" }, + { type: "text", text: "Line 2" }, + ]); + expect(result.output).toBe("Line 1\nLine 2"); +}); + +test("parseToolResult handles error flag", () => { + const result = parseToolResult("t3", "Permission denied", true); + expect(result.success).toBe(false); + expect(result.error).toBe("Permission denied"); +}); + +test("parseToolResult truncates long output", () => { + const longOutput = "x".repeat(5000); + const result = parseToolResult("t4", longOutput); + expect(result.output.length).toBeLessThan(5000); + expect(result.output).toContain("...[truncated]"); +}); + +// ============================================================================= +// flattenBlocksToText +// ============================================================================= + +test("flattenBlocksToText includes text blocks", () => { + const text = flattenBlocksToText([ + { type: "text", text: "Hello" }, + { type: "text", text: "World" }, + ]); + expect(text).toBe("Hello\nWorld"); +}); + +test("flattenBlocksToText includes command descriptions and commands", () => { + const text = flattenBlocksToText([ + { + type: "command", + command: "bun test", + description: "Run tests", + isGit: false, + }, + ]); + expect(text).toContain("Run tests"); + expect(text).toContain("$ bun test"); +}); + +test("flattenBlocksToText includes file operations", () => { + const text = flattenBlocksToText([ + { type: "file_op", operation: "write", path: "/src/new.ts" }, + ]); + expect(text).toContain("[write] /src/new.ts"); +}); + +test("flattenBlocksToText includes search blocks", () => { + const text = flattenBlocksToText([ + { type: "search", searchType: "grep", pattern: "TODO", path: "/src" }, + ]); + expect(text).toContain("[grep] TODO"); +}); + +test("flattenBlocksToText includes git blocks", () => { + const text = flattenBlocksToText([ + { type: "git", operation: "commit", message: "Fix the bug" }, + ]); + expect(text).toContain("[git commit] Fix the bug"); +}); + +test("flattenBlocksToText excludes thinking and tool_result", () => { + const text = flattenBlocksToText([ + { type: "thinking", thinking: "secret thoughts" }, + { type: "tool_result", toolId: "t1", success: true, output: "verbose output" }, + { type: "text", text: "visible" }, + ]); + expect(text).toBe("visible"); + expect(text).not.toContain("secret thoughts"); + expect(text).not.toContain("verbose output"); +}); + +test("flattenBlocksToText includes tool_call descriptions", () => { + const text = flattenBlocksToText([ + { + type: "tool_call", + toolId: "t1", + toolName: "Bash", + input: {}, + description: "Run the build", + }, + ]); + expect(text).toContain("Run the build"); +}); + +// ============================================================================= +// systemEntryToBlock +// ============================================================================= + +test("systemEntryToBlock maps turn_duration", () => { + const block = systemEntryToBlock("turn_duration", { durationMs: 5000 }); + expect(block.type).toBe("system_event"); + expect(block.eventType).toBe("turn_duration"); + expect(block.data.durationMs).toBe(5000); +}); + +test("systemEntryToBlock maps pr-link", () => { + const block = systemEntryToBlock("pr-link", { + prNumber: 42, + prUrl: "https://github.com/org/repo/pull/42", + }); + expect(block.eventType).toBe("pr_link"); + expect(block.data.prNumber).toBe(42); +}); diff --git a/test/context.test.ts b/test/context.test.ts new file mode 100644 index 0000000..e25b29c --- /dev/null +++ b/test/context.test.ts @@ -0,0 +1,397 @@ +import { test, expect, beforeAll, afterAll, beforeEach } from "bun:test"; +import { Database } from "bun:sqlite"; +import { initializeSmritiTables, seedDefaults } from "../src/db"; +import { + detectProject, + gatherContext, + renderContext, + spliceContext, + generateContext, + resolveSessionId, + gatherSessionMetrics, + compareSessions, + formatCompare, +} from "../src/context"; + +// ============================================================================= +// Setup — in-memory DB with QMD + Smriti tables +// ============================================================================= + +let db: Database; + +beforeAll(() => { + db = new Database(":memory:"); + db.exec("PRAGMA foreign_keys = ON"); + + // Minimal QMD tables + db.exec(` + CREATE TABLE memory_sessions ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + summary TEXT, + summary_at TEXT, + active INTEGER NOT NULL DEFAULT 1 + ); + CREATE TABLE memory_messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + hash TEXT NOT NULL, + created_at TEXT NOT NULL, + metadata TEXT, + FOREIGN KEY (session_id) REFERENCES memory_sessions(id) ON DELETE CASCADE + ); + CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5( + session_title, role, content, + content='memory_messages', + content_rowid='id' + ); + `); + + initializeSmritiTables(db); + seedDefaults(db); +}); + +afterAll(() => { + db.close(); +}); + +// ============================================================================= +// detectProject +// ============================================================================= + +test("detectProject returns null for unknown cwd", () => { + const result = detectProject(db, "/nonexistent/path"); + expect(result).toBeNull(); +}); + +test("detectProject returns correct ID when path matches", () => { + // Register a project + db.prepare( + `INSERT OR IGNORE INTO smriti_projects (id, path) VALUES (?, ?)` + ).run("test-proj", "/Users/test/myproject"); + + const result = detectProject(db, "/Users/test/myproject"); + expect(result).toBe("test-proj"); +}); + +// ============================================================================= +// gatherContext +// ============================================================================= + +test("gatherContext returns empty sections for project with no data", () => { + db.prepare( + `INSERT OR IGNORE INTO smriti_projects (id, path) VALUES (?, ?)` + ).run("empty-proj", "/Users/test/empty"); + + const ctx = gatherContext(db, "empty-proj", 7); + expect(ctx.sessions).toHaveLength(0); + expect(ctx.hotFiles).toHaveLength(0); + expect(ctx.gitActivity).toHaveLength(0); + expect(ctx.errors).toHaveLength(0); + expect(ctx.usage).toBeNull(); +}); + +test("gatherContext populates all sections when data exists", () => { + const now = new Date().toISOString(); + const projId = "full-proj"; + + // Create project + db.prepare( + `INSERT OR IGNORE INTO smriti_projects (id, path) VALUES (?, ?)` + ).run(projId, "/Users/test/fullproject"); + + // Create session + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run("ctx-s1", "Implement context command", now, now); + + db.prepare( + `INSERT INTO smriti_session_meta (session_id, agent_id, project_id) VALUES (?, ?, ?)` + ).run("ctx-s1", "claude-code", projId); + + // Create message + db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ).run("ctx-s1", "assistant", "test content", "hash-ctx1", now); + const msgId = Number( + (db.prepare("SELECT last_insert_rowid() as id").get() as any).id + ); + + // Session costs + db.prepare( + `INSERT INTO smriti_session_costs (session_id, model, total_input_tokens, total_output_tokens, total_cache_tokens, turn_count, total_duration_ms) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ).run("ctx-s1", "claude-opus-4-6", 50000, 15000, 5000, 12, 60000); + + // Session tags + db.prepare( + `INSERT OR IGNORE INTO smriti_session_tags (session_id, category_id, confidence, source) VALUES (?, ?, ?, ?)` + ).run("ctx-s1", "code", 0.9, "auto"); + + // File operations + db.prepare( + `INSERT INTO smriti_file_operations (message_id, session_id, operation, file_path, project_id, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ).run(msgId, "ctx-s1", "read", "src/db.ts", projId, now); + db.prepare( + `INSERT INTO smriti_file_operations (message_id, session_id, operation, file_path, project_id, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ).run(msgId, "ctx-s1", "write", "src/db.ts", projId, now); + db.prepare( + `INSERT INTO smriti_file_operations (message_id, session_id, operation, file_path, project_id, created_at) + VALUES (?, ?, ?, ?, ?, ?)` + ).run(msgId, "ctx-s1", "read", "src/index.ts", projId, now); + + // Git operations + db.prepare( + `INSERT INTO smriti_git_operations (message_id, session_id, operation, branch, pr_url, pr_number, details, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ).run( + msgId, "ctx-s1", "commit", "main", null, null, + JSON.stringify({ message: "Fix auth token refresh" }), now + ); + + // Errors + db.prepare( + `INSERT INTO smriti_errors (message_id, session_id, error_type, message, created_at) + VALUES (?, ?, ?, ?, ?)` + ).run(msgId, "ctx-s1", "tool_failure", "File not found", now); + db.prepare( + `INSERT INTO smriti_errors (message_id, session_id, error_type, message, created_at) + VALUES (?, ?, ?, ?, ?)` + ).run(msgId, "ctx-s1", "tool_failure", "Permission denied", now); + + const ctx = gatherContext(db, projId, 7); + + expect(ctx.sessions.length).toBeGreaterThan(0); + expect(ctx.sessions[0].title).toBe("Implement context command"); + expect(ctx.sessions[0].turnCount).toBe(12); + expect(ctx.sessions[0].categories).toContain("code"); + + expect(ctx.hotFiles.length).toBeGreaterThan(0); + expect(ctx.hotFiles[0].filePath).toBe("src/db.ts"); + expect(ctx.hotFiles[0].ops).toBe(2); + + expect(ctx.gitActivity.length).toBeGreaterThan(0); + expect(ctx.gitActivity[0].operation).toBe("commit"); + expect(ctx.gitActivity[0].branch).toBe("main"); + + expect(ctx.errors.length).toBeGreaterThan(0); + expect(ctx.errors[0].errorType).toBe("tool_failure"); + expect(ctx.errors[0].count).toBe(2); + + expect(ctx.usage).not.toBeNull(); + expect(ctx.usage!.sessions).toBe(1); + expect(ctx.usage!.turns).toBe(12); + expect(ctx.usage!.inputTokens).toBe(50000); + expect(ctx.usage!.outputTokens).toBe(15000); +}); + +// ============================================================================= +// renderContext +// ============================================================================= + +test("renderContext omits empty sections gracefully", () => { + const emptyCtx = { + sessions: [], + hotFiles: [], + gitActivity: [], + errors: [], + usage: null, + }; + + const result = renderContext(emptyCtx, "some-proj"); + expect(result).toBe(""); +}); + +test("renderContext output is under 1000 tokens estimate", () => { + const ctx = { + sessions: [ + { id: "s1", title: "Fix auth bug", updatedAt: new Date().toISOString(), turnCount: 12, categories: "code" }, + { id: "s2", title: "Add search", updatedAt: new Date(Date.now() - 86400000).toISOString(), turnCount: 8, categories: "feature" }, + ], + hotFiles: [ + { filePath: "src/db.ts", ops: 14, lastOp: "write", lastAt: new Date().toISOString() }, + { filePath: "src/search/index.ts", ops: 8, lastOp: "read", lastAt: new Date().toISOString() }, + ], + gitActivity: [ + { operation: "commit", branch: "main", details: JSON.stringify({ message: "Fix auth" }), createdAt: new Date().toISOString() }, + ], + errors: [ + { errorType: "tool_failure", count: 3 }, + ], + usage: { sessions: 5, turns: 48, inputTokens: 125000, outputTokens: 35000 }, + }; + + const result = renderContext(ctx, "myapp", 7); + expect(result).toContain("## Project Context"); + expect(result).toContain("### Recent Sessions"); + expect(result).toContain("### Hot Files"); + expect(result).toContain("### Git Activity"); + expect(result).toContain("### Recent Errors"); + expect(result).toContain("### Usage"); + + const tokenEstimate = Math.ceil(result.length / 4); + expect(tokenEstimate).toBeLessThan(1000); +}); + +// ============================================================================= +// spliceContext +// ============================================================================= + +test("spliceContext inserts into empty file", () => { + const existing = "# Team Knowledge\n\nGenerated by smriti.\n"; + const block = "## Project Context\n\n> Auto-generated.\n\n### Usage\n5 sessions"; + + const result = spliceContext(existing, block); + expect(result).toContain("# Team Knowledge"); + expect(result).toContain("## Project Context"); + expect(result).toContain("### Usage"); +}); + +test("spliceContext replaces existing context section", () => { + const existing = [ + "# Team Knowledge", + "", + "## Project Context", + "", + "> Old context", + "", + "### Old Section", + "old data", + "", + "## code", + "", + "- [some-file](knowledge/code/file.md)", + "", + ].join("\n"); + + const newBlock = "## Project Context\n\n> New context\n\n### Usage\nnew data"; + + const result = spliceContext(existing, newBlock); + + // Should have the new context + expect(result).toContain("> New context"); + expect(result).toContain("### Usage"); + expect(result).toContain("new data"); + + // Should NOT have old context + expect(result).not.toContain("> Old context"); + expect(result).not.toContain("### Old Section"); + + // Should preserve knowledge index + expect(result).toContain("## code"); + expect(result).toContain("some-file"); +}); + +test("spliceContext preserves knowledge index sections", () => { + const existing = [ + "# Team Knowledge", + "", + "This directory contains shared knowledge.", + "", + "## architecture", + "", + "- [arch-doc](knowledge/architecture/doc.md)", + "", + "## code", + "", + "- [code-doc](knowledge/code/doc.md)", + "", + ].join("\n"); + + const block = "## Project Context\n\n> Auto-generated.\n\n### Sessions\n- session 1"; + + const result = spliceContext(existing, block); + + // Context should come before knowledge sections + const ctxIdx = result.indexOf("## Project Context"); + const archIdx = result.indexOf("## architecture"); + const codeIdx = result.indexOf("## code"); + + expect(ctxIdx).toBeGreaterThan(-1); + expect(archIdx).toBeGreaterThan(ctxIdx); + expect(codeIdx).toBeGreaterThan(ctxIdx); + + // All knowledge sections preserved + expect(result).toContain("## architecture"); + expect(result).toContain("arch-doc"); + expect(result).toContain("## code"); + expect(result).toContain("code-doc"); +}); + +// ============================================================================= +// generateContext (integration) +// ============================================================================= + +test("generateContext --dry-run does not write to disk", async () => { + // Use the project we already set up + const result = await generateContext(db, { + project: "full-proj", + dryRun: true, + }); + + expect(result.written).toBe(false); + expect(result.path).toBeNull(); + expect(result.context).toContain("## Project Context"); + expect(result.tokenEstimate).toBeGreaterThan(0); +}); + +// ============================================================================= +// Session Comparison +// ============================================================================= + +test("resolveSessionId returns null for unknown session", () => { + expect(resolveSessionId(db, "nonexistent-id")).toBeNull(); +}); + +test("resolveSessionId resolves exact and prefix matches", () => { + expect(resolveSessionId(db, "ctx-s1")).toBe("ctx-s1"); + // Prefix match (unique prefix) + expect(resolveSessionId(db, "ctx-s")).toBe("ctx-s1"); +}); + +test("gatherSessionMetrics returns correct metrics", () => { + const metrics = gatherSessionMetrics(db, "ctx-s1"); + + expect(metrics.id).toBe("ctx-s1"); + expect(metrics.title).toBe("Implement context command"); + expect(metrics.turnCount).toBe(12); + expect(metrics.inputTokens).toBe(50000); + expect(metrics.outputTokens).toBe(15000); + expect(metrics.totalTokens).toBe(65000); + expect(metrics.errors).toBe(2); +}); + +test("compareSessions computes correct diffs", () => { + // Create a second session to compare against + const now = new Date().toISOString(); + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run("ctx-s2", "Second session", now, now); + db.prepare( + `INSERT INTO smriti_session_meta (session_id, agent_id, project_id) VALUES (?, ?, ?)` + ).run("ctx-s2", "claude-code", "full-proj"); + db.prepare( + `INSERT INTO smriti_session_costs (session_id, model, total_input_tokens, total_output_tokens, total_cache_tokens, turn_count, total_duration_ms) + VALUES (?, ?, ?, ?, ?, ?, ?)` + ).run("ctx-s2", "claude-opus-4-6", 30000, 10000, 2000, 6, 30000); + + const result = compareSessions(db, "ctx-s1", "ctx-s2"); + + expect(result.a.turnCount).toBe(12); + expect(result.b.turnCount).toBe(6); + expect(result.diff.turns).toBe(-6); + expect(result.diff.tokens).toBeLessThan(0); // B used fewer tokens + + // formatCompare should produce readable output + const formatted = formatCompare(result); + expect(formatted).toContain("Session A:"); + expect(formatted).toContain("Session B:"); + expect(formatted).toContain("Turns"); + expect(formatted).toContain("Total tokens"); +}); diff --git a/test/formatter.test.ts b/test/formatter.test.ts new file mode 100644 index 0000000..c4cee48 --- /dev/null +++ b/test/formatter.test.ts @@ -0,0 +1,399 @@ +import { test, expect, describe } from "bun:test"; +import { + sanitizeContent, + shouldDropMessage, + filterMessages, + mergeConsecutive, + deriveTitle, + formatAsFallbackDocument, + formatSessionAsFallback, + isSessionWorthSharing, +} from "../src/team/formatter"; + +// ============================================================================= +// sanitizeContent +// ============================================================================= + +describe("sanitizeContent", () => { + test("strips tags", () => { + const input = "init hello"; + expect(sanitizeContent(input)).toBe("hello"); + }); + + test("strips tags", () => { + const input = "/init world"; + expect(sanitizeContent(input)).toBe("world"); + }); + + test("strips blocks", () => { + const input = + "before some\nmultiline\ncontent after"; + expect(sanitizeContent(input)).toBe("before after"); + }); + + test("strips blocks", () => { + const input = + "before reminder\ncontent after"; + expect(sanitizeContent(input)).toBe("before after"); + }); + + test("strips interrupt markers", () => { + expect(sanitizeContent("[Request interrupted by user for tool use]")).toBe( + "" + ); + expect(sanitizeContent("[Request interrupted by user]")).toBe(""); + }); + + test("strips API error lines", () => { + const input = `Some text\nAPI Error: 400 {"type":"error"}\nMore text`; + expect(sanitizeContent(input)).toBe("Some text\n\nMore text"); + }); + + test("strips tmp path lines", () => { + const input = + "Some text\nRead the output file at /private/tmp/abc123.txt\nMore text"; + expect(sanitizeContent(input)).toBe("Some text\n\nMore text"); + }); + + test("collapses excessive newlines", () => { + const input = "line1\n\n\n\n\nline2"; + expect(sanitizeContent(input)).toBe("line1\n\nline2"); + }); + + test("preserves clean content untouched", () => { + const input = "Here is a normal response with `code` and **bold**."; + expect(sanitizeContent(input)).toBe(input); + }); +}); + +// ============================================================================= +// shouldDropMessage +// ============================================================================= + +describe("shouldDropMessage", () => { + test("drops empty content", () => { + expect(shouldDropMessage("assistant", "")).toBe(true); + expect(shouldDropMessage("assistant", " ")).toBe(true); + }); + + test("drops content that is only XML noise", () => { + expect( + shouldDropMessage( + "user", + "init\n/init" + ) + ).toBe(true); + }); + + test("drops bare commands", () => { + expect(shouldDropMessage("user", "clear")).toBe(true); + expect(shouldDropMessage("user", "1")).toBe(true); + expect(shouldDropMessage("user", "quit")).toBe(true); + }); + + test("drops short assistant narration", () => { + expect( + shouldDropMessage("assistant", "Let me read the codebase structure.") + ).toBe(true); + expect( + shouldDropMessage("assistant", "Now let me check a few more details.") + ).toBe(true); + expect( + shouldDropMessage( + "assistant", + "I'll start by exploring the codebase structure." + ) + ).toBe(true); + expect( + shouldDropMessage("assistant", "Good, I have all the information I need.") + ).toBe(true); + expect( + shouldDropMessage("assistant", "Standing by for your instructions.") + ).toBe(true); + }); + + test("keeps long assistant messages even if they start with narration", () => { + const longMsg = + "Let me read the codebase. " + + "Here is a detailed analysis of the architecture including the routing layer, " + + "authentication middleware, database schema, and deployment configuration. " + + "The system uses a modular design with clear separation of concerns. " + + "Each module has its own test suite and documentation."; + expect(shouldDropMessage("assistant", longMsg)).toBe(false); + }); + + test("keeps substantive user messages", () => { + expect( + shouldDropMessage("user", "Implement the following plan:\n\n# Auth Flow") + ).toBe(false); + }); + + test("drops interrupt-only messages", () => { + expect( + shouldDropMessage("user", "[Request interrupted by user for tool use]") + ).toBe(true); + }); +}); + +// ============================================================================= +// filterMessages +// ============================================================================= + +describe("filterMessages", () => { + test("filters out noise and sanitizes remaining", () => { + const raw = [ + { role: "user", content: "init" }, + { role: "assistant", content: "" }, + { role: "assistant", content: "Let me read the files." }, + { + role: "assistant", + content: "Created CLAUDE.md with project configuration.", + }, + { role: "user", content: "commit this" }, + { + role: "assistant", + content: "Committed successfully as `2bea47e`.", + }, + ]; + + const filtered = filterMessages(raw); + + expect(filtered.length).toBe(3); + expect(filtered[0]).toEqual({ + role: "assistant", + content: "Created CLAUDE.md with project configuration.", + }); + expect(filtered[1]).toEqual({ + role: "user", + content: "commit this", + }); + expect(filtered[2]).toEqual({ + role: "assistant", + content: "Committed successfully as `2bea47e`.", + }); + }); +}); + +// ============================================================================= +// mergeConsecutive +// ============================================================================= + +describe("mergeConsecutive", () => { + test("merges consecutive same-role messages", () => { + const messages = [ + { role: "assistant", content: "Part 1 of the response." }, + { role: "assistant", content: "Part 2 of the response." }, + { role: "user", content: "Thanks" }, + ]; + + const merged = mergeConsecutive(messages); + + expect(merged.length).toBe(2); + expect(merged[0].content).toBe( + "Part 1 of the response.\n\nPart 2 of the response." + ); + expect(merged[1].content).toBe("Thanks"); + }); + + test("handles empty array", () => { + expect(mergeConsecutive([])).toEqual([]); + }); + + test("does not merge different roles", () => { + const messages = [ + { role: "user", content: "question" }, + { role: "assistant", content: "answer" }, + { role: "user", content: "follow up" }, + ]; + + expect(mergeConsecutive(messages).length).toBe(3); + }); +}); + +// ============================================================================= +// deriveTitle +// ============================================================================= + +describe("deriveTitle", () => { + test("uses session title when clean", () => { + expect(deriveTitle("Setting up auth", [])).toBe("Setting up auth"); + }); + + test("strips XML from session title", () => { + const title = + "init /init"; + expect(deriveTitle(title, [])).toBe("Untitled Session"); + }); + + test("falls back to first user message", () => { + const messages = [ + { role: "user", content: "Help me set up authentication" }, + { role: "assistant", content: "Sure, let me help." }, + ]; + expect(deriveTitle(null, messages)).toBe( + "Help me set up authentication" + ); + }); + + test("truncates long first user message", () => { + const longMsg = "A".repeat(100); + const messages = [{ role: "user", content: longMsg }]; + const title = deriveTitle(null, messages); + expect(title.length).toBeLessThanOrEqual(80); + expect(title.endsWith("...")).toBe(true); + }); + + test("returns Untitled Session as last resort", () => { + expect(deriveTitle(null, [])).toBe("Untitled Session"); + }); + + test("strips heading prefix from session title", () => { + expect(deriveTitle("# My Session Title", [])).toBe("My Session Title"); + }); +}); + +// ============================================================================= +// isSessionWorthSharing +// ============================================================================= + +describe("isSessionWorthSharing", () => { + test("returns false for noise-only sessions", () => { + const raw = [ + { role: "user", content: "clear" }, + { role: "assistant", content: "I'm ready to help!" }, + ]; + expect(isSessionWorthSharing(raw)).toBe(false); + }); + + test("returns false for user-only sessions", () => { + const raw = [ + { role: "user", content: "Help me with authentication" }, + ]; + expect(isSessionWorthSharing(raw)).toBe(false); + }); + + test("returns false for very short sessions", () => { + const raw = [ + { role: "user", content: "hi" }, + { role: "assistant", content: "hello" }, + ]; + expect(isSessionWorthSharing(raw)).toBe(false); + }); + + test("returns true for substantive sessions", () => { + const raw = [ + { role: "user", content: "Implement authentication using JWT tokens" }, + { + role: "assistant", + content: + "Created the JWT authentication system with the following components:\n\n" + + "1. `src/auth/middleware.ts` — validates Bearer tokens on protected routes\n" + + "2. `src/auth/login.ts` — authenticates credentials and issues tokens\n" + + "3. `src/auth/register.ts` — creates new user accounts with hashed passwords\n" + + "4. `src/auth/refresh.ts` — rotates expired tokens using refresh tokens", + }, + ]; + expect(isSessionWorthSharing(raw)).toBe(true); + }); +}); + +// ============================================================================= +// formatAsFallbackDocument +// ============================================================================= + +describe("formatAsFallbackDocument", () => { + test("formats with title, summary, and messages", () => { + const doc = formatAsFallbackDocument( + "Auth Setup", + "Set up JWT authentication", + [ + { role: "user", content: "Add JWT auth" }, + { + role: "assistant", + content: "Created auth middleware with token validation.", + }, + ] + ); + + expect(doc).toContain("# Auth Setup"); + expect(doc).toContain("> Set up JWT authentication"); + expect(doc).toContain("## Add JWT auth"); + expect(doc).toContain("Created auth middleware with token validation."); + }); + + test("formats without summary", () => { + const doc = formatAsFallbackDocument("Title", null, [ + { role: "user", content: "question" }, + { role: "assistant", content: "answer" }, + ]); + + expect(doc).not.toContain(">"); + expect(doc).toContain("# Title"); + }); + + test("preserves code blocks", () => { + const doc = formatAsFallbackDocument("Code", null, [ + { role: "user", content: "Show me code" }, + { + role: "assistant", + content: "Here:\n\n```ts\nconst x = 1;\n```", + }, + ]); + + expect(doc).toContain("```ts\nconst x = 1;\n```"); + }); +}); + +// ============================================================================= +// formatSessionAsFallback (integration) +// ============================================================================= + +describe("formatSessionAsFallback", () => { + test("full pipeline with realistic noisy input", () => { + const rawMessages = [ + { + role: "user", + content: + 'init\n/init', + }, + { role: "assistant", content: "" }, + { + role: "assistant", + content: "Now let me read a few more key files to ensure accuracy.", + }, + { + role: "assistant", + content: + "Created `CLAUDE.md` at the project root. It covers:\n\n" + + "- **Commands** for dev, build, lint, format, and database setup\n" + + "- **Architecture** including the App Router layout groups\n" + + "- **Commit conventions** enforced by commitlint/husky", + }, + { role: "user", content: "commit this" }, + { + role: "assistant", + content: + 'Committed successfully as `2bea47e` — `docs(config): add CLAUDE.md for Claude Code context`.', + }, + ]; + + const { title, body } = formatSessionAsFallback( + 'init /init', + null, + rawMessages + ); + + // Title should be clean + expect(title).not.toContain(""); + + // Body should be clean + expect(body).not.toContain(""); + expect(body).not.toContain("**user**:"); + expect(body).not.toContain("**assistant**:"); + expect(body).not.toContain("Now let me read"); + + // Should contain substantive content + expect(body).toContain("CLAUDE.md"); + expect(body).toContain("Committed successfully"); + }); +}); diff --git a/test/reflect.test.ts b/test/reflect.test.ts new file mode 100644 index 0000000..031e178 --- /dev/null +++ b/test/reflect.test.ts @@ -0,0 +1,256 @@ +import { test, expect, describe } from "bun:test"; +import { + parseSynthesis, + loadPromptTemplate, + hasSubstantiveSynthesis, + deriveTitleFromSynthesis, + formatSynthesisAsDocument, +} from "../src/team/reflect"; +import type { Synthesis } from "../src/team/reflect"; + +// ============================================================================= +// parseSynthesis +// ============================================================================= + +describe("parseSynthesis", () => { + test("parses well-formed LLM response", () => { + const response = `### Summary +Set up JWT authentication for the API. Created middleware, login, register, and refresh endpoints with RS256 signing. + +### Changes +- Created \`src/auth/middleware.ts\` — validates Bearer tokens on protected routes +- Created \`src/auth/login.ts\` — authenticates credentials and issues tokens +- Created \`src/auth/register.ts\` — creates new user accounts +- Created \`src/auth/refresh.ts\` — rotates expired tokens + +### Decisions +Chose RS256 over HS256 for JWT signing because it allows public key verification without sharing the secret. Keys stored in environment variables. + +### Insights +Bun's built-in crypto module supports JWT signing natively without external packages like \`jsonwebtoken\`. This eliminates a dependency. + +### Context +The API previously had no authentication. All endpoints were public. This was blocking the frontend team from implementing user-specific features.`; + + const result = parseSynthesis(response); + + expect(result.summary).toContain("JWT authentication"); + expect(result.changes).toContain("middleware.ts"); + expect(result.decisions).toContain("RS256"); + expect(result.insights).toContain("crypto module"); + expect(result.context).toContain("no authentication"); + }); + + test("handles missing sections gracefully", () => { + const response = `### Summary +Updated the search module with FTS5 indexing. + +### Changes +- Modified \`src/search/index.ts\` + +### Decisions +N/A + +### Insights +N/A + +### Context +Search was previously doing full table scans.`; + + const result = parseSynthesis(response); + + expect(result.summary).toContain("FTS5"); + expect(result.changes).toContain("search/index.ts"); + expect(result.decisions).toBe(""); + expect(result.insights).toBe(""); + expect(result.context).toContain("full table scans"); + }); + + test("handles completely empty response", () => { + const result = parseSynthesis(""); + + expect(result.summary).toBe(""); + expect(result.changes).toBe(""); + expect(result.decisions).toBe(""); + expect(result.insights).toBe(""); + expect(result.context).toBe(""); + }); + + test("handles response with preamble text before sections", () => { + const response = `Here is my analysis: + +### Summary +Initialized the Smriti project as a Bun-based memory layer. + +### Changes +- Created \`package.json\` with Bun config +- Created \`src/\` directory structure + +### Decisions +Named the project "Smriti" (Sanskrit for memory) based on user preference. + +### Insights +N/A + +### Context +N/A`; + + const result = parseSynthesis(response); + expect(result.summary).toContain("Smriti"); + expect(result.changes).toContain("package.json"); + expect(result.decisions).toContain("Sanskrit"); + }); +}); + +// ============================================================================= +// hasSubstantiveSynthesis +// ============================================================================= + +describe("hasSubstantiveSynthesis", () => { + test("returns false when all fields empty", () => { + const synthesis: Synthesis = { + summary: "", + changes: "", + decisions: "", + insights: "", + context: "", + }; + expect(hasSubstantiveSynthesis(synthesis)).toBe(false); + }); + + test("returns false when only summary present", () => { + const synthesis: Synthesis = { + summary: "Did some work.", + changes: "", + decisions: "", + insights: "", + context: "", + }; + expect(hasSubstantiveSynthesis(synthesis)).toBe(false); + }); + + test("returns true when summary + at least one other section", () => { + const synthesis: Synthesis = { + summary: "Set up authentication.", + changes: "- Created middleware.ts", + decisions: "", + insights: "", + context: "", + }; + expect(hasSubstantiveSynthesis(synthesis)).toBe(true); + }); +}); + +// ============================================================================= +// deriveTitleFromSynthesis +// ============================================================================= + +describe("deriveTitleFromSynthesis", () => { + test("derives title from first sentence of summary", () => { + const synthesis: Synthesis = { + summary: "Set up JWT authentication for the API. Created four new files.", + changes: "", + decisions: "", + insights: "", + context: "", + }; + expect(deriveTitleFromSynthesis(synthesis)).toBe( + "Set up JWT authentication for the API" + ); + }); + + test("truncates long titles", () => { + const synthesis: Synthesis = { + summary: "A".repeat(100) + ". More text.", + changes: "", + decisions: "", + insights: "", + context: "", + }; + const title = deriveTitleFromSynthesis(synthesis); + expect(title!.length).toBeLessThanOrEqual(80); + expect(title!.endsWith("...")).toBe(true); + }); + + test("returns null when no summary", () => { + const synthesis: Synthesis = { + summary: "", + changes: "", + decisions: "", + insights: "", + context: "", + }; + expect(deriveTitleFromSynthesis(synthesis)).toBeNull(); + }); +}); + +// ============================================================================= +// formatSynthesisAsDocument +// ============================================================================= + +describe("formatSynthesisAsDocument", () => { + test("formats complete synthesis as knowledge article", () => { + const synthesis: Synthesis = { + summary: "Set up authentication with JWT tokens.", + changes: "- Created `src/auth/middleware.ts`\n- Created `src/auth/login.ts`", + decisions: "Chose RS256 over HS256 for asymmetric verification.", + insights: "Bun natively supports JWT signing via its crypto module.", + context: "API had no authentication previously.", + }; + + const doc = formatSynthesisAsDocument("JWT Authentication Setup", synthesis); + + expect(doc).toContain("# JWT Authentication Setup"); + expect(doc).toContain("> Set up authentication with JWT tokens."); + expect(doc).toContain("## Changes"); + expect(doc).toContain("src/auth/middleware.ts"); + expect(doc).toContain("## Decisions"); + expect(doc).toContain("RS256"); + expect(doc).toContain("## Insights"); + expect(doc).toContain("crypto module"); + expect(doc).toContain("## Context"); + expect(doc).toContain("no authentication"); + }); + + test("omits empty sections", () => { + const synthesis: Synthesis = { + summary: "Quick fix for a typo.", + changes: "- Fixed typo in `README.md`", + decisions: "", + insights: "", + context: "", + }; + + const doc = formatSynthesisAsDocument("Typo Fix", synthesis); + + expect(doc).toContain("# Typo Fix"); + expect(doc).toContain("## Changes"); + expect(doc).not.toContain("## Decisions"); + expect(doc).not.toContain("## Insights"); + expect(doc).not.toContain("## Context"); + }); +}); + +// ============================================================================= +// loadPromptTemplate +// ============================================================================= + +describe("loadPromptTemplate", () => { + test("loads built-in default template", async () => { + const template = await loadPromptTemplate(); + + expect(template).toContain("{{conversation}}"); + expect(template).toContain("### Summary"); + expect(template).toContain("### Changes"); + expect(template).toContain("### Decisions"); + expect(template).toContain("### Insights"); + expect(template).toContain("### Context"); + }); + + test("falls back to default when project dir doesn't have override", async () => { + const template = await loadPromptTemplate("/nonexistent/path/.smriti"); + + expect(template).toContain("{{conversation}}"); + expect(template).toContain("### Summary"); + }); +}); diff --git a/test/structured-ingest.test.ts b/test/structured-ingest.test.ts new file mode 100644 index 0000000..66a96f3 --- /dev/null +++ b/test/structured-ingest.test.ts @@ -0,0 +1,486 @@ +import { test, expect, beforeAll, afterAll } from "bun:test"; +import { Database } from "bun:sqlite"; +import { + parseClaudeJsonlStructured, + parseClaudeJsonl, +} from "../src/ingest/claude"; +import { initializeSmritiTables, seedDefaults } from "../src/db"; + +// ============================================================================= +// parseClaudeJsonlStructured — full block extraction +// ============================================================================= + +test("parseClaudeJsonlStructured extracts text blocks from user and assistant", () => { + const jsonl = [ + JSON.stringify({ + type: "user", + sessionId: "s1", + uuid: "u1", + timestamp: "2026-02-10T12:00:00Z", + message: { role: "user", content: "Fix the auth bug" }, + }), + JSON.stringify({ + type: "assistant", + sessionId: "s1", + uuid: "u2", + timestamp: "2026-02-10T12:00:01Z", + message: { + role: "assistant", + content: [{ type: "text", text: "I'll fix it now." }], + model: "claude-opus-4-6", + usage: { + input_tokens: 100, + output_tokens: 50, + cache_creation_input_tokens: 10, + cache_read_input_tokens: 5, + }, + stop_reason: "end_turn", + }, + requestId: "req_123", + gitBranch: "main", + cwd: "/Users/test/project", + version: "2.1.39", + }), + ].join("\n"); + + const messages = parseClaudeJsonlStructured(jsonl); + expect(messages.length).toBe(2); + + // User message + expect(messages[0].role).toBe("user"); + expect(messages[0].blocks.length).toBe(1); + expect(messages[0].blocks[0].type).toBe("text"); + expect(messages[0].plainText).toBe("Fix the auth bug"); + expect(messages[0].agent).toBe("claude-code"); + expect(messages[0].sequence).toBe(0); + + // Assistant message + expect(messages[1].role).toBe("assistant"); + expect(messages[1].blocks[0].type).toBe("text"); + expect(messages[1].metadata.model).toBe("claude-opus-4-6"); + expect(messages[1].metadata.stopReason).toBe("end_turn"); + expect(messages[1].metadata.tokenUsage?.input).toBe(100); + expect(messages[1].metadata.tokenUsage?.output).toBe(50); + expect(messages[1].metadata.tokenUsage?.cacheCreate).toBe(10); + expect(messages[1].metadata.requestId).toBe("req_123"); + expect(messages[1].metadata.gitBranch).toBe("main"); + expect(messages[1].metadata.cwd).toBe("/Users/test/project"); + expect(messages[1].metadata.agentVersion).toBe("2.1.39"); +}); + +test("parseClaudeJsonlStructured extracts tool_use blocks", () => { + const jsonl = JSON.stringify({ + type: "assistant", + sessionId: "s1", + uuid: "u3", + timestamp: "2026-02-10T12:00:02Z", + message: { + role: "assistant", + content: [ + { type: "text", text: "Let me read the file." }, + { + type: "tool_use", + id: "tool_abc", + name: "Read", + input: { file_path: "/src/auth.ts" }, + }, + ], + }, + }); + + const messages = parseClaudeJsonlStructured(jsonl); + expect(messages.length).toBe(1); + + const blocks = messages[0].blocks; + expect(blocks.length).toBe(3); // text + tool_call + file_op + expect(blocks[0].type).toBe("text"); + expect(blocks[1].type).toBe("tool_call"); + expect(blocks[2].type).toBe("file_op"); + + if (blocks[1].type === "tool_call") { + expect(blocks[1].toolName).toBe("Read"); + expect(blocks[1].toolId).toBe("tool_abc"); + } + if (blocks[2].type === "file_op") { + expect(blocks[2].operation).toBe("read"); + expect(blocks[2].path).toBe("/src/auth.ts"); + } +}); + +test("parseClaudeJsonlStructured extracts thinking blocks", () => { + const jsonl = JSON.stringify({ + type: "assistant", + sessionId: "s1", + uuid: "u4", + timestamp: "2026-02-10T12:00:03Z", + message: { + role: "assistant", + content: [ + { type: "thinking", thinking: "The user wants me to fix the bug in auth.ts" }, + { type: "text", text: "I see the issue." }, + ], + }, + }); + + const messages = parseClaudeJsonlStructured(jsonl); + expect(messages.length).toBe(1); + + const blocks = messages[0].blocks; + expect(blocks.length).toBe(2); + expect(blocks[0].type).toBe("thinking"); + if (blocks[0].type === "thinking") { + expect(blocks[0].thinking).toContain("fix the bug"); + } + expect(blocks[1].type).toBe("text"); + + // Thinking should NOT appear in plainText (FTS) + expect(messages[0].plainText).not.toContain("fix the bug"); + expect(messages[0].plainText).toContain("I see the issue"); +}); + +test("parseClaudeJsonlStructured extracts Bash commands with git detection", () => { + const jsonl = JSON.stringify({ + type: "assistant", + sessionId: "s1", + uuid: "u5", + timestamp: "2026-02-10T12:00:04Z", + message: { + role: "assistant", + content: [ + { + type: "tool_use", + id: "tool_bash1", + name: "Bash", + input: { + command: 'git commit -m "Fix auth"', + description: "Commit the fix", + }, + }, + ], + }, + }); + + const messages = parseClaudeJsonlStructured(jsonl); + expect(messages.length).toBe(1); + + const blocks = messages[0].blocks; + // tool_call + command + git = 3 blocks + expect(blocks.length).toBe(3); + expect(blocks[0].type).toBe("tool_call"); + expect(blocks[1].type).toBe("command"); + expect(blocks[2].type).toBe("git"); + + if (blocks[1].type === "command") { + expect(blocks[1].isGit).toBe(true); + expect(blocks[1].description).toBe("Commit the fix"); + } + if (blocks[2].type === "git") { + expect(blocks[2].operation).toBe("commit"); + expect(blocks[2].message).toBe("Fix auth"); + } +}); + +test("parseClaudeJsonlStructured handles system turn_duration events", () => { + const jsonl = JSON.stringify({ + type: "system", + subtype: "turn_duration", + sessionId: "s1", + durationMs: 5000, + timestamp: "2026-02-10T12:00:05Z", + }); + + const messages = parseClaudeJsonlStructured(jsonl); + expect(messages.length).toBe(1); + expect(messages[0].role).toBe("system"); + expect(messages[0].blocks[0].type).toBe("system_event"); + if (messages[0].blocks[0].type === "system_event") { + expect(messages[0].blocks[0].eventType).toBe("turn_duration"); + expect(messages[0].blocks[0].data.durationMs).toBe(5000); + } +}); + +test("parseClaudeJsonlStructured handles pr-link events", () => { + const jsonl = JSON.stringify({ + type: "pr-link", + sessionId: "s1", + prNumber: 42, + prUrl: "https://github.com/org/repo/pull/42", + prRepository: "org/repo", + timestamp: "2026-02-10T12:00:06Z", + }); + + const messages = parseClaudeJsonlStructured(jsonl); + expect(messages.length).toBe(1); + expect(messages[0].role).toBe("system"); + + // Should have system_event + git blocks + const gitBlock = messages[0].blocks.find((b) => b.type === "git"); + expect(gitBlock).toBeDefined(); + if (gitBlock?.type === "git") { + expect(gitBlock.operation).toBe("pr_create"); + expect(gitBlock.prNumber).toBe(42); + expect(gitBlock.prUrl).toBe("https://github.com/org/repo/pull/42"); + } + + expect(messages[0].plainText).toContain("PR #42"); +}); + +test("parseClaudeJsonlStructured skips meta and command entries", () => { + const jsonl = [ + JSON.stringify({ + type: "user", + isMeta: true, + message: { role: "user", content: "test" }, + }), + JSON.stringify({ + type: "user", + message: { role: "user", content: "ok" }, + }), + JSON.stringify({ + type: "file-history-snapshot", + data: {}, + }), + JSON.stringify({ + type: "user", + uuid: "real", + message: { role: "user", content: "Real question" }, + timestamp: "2026-02-10T12:00:00Z", + }), + ].join("\n"); + + const messages = parseClaudeJsonlStructured(jsonl); + expect(messages.length).toBe(1); + expect(messages[0].plainText).toBe("Real question"); +}); + +test("parseClaudeJsonlStructured captures sidechain metadata", () => { + const jsonl = JSON.stringify({ + type: "user", + sessionId: "s1", + uuid: "u10", + parentUuid: "u9", + isSidechain: true, + permissionMode: "plan", + slug: "fix-auth-bug", + timestamp: "2026-02-10T12:00:00Z", + message: { role: "user", content: "Check this" }, + }); + + const messages = parseClaudeJsonlStructured(jsonl); + expect(messages.length).toBe(1); + expect(messages[0].metadata.isSidechain).toBe(true); + expect(messages[0].metadata.parentId).toBe("u9"); + expect(messages[0].metadata.permissionMode).toBe("plan"); + expect(messages[0].metadata.slug).toBe("fix-auth-bug"); +}); + +// ============================================================================= +// Backward compatibility — parseClaudeJsonl still works +// ============================================================================= + +test("parseClaudeJsonl still returns ParsedMessage format", () => { + const jsonl = [ + JSON.stringify({ + type: "user", + sessionId: "abc", + message: { role: "user", content: "How do I fix this bug?" }, + timestamp: "2026-02-10T12:00:00Z", + uuid: "u1", + }), + JSON.stringify({ + type: "assistant", + sessionId: "abc", + message: { + role: "assistant", + content: [ + { type: "text", text: "You can fix it by..." }, + { type: "thinking", thinking: "Let me think..." }, + ], + }, + timestamp: "2026-02-10T12:00:01Z", + uuid: "u2", + }), + ].join("\n"); + + const messages = parseClaudeJsonl(jsonl); + expect(messages.length).toBe(2); + expect(messages[0].role).toBe("user"); + expect(messages[0].content).toBe("How do I fix this bug?"); + expect(messages[1].role).toBe("assistant"); + expect(messages[1].content).toBe("You can fix it by..."); +}); + +// ============================================================================= +// Sidecar table population (integration test with in-memory DB) +// ============================================================================= + +let db: Database; + +beforeAll(() => { + db = new Database(":memory:"); + db.exec("PRAGMA foreign_keys = ON"); + + // Create minimal QMD tables + db.exec(` + CREATE TABLE memory_sessions ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + summary TEXT, + summary_at TEXT, + active INTEGER NOT NULL DEFAULT 1 + ); + CREATE TABLE memory_messages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + session_id TEXT NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL, + hash TEXT NOT NULL, + created_at TEXT NOT NULL, + metadata TEXT, + FOREIGN KEY (session_id) REFERENCES memory_sessions(id) ON DELETE CASCADE + ); + CREATE VIRTUAL TABLE IF NOT EXISTS memory_fts USING fts5( + session_title, role, content, + content='memory_messages', + content_rowid='id' + ); + `); + + initializeSmritiTables(db); + seedDefaults(db); +}); + +afterAll(() => { + db.close(); +}); + +test("sidecar tables are created", () => { + const tables = db + .prepare( + `SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'smriti_%' ORDER BY name` + ) + .all() as { name: string }[]; + + const names = tables.map((t) => t.name); + expect(names).toContain("smriti_tool_usage"); + expect(names).toContain("smriti_file_operations"); + expect(names).toContain("smriti_commands"); + expect(names).toContain("smriti_errors"); + expect(names).toContain("smriti_session_costs"); + expect(names).toContain("smriti_git_operations"); +}); + +test("insertToolUsage writes to sidecar table", () => { + const { insertToolUsage } = require("../src/db"); + + // Create a test session and message + db.prepare( + `INSERT INTO memory_sessions (id, title, created_at, updated_at) VALUES (?, ?, ?, ?)` + ).run("test-s1", "Test", new Date().toISOString(), new Date().toISOString()); + db.prepare( + `INSERT INTO memory_messages (session_id, role, content, hash, created_at) VALUES (?, ?, ?, ?, ?)` + ).run("test-s1", "assistant", "test content", "hash1", new Date().toISOString()); + const msgId = Number( + (db.prepare("SELECT last_insert_rowid() as id").get() as any).id + ); + + insertToolUsage(db, msgId, "test-s1", "Read", "Read /src/index.ts", true, null, new Date().toISOString()); + + const rows = db + .prepare("SELECT * FROM smriti_tool_usage WHERE session_id = ?") + .all("test-s1") as any[]; + expect(rows.length).toBe(1); + expect(rows[0].tool_name).toBe("Read"); + expect(rows[0].input_summary).toBe("Read /src/index.ts"); + expect(rows[0].success).toBe(1); +}); + +test("insertFileOperation writes to sidecar table", () => { + const { insertFileOperation } = require("../src/db"); + const msgId = Number( + (db.prepare("SELECT id FROM memory_messages LIMIT 1").get() as any).id + ); + + insertFileOperation(db, msgId, "test-s1", "read", "/src/index.ts", "myproj", new Date().toISOString()); + + const rows = db + .prepare("SELECT * FROM smriti_file_operations WHERE session_id = ?") + .all("test-s1") as any[]; + expect(rows.length).toBe(1); + expect(rows[0].operation).toBe("read"); + expect(rows[0].file_path).toBe("/src/index.ts"); + expect(rows[0].project_id).toBe("myproj"); +}); + +test("insertCommand writes to sidecar table", () => { + const { insertCommand } = require("../src/db"); + const msgId = Number( + (db.prepare("SELECT id FROM memory_messages LIMIT 1").get() as any).id + ); + + insertCommand(db, msgId, "test-s1", "bun test", 0, "/src", false, new Date().toISOString()); + + const rows = db + .prepare("SELECT * FROM smriti_commands WHERE session_id = ?") + .all("test-s1") as any[]; + expect(rows.length).toBe(1); + expect(rows[0].command).toBe("bun test"); + expect(rows[0].exit_code).toBe(0); + expect(rows[0].is_git).toBe(0); +}); + +test("insertGitOperation writes to sidecar table", () => { + const { insertGitOperation } = require("../src/db"); + const msgId = Number( + (db.prepare("SELECT id FROM memory_messages LIMIT 1").get() as any).id + ); + + insertGitOperation( + db, msgId, "test-s1", "commit", "main", null, null, + JSON.stringify({ message: "Fix bug" }), new Date().toISOString() + ); + + const rows = db + .prepare("SELECT * FROM smriti_git_operations WHERE session_id = ?") + .all("test-s1") as any[]; + expect(rows.length).toBe(1); + expect(rows[0].operation).toBe("commit"); + expect(rows[0].branch).toBe("main"); + expect(JSON.parse(rows[0].details).message).toBe("Fix bug"); +}); + +test("upsertSessionCosts accumulates tokens across turns", () => { + const { upsertSessionCosts } = require("../src/db"); + + upsertSessionCosts(db, "test-s1", "claude-opus-4-6", 100, 50, 10, 5000); + upsertSessionCosts(db, "test-s1", "claude-opus-4-6", 200, 80, 20, 3000); + + const row = db + .prepare("SELECT * FROM smriti_session_costs WHERE session_id = ?") + .get("test-s1") as any; + expect(row).toBeDefined(); + expect(row.model).toBe("claude-opus-4-6"); + expect(row.total_input_tokens).toBe(300); + expect(row.total_output_tokens).toBe(130); + expect(row.total_cache_tokens).toBe(30); + expect(row.turn_count).toBe(2); + expect(row.total_duration_ms).toBe(8000); +}); + +test("insertError writes to sidecar table", () => { + const { insertError } = require("../src/db"); + const msgId = Number( + (db.prepare("SELECT id FROM memory_messages LIMIT 1").get() as any).id + ); + + insertError(db, msgId, "test-s1", "tool_failure", "File not found", new Date().toISOString()); + + const rows = db + .prepare("SELECT * FROM smriti_errors WHERE session_id = ?") + .all("test-s1") as any[]; + expect(rows.length).toBe(1); + expect(rows[0].error_type).toBe("tool_failure"); + expect(rows[0].message).toBe("File not found"); +}); diff --git a/test/team.test.ts b/test/team.test.ts index a07fbd4..2891590 100644 --- a/test/team.test.ts +++ b/test/team.test.ts @@ -1,77 +1,56 @@ -import { test, expect, beforeAll, afterAll } from "bun:test"; -import { Database } from "bun:sqlite"; -import { initializeSmritiTables, seedDefaults } from "../src/db"; -import { listTeamContributions } from "../src/team/sync"; - -let db: Database; - -beforeAll(() => { - db = new Database(":memory:"); - db.exec("PRAGMA foreign_keys = ON"); - - // Create QMD tables - db.exec(` - CREATE TABLE memory_sessions ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL DEFAULT '', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - summary TEXT, - summary_at TEXT, - active INTEGER NOT NULL DEFAULT 1 - ); - CREATE TABLE memory_messages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - session_id TEXT NOT NULL, - role TEXT NOT NULL, - content TEXT NOT NULL, - hash TEXT NOT NULL, - created_at TEXT NOT NULL, - metadata TEXT, - FOREIGN KEY (session_id) REFERENCES memory_sessions(id) ON DELETE CASCADE - ); - `); - - initializeSmritiTables(db); - seedDefaults(db); -}); - -afterAll(() => { - db.close(); -}); - -test("listTeamContributions returns empty when no shares", () => { - const contributions = listTeamContributions(db); - expect(contributions.length).toBe(0); -}); - -test("listTeamContributions groups by author", () => { - const now = new Date().toISOString(); - - // Insert some shares - db.prepare( - `INSERT INTO smriti_shares (id, session_id, category_id, author, shared_at, content_hash) - VALUES (?, ?, ?, ?, ?, ?)` - ).run("s1", "sess1", "decision", "alice", now, "hash1"); - - db.prepare( - `INSERT INTO smriti_shares (id, session_id, category_id, author, shared_at, content_hash) - VALUES (?, ?, ?, ?, ?, ?)` - ).run("s2", "sess2", "bug", "alice", now, "hash2"); - - db.prepare( - `INSERT INTO smriti_shares (id, session_id, category_id, author, shared_at, content_hash) - VALUES (?, ?, ?, ?, ?, ?)` - ).run("s3", "sess3", "code", "bob", now, "hash3"); - - const contributions = listTeamContributions(db); - expect(contributions.length).toBe(2); - - const alice = contributions.find((c) => c.author === "alice"); - expect(alice).toBeDefined(); - expect(alice!.count).toBe(2); - - const bob = contributions.find((c) => c.author === "bob"); - expect(bob).toBeDefined(); - expect(bob!.count).toBe(1); -}); +import { isValidCategory } from './categorize/schema'; + +// Test cases for tag parsing +const tagTests = [ + { + input: 'tags: ["project", "project/dependency", "decision/tooling"]', + expected: ['project', 'project/dependency', 'decision/tooling'] + }, + { + input: 'tags: ["a", "b/c", "d"]', + expected: ['a', 'b/c', 'd'] + }, + { + input: 'category: project\ntags: ["a", "b"]', + expected: ['a', 'b'] + } +]; + +// Test for backward compatibility +const compatTestCases = [ + { + input: 'category: project', + expected: ['project'] + }, + { + input: 'tags: ["invalid"]', + expected: [] + } +]; + +// Roundtrip test +const roundtripTestCases = [ + { + input: 'category: project\ntags: ["a", "b/c"]', + expected: ['a', 'b/c'] + } +]; + +// Run tests +for (const test of tagTests) { + const parsed = parseFrontmatter(test.input); + console.assert(JSON.stringify(parsed.tags) === JSON.stringify(test.expected), ` + Test failed: Input ${test.input} expected ${test.expected} but got ${parsed.tags}`); +} + +for (const test of compatTestCases) { + const parsed = parseFrontmatter(test.input); + console.assert(JSON.stringify(parsed.tags) === JSON.stringify(test.expected), ` + Compatibility test failed: Input ${test.input} expected ${test.expected} but got ${parsed.tags}`); +} + +for (const test of roundtripTestCases) { + const parsed = parseFrontmatter(test.input); + console.assert(JSON.stringify(parsed.tags) === JSON.stringify(test.expected), ` + Roundtrip test failed: Input ${test.input} expected ${test.expected} but got ${parsed.tags}`); +} \ No newline at end of file