V2 (extended flag)

.gitattributes

@@ -1 +1,2 @@
 datasets/v1-compressed/** filter=lfs diff=lfs merge=lfs -text
+datasets/extended-compressed/** filter=lfs diff=lfs merge=lfs -text

.github/workflows/build_wheels.yaml

@@ -117,7 +117,7 @@ jobs:
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_TEST_REQUIRES: pytest
-          CIBW_TEST_COMMAND: pytest {package}/tests
+          CIBW_TEST_COMMAND: pytest {package}/tests --no-dataset
 
       - uses: actions/upload-artifact@v4
         with:
@@ -164,7 +164,7 @@ jobs:
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_TEST_REQUIRES: pytest
-          CIBW_TEST_COMMAND: pytest {package}/tests
+          CIBW_TEST_COMMAND: pytest {package}/tests --no-dataset
 
       - uses: actions/upload-artifact@v4
         with:
@@ -212,7 +212,7 @@ jobs:
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_TEST_REQUIRES: pytest
-          CIBW_TEST_COMMAND: pytest {package}/tests
+          CIBW_TEST_COMMAND: pytest {package}/tests --no-dataset
 
       - uses: actions/upload-artifact@v4
         with:
@@ -254,7 +254,7 @@ jobs:
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_TEST_REQUIRES: pytest
-          CIBW_TEST_COMMAND: pytest {package}/tests
+          CIBW_TEST_COMMAND: pytest {package}/tests --no-dataset
 
       - uses: actions/upload-artifact@v4
         with:
@@ -302,7 +302,7 @@ jobs:
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_TEST_REQUIRES: pytest
-          CIBW_TEST_COMMAND: pytest {package}/tests
+          CIBW_TEST_COMMAND: pytest {package}/tests --no-dataset
 
       - uses: actions/upload-artifact@v4
         with:
@@ -344,7 +344,7 @@ jobs:
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_TEST_REQUIRES: pytest
-          CIBW_TEST_COMMAND: pytest {package}/tests
+          CIBW_TEST_COMMAND: pytest {package}/tests --no-dataset
 
       - uses: actions/upload-artifact@v4
         with:
@@ -385,7 +385,7 @@ jobs:
           CIBW_ARCHS: ${{ matrix.cibw_archs }}
           CIBW_BUILD: ${{ matrix.cibw_build }}
           CIBW_TEST_REQUIRES: pytest
-          CIBW_TEST_COMMAND: pytest {package}/tests
+          CIBW_TEST_COMMAND: pytest {package}/tests --no-dataset
 
       - uses: actions/upload-artifact@v4
         with:

.github/workflows/javascript.yaml

@@ -99,9 +99,9 @@ jobs:
           cd build
           HASH=$(sha256sum enwik8-js.tamp | cut -d' ' -f1)
           echo "Compression hash: $HASH"
-          if [ "$HASH" != "02e05af059a0040d641988075cf1dfc479a084f9a34b5c8a348354211c5fa038" ]; then
+          if [ "$HASH" != "dd5e431b0cbaa6ee001b10493c2b08e6235f42e3f4ce00958e88b1b97581872e" ]; then
             echo "❌ Hash mismatch!"
-            echo "Expected: 02e05af059a0040d641988075cf1dfc479a084f9a34b5c8a348354211c5fa038"
+            echo "Expected: dd5e431b0cbaa6ee001b10493c2b08e6235f42e3f4ce00958e88b1b97581872e"
             echo "Got:      $HASH"
             exit 1
           fi

.github/workflows/tests.yaml

@@ -94,8 +94,48 @@ jobs:
           source .venv/bin/activate
           SKIP=wasm-eslint,wasm-npm-test,wasm-file-validation,typescript-check,package-json-lint pre-commit run --show-diff-on-failure --color=always --all-files
 
+  cache-lfs:
+    name: 'Cache LFS files'
+    runs-on: ubuntu-latest
+    outputs:
+      cache-key: ${{ steps.lfs-key.outputs.key }}
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          lfs: false
+
+      - name: Compute LFS cache key
+        id: lfs-key
+        run: |
+          # Hash pointer files before they get replaced by git lfs pull
+          # Use find for reliable recursive globbing, sort for deterministic order
+          hash=$(find datasets/v1-compressed datasets/extended-compressed -name '*.tamp' -type f | sort | xargs cat | sha256sum | cut -d' ' -f1)
+          echo "key=lfs-${hash}" >> $GITHUB_OUTPUT
+          echo "Cache key: lfs-${hash}"
+
+      - name: Restore LFS cache
+        uses: actions/cache/restore@v4
+        id: lfs-cache
+        with:
+          path: .git/lfs
+          key: ${{ steps.lfs-key.outputs.key }}
+
+      - name: Pull LFS files
+        if: steps.lfs-cache.outputs.cache-hit != 'true'
+        run: git lfs pull
+
+      - name: Save LFS cache
+        if: steps.lfs-cache.outputs.cache-hit != 'true'
+        uses: actions/cache/save@v4
+        with:
+          path: .git/lfs
+          key: ${{ steps.lfs-key.outputs.key }}
+
   test:
     name: 'Test Python ${{ matrix.python-version }}'
+    needs: cache-lfs
     timeout-minutes: 15
     runs-on: ubuntu-latest
     strategy:
@@ -110,7 +150,17 @@ jobs:
         uses: actions/checkout@v4
         with:
           submodules: recursive
-          lfs: true
+          lfs: false
+
+      - name: Restore LFS cache
+        uses: actions/cache/restore@v4
+        with:
+          path: .git/lfs
+          key: ${{ needs.cache-lfs.outputs.cache-key }}
+          fail-on-cache-miss: true
+
+      - name: Pull LFS files
+        run: git lfs pull
 
       - name: Set up python 3.13 (for Poetry)
         id: setup-python-system
@@ -217,7 +267,7 @@ jobs:
         implementation: [desktop, embedded]
     env:
       POETRY_HOME: '~/poetry'
-      EXPECTED_COMPRESSED_HASH: '02e05af059a0040d641988075cf1dfc479a084f9a34b5c8a348354211c5fa038'
+      EXPECTED_COMPRESSED_HASH: 'dd5e431b0cbaa6ee001b10493c2b08e6235f42e3f4ce00958e88b1b97581872e'
 
     steps:
       - name: Check out repository

.gitignore

@@ -249,10 +249,12 @@ Temporary Items
 # Compression benchmark datasets
 datasets/*
 !datasets/v1-compressed/
+!datasets/extended-compressed/
 enwik8*
 *.pkl
 *.tamp
 !datasets/v1-compressed/**
+!datasets/extended-compressed/**
 
 # Cython-generated files
 tamp/_c_compressor.c
@@ -435,6 +437,9 @@ wasm/build/
 *.swo
 *~
 
+# clangd (C/C++ language server)
+.clangd
+
 # Emacs
 *~
 \#*\#

.pre-commit-config.yaml

@@ -57,10 +57,13 @@ repos:
         args: ['-style=file', '-i']
         exclude: ^espidf/tamp/private/tamp_search\.hpp$
 
-  - repo: https://github.com/pocc/pre-commit-hooks
-    rev: v1.3.5
+  - repo: local
     hooks:
       - id: cppcheck
+        name: cppcheck
+        entry: cppcheck
+        language: system
+        files: \.(c|h|cpp|hpp)$
         exclude: ^(espidf|mpy_bindings|ctests|tools)/
         args: [
             '-Itamp/_c_src',

CLAUDE.md

@@ -26,9 +26,13 @@ different platforms:
 **Shared C Source:** All implementations use the same C source code in
 `tamp/_c_src/tamp/`:
 
-- `common.h/c` - Shared utilities and data structures
-- `compressor.h/c` - Compression implementation
+- `common.h/c` - Shared utilities, data structures, stream I/O callbacks, and
+  dictionary initialization
+- `compressor.h/c` - Compression implementation (sink/poll low-level API and
+  higher-level compress/flush API)
 - `decompressor.h/c` - Decompression implementation
+- `compressor_find_match_desktop.c` - Desktop-optimized match finding (included
+  by `compressor.c` on non-embedded targets)
 
 ## Development Commands
 
@@ -162,38 +166,81 @@ make website-clean         # Clean website build artifacts
 **WebAssembly Build Process:**
 
 1. `wasm/Makefile` compiles C source to WebAssembly using Emscripten
-2. `wasm/scripts/build.js` generates multiple JS/TS distribution formats
+2. `tsup` (via `npm run build:js`) bundles into multiple JS/TS distribution
+   formats (CJS, ESM, `.d.ts`)
 3. Exports specific C functions and runtime methods for JS interop
 
-**Configuration Flags:**
+**Configuration Flags (compile-time `-D` defines):**
 
-- `TAMP_LAZY_MATCHING=1` - Enable lazy matching optimization (default)
-- `TAMP_ESP32=1` - ESP32-specific optimizations
+- `TAMP_LAZY_MATCHING=1` - Enable lazy matching optimization (default in
+  build.py)
+- `TAMP_ESP32=1` - ESP32-specific optimizations (avoids bitfields for speed)
 - `TAMP_COMPRESSOR`/`TAMP_DECOMPRESSOR` - Include/exclude components
+- `TAMP_EXTENDED=1` - Master switch for extended format: RLE and extended match
+  (default: 1). `TAMP_EXTENDED_COMPRESS` and `TAMP_EXTENDED_DECOMPRESS` can
+  individually override.
+- `TAMP_STREAM=1` - Include stream API (default: 1). Disable with
+  `-DTAMP_STREAM=0` to save ~2.8KB.
+- `TAMP_STREAM_WORK_BUFFER_SIZE=32` - Stack-allocated work buffer for stream API
+  (default: 32 bytes, 256+ recommended for performance)
+- `TAMP_STREAM_MEMORY` / `TAMP_STREAM_STDIO` / `TAMP_STREAM_LITTLEFS` /
+  `TAMP_STREAM_FATFS` - Enable built-in I/O handlers for specific backends
+- `TAMP_USE_EMBEDDED_MATCH=1` - Force embedded `find_best_match` implementation
+  on desktop (for testing)
+
+**Build Environment Variables (Python):**
+
+- `TAMP_SANITIZE=1` - Enable AddressSanitizer + UBSan
+- `TAMP_PROFILE=1` - Enable profiling (line trace, debug info)
+- `TAMP_USE_EMBEDDED_MATCH=1` - Force embedded match finding
+- `TAMP_BUILD_C_EXTENSIONS=0` - Skip building C extensions entirely
+- `CIBUILDWHEEL=1` - CI wheel building mode (disables allowed_to_fail)
 
 ### Testing Strategy
 
 **Multi-layered Testing:**
 
-- **Python tests** (`tests/`) - Core algorithm testing using pytest
+- **Python tests** (`tests/`) - Core algorithm testing using pytest. Includes
+  bit reader/writer, compressor, decompressor, round-trip, CLI, dataset
+  regression, and file interface tests.
 - **WebAssembly tests** (`wasm/test/`) - JS/TS API testing with Node.js test
-  runner
+  runner (`node --test`)
 - **C tests** (`ctests/`) - Low-level C API testing using Unity framework
+  (submodule at `ctests/Unity/`). Includes stream API tests and filesystem
+  integration tests with LittleFS and FatFS RAM backends.
 - **Integration tests** - Cross-platform compatibility and performance
   benchmarks
 
 **Test Data Sources:**
 
-- Enwik8 dataset (100MB) for performance benchmarking
-- Silesia corpus for compression ratio evaluation
+- Enwik8 dataset (100MB) for performance benchmarking (`make download-enwik8`)
+- Silesia corpus for compression ratio evaluation (`make download-silesia`)
 - Custom test cases for edge conditions
 
+### Compressor Architecture
+
+The C compressor uses a two-phase low-level API:
+
+1. `tamp_compressor_sink()` - Copies input bytes into a 16-byte internal ring
+   buffer (cheap/fast)
+2. `tamp_compressor_poll()` - Runs one compression iteration on the internal
+   buffer (computationally intensive)
+
+Higher-level convenience functions (`tamp_compressor_compress`,
+`tamp_compressor_compress_and_flush`) wrap these. Callback variants (`_cb`
+suffix) accept a `tamp_callback_t` progress callback.
+
+The stream API (`tamp_compress_stream`, `tamp_decompress_stream`) provides a
+file-oriented interface using read/write callbacks, supporting multiple I/O
+backends (memory, stdio, LittleFS, FatFS).
+
 ### Memory Management Patterns
 
 **Key Principle:** Fixed memory usage during compression/decompression
 
 - Window size determines memory usage: `(1 << windowBits)` bytes
 - No dynamic allocation during compression/decompression operations
+- Stream API uses a stack-allocated work buffer (`TAMP_STREAM_WORK_BUFFER_SIZE`)
 - Streaming interfaces require explicit resource management (`destroy()` calls
   in JS/TS)
 
@@ -202,7 +249,9 @@ make website-clean         # Clean website build artifacts
 ### Making Changes to Core Algorithm
 
 1. **Modify C source** in `tamp/_c_src/tamp/`
-2. **Rebuild all implementations:**
+2. **Update pure Python reference** in `tamp/compressor.py` /
+   `tamp/decompressor.py` to match
+3. **Rebuild all implementations:**
 
    ```bash
    # Python
@@ -212,11 +261,12 @@ make website-clean         # Clean website build artifacts
    cd wasm && npm run build
    ```
 
-3. **Run comprehensive tests:**
+4. **Run comprehensive tests:**
    ```bash
-   make test              # Python + MicroPython
+   poetry run pytest      # Python tests
+   make c-test            # C unit tests with sanitizers
+   make c-test-embedded   # C tests with embedded match finding
    cd wasm && npm test    # WebAssembly
-   make c-test           # C unit tests
    ```
 
 ### Adding New Features
@@ -232,11 +282,13 @@ make website-clean         # Clean website build artifacts
 - **Use provided benchmarking tools:**
   ```bash
   make on-device-compression-benchmark     # MicroPython performance
-  npm run test:enwik8                     # WebAssembly performance
-  python tools/performance-benchmark.sh   # Python performance
+  cd wasm && npm run test:enwik8          # WebAssembly performance
+  bash tools/performance-benchmark.sh     # Python performance
+  make c-benchmark-stream                 # C stream API benchmark
+  make binary-size                        # ARM binary size table
   ```
-- **Profile with:** `tools/profiler.py` for Python, browser dev tools for
-  WebAssembly
+- **Profile with:** `tools/profiler.py` for Python (requires `TAMP_PROFILE=1`),
+  browser dev tools for WebAssembly
 
 ### Release Process
 
@@ -247,6 +299,29 @@ make website-clean         # Clean website build artifacts
    - WebAssembly npm package
 3. **CI/CD handles** cross-platform builds and testing
 
+### Python Import Fallback Chain
+
+`tamp/__init__.py` imports Compressor/Decompressor using this priority:
+
+1. Viper (MicroPython optimized) - only available on MicroPython
+2. Cython C extensions (`_c_compressor`/`_c_decompressor`) - primary on CPython
+3. Pure Python reference (`compressor.py`/`decompressor.py`) - fallback
+
+When modifying compression behavior, changes to the C source must be mirrored in
+the pure Python reference implementation to keep them in sync.
+
+### CI/CD
+
+GitHub Actions workflows (`.github/workflows/`):
+
+- `tests.yaml` - Lint (ruff, pre-commit) and test across Python 3.9/3.12/3.13
+  and multiple OS. Also runs `c-test` and `c-test-embedded`.
+- `build_wheels.yaml` - Cross-platform wheel builds via cibuildwheel
+- `javascript.yaml` - WebAssembly tests on Node 18/20
+- `mpy_native_module.yaml` - MicroPython native module builds for ARM
+  architectures
+- `esp_upload_component.yml` - ESP-IDF component registry upload
+
 ## Documentation Style
 
 - Avoid "fake" subsections (e.g., bold text like `**Error Promotion:**` acting