diff --git a/dissect/database/sqlite3/sqlite3.py b/dissect/database/sqlite3/sqlite3.py index 0c6a6fb..25b32e6 100644 --- a/dissect/database/sqlite3/sqlite3.py +++ b/dissect/database/sqlite3/sqlite3.py @@ -2,9 +2,9 @@ import itertools import re -import struct from functools import lru_cache from io import BytesIO +from pathlib import Path from typing import TYPE_CHECKING, Any, BinaryIO from dissect.database.sqlite3.c_sqlite3 import c_sqlite3 @@ -15,6 +15,7 @@ NoCellData, ) from dissect.database.sqlite3.util import parse_table_columns_constraints +from dissect.database.sqlite3.wal import WAL, Checkpoint if TYPE_CHECKING: from collections.abc import Iterator @@ -47,19 +48,50 @@ 9: lambda fh: 1, } +# See https://sqlite.org/fileformat2.html#magic_header_string SQLITE3_HEADER_MAGIC = b"SQLite format 3\x00" -WAL_HEADER_MAGIC_LE = 0x377F0682 -WAL_HEADER_MAGIC_BE = 0x377F0683 -WAL_HEADER_MAGIC = {WAL_HEADER_MAGIC_LE, WAL_HEADER_MAGIC_BE} - class SQLite3: - def __init__(self, fh: BinaryIO, wal_fh: BinaryIO | None = None): + """SQLite3 database class. + + Loads a SQLite3 database from the given file-like object or path. If a path is provided (or can be deduced + from the file-like object), a WAL file will be automatically looked for with a few common suffixes. + Optionally a WAL file-like object or path can be directly provided to read changes from the WAL (this takes + priority over the aforementioned WAL lookup). Additionally, a specific checkpoint from the WAL can be applied. + + Args: + fh: The path or file-like object to open a SQLite3 database on. + wal: The path or file-like object to open a SQLite3 WAL file on. + checkpoint: The checkpoint to apply from the WAL file. Can be a :class:`Checkpoint` object or an integer index. + + Raises: + InvalidDatabase: If the file-like object does not look like a SQLite3 database based on the header magic. + + References: + - https://sqlite.org/fileformat2.html + """ + + def __init__( + self, + fh: Path | BinaryIO, + wal: WAL | Path | BinaryIO | None = None, + checkpoint: Checkpoint | int | None = None, + ): + # Use the provided file handle or try to open the file path. + if hasattr(fh, "read"): + name = getattr(fh, "name", None) + path = Path(name) if name else None + else: + path = fh + fh = path.open("rb") + self.fh = fh - self.wal = WAL(wal_fh) if wal_fh else None + self.path = path + self.wal = None + self.checkpoint = None - self.header = c_sqlite3.header(fh) + self.header = c_sqlite3.header(self.fh) if self.header.magic != SQLITE3_HEADER_MAGIC: raise InvalidDatabase("Invalid header magic") @@ -72,10 +104,31 @@ def __init__(self, fh: BinaryIO, wal_fh: BinaryIO | None = None): if self.usable_page_size < 480: raise InvalidDatabase("Usable page size is too small") + if wal: + self.wal = WAL(wal) if not isinstance(wal, WAL) else wal + elif path: + # Check for WAL sidecar next to the DB. + wal_path = path.with_name(f"{path.name}-wal") + if wal_path.exists(): + self.wal = WAL(wal_path) + + # If a checkpoint index was provided, resolve it to a Checkpoint object. + if self.wal and isinstance(checkpoint, int): + if checkpoint < 0 or checkpoint >= len(self.wal.checkpoints): + raise IndexError("WAL checkpoint index out of range") + self.checkpoint = self.wal.checkpoints[checkpoint] + else: + self.checkpoint = checkpoint + self.page = lru_cache(256)(self.page) - def open_wal(self, fh: BinaryIO) -> None: - self.wal = WAL(fh) + def checkpoints(self) -> Iterator[SQLite3]: + """Yield instances of the database at all available checkpoints in the WAL file, if applicable.""" + if not self.wal: + return + + for checkpoint in self.wal.checkpoints: + yield SQLite3(self.fh, self.wal, checkpoint) def table(self, name: str) -> Table | None: name = name.lower() @@ -108,10 +161,33 @@ def indices(self) -> Iterator[Index]: yield Index(self, *cell.values) def raw_page(self, num: int) -> bytes: + """Retrieve the raw frame data for the given page number. + + Reads the page from a checkpoint, if this class was initialized with a WAL checkpoint. + + If a WAL is available, will first check if the WAL contains a more recent version of the page, + otherwise it will read the page from the database file. + + References: + - https://sqlite.org/fileformat2.html#reader_algorithm + """ # Only throw an out of bounds exception if the header contains a page_count. # Some old versions of SQLite3 do not set/update the page_count correctly. if (num < 1 or num > self.header.page_count) and self.header.page_count > 0: raise InvalidPageNumber("Page number exceeds boundaries") + + # If a specific WAL checkpoint was provided, use it instead of the on-disk page. + if self.checkpoint is not None and (frame := self.checkpoint.get(num)): + return frame.data + + # Check if the latest valid instance of the page is committed (either the frame itself + # is the commit frame or it is included in a commit's frames). If so, return that frame's data. + if self.wal: + for commit in reversed(self.wal.commits): + if (frame := commit.get(num)) and frame.valid: + return frame.data + + # Else we read the page from the database file. if num == 1: # Page 1 is root self.fh.seek(len(c_sqlite3.header)) else: @@ -465,127 +541,6 @@ def values(self) -> list[int | float | str | bytes | None]: return self._values -class WAL: - def __init__(self, fh: BinaryIO): - self.fh = fh - self.header = c_sqlite3.wal_header(fh) - - if self.header.magic not in WAL_HEADER_MAGIC: - raise InvalidDatabase("Invalid header magic") - - self.checksum_endian = "<" if self.header.magic == WAL_HEADER_MAGIC_LE else ">" - self._checkpoints = None - - self.frame = lru_cache(1024)(self.frame) - - def frame(self, frame_idx: int) -> WALFrame: - frame_size = len(c_sqlite3.wal_frame) + self.header.page_size - offset = len(c_sqlite3.wal_header) + frame_idx * frame_size - return WALFrame(self, offset) - - def frames(self) -> Iterator[WALFrame]: - frame_idx = 0 - while True: - try: - yield self.frame(frame_idx) - frame_idx += 1 - except EOFError: # noqa: PERF203 - break - - def checkpoints(self) -> list[WALCheckpoint]: - if not self._checkpoints: - checkpoints = [] - frames = [] - - for frame in self.frames(): - frames.append(frame) - - if frame.page_count != 0: - checkpoints.append(WALCheckpoint(self, frames)) - frames = [] - - self._checkpoints = checkpoints - - return self._checkpoints - - -class WALFrame: - def __init__(self, wal: WAL, offset: int): - self.wal = wal - self.offset = offset - - self.fh = wal.fh - self._data = None - - self.fh.seek(offset) - self.header = c_sqlite3.wal_frame(self.fh) - - def __repr__(self) -> str: - return f"" - - @property - def valid(self) -> bool: - salt1_match = self.header.salt1 == self.wal.header.salt1 - salt2_match = self.header.salt2 == self.wal.header.salt2 - - return salt1_match and salt2_match - - @property - def data(self) -> bytes: - if not self._data: - self.fh.seek(self.offset + len(c_sqlite3.wal_frame)) - self._data = self.fh.read(self.wal.header.page_size) - return self._data - - @property - def page_number(self) -> int: - return self.header.page_number - - @property - def page_count(self) -> int: - return self.header.page_count - - -class WALCheckpoint: - def __init__(self, wal: WAL, frames: list[WALFrame]): - self.wal = wal - self.frames = frames - self._page_map = None - - def __contains__(self, page: int) -> bool: - return page in self.page_map - - def __getitem__(self, page: int) -> WALFrame: - return self.page_map[page] - - def __repr__(self) -> str: - return f"" - - @property - def page_map(self) -> dict[int, WALFrame]: - if not self._page_map: - self._page_map = {frame.page_number: frame for frame in self.frames} - - return self._page_map - - def get(self, page: int, default: Any = None) -> WALFrame: - return self.page_map.get(page, default) - - -def wal_checksum(buf: bytes, endian: str = ">") -> tuple[int, int]: - """For future use, will be used when WAL is fully implemented""" - - s0 = s1 = 0 - num_ints = len(buf) // 4 - arr = struct.unpack(f"{endian}{num_ints}I", buf) - - for int_num in range(0, num_ints, 2): - s0 = (s0 + (arr[int_num] + s1)) & 0xFFFFFFFF - s1 = (s1 + (arr[int_num + 1] + s0)) & 0xFFFFFFFF - - return s0, s1 - - def walk_tree(sqlite: SQLite3, page: Page) -> Iterator[Cell]: if page.header.flags in ( c_sqlite3.PAGE_TYPE_LEAF_TABLE, diff --git a/dissect/database/sqlite3/wal.py b/dissect/database/sqlite3/wal.py new file mode 100644 index 0000000..309f156 --- /dev/null +++ b/dissect/database/sqlite3/wal.py @@ -0,0 +1,196 @@ +from __future__ import annotations + +import logging +import os +import struct +from functools import cached_property, lru_cache +from pathlib import Path +from typing import TYPE_CHECKING, Any, BinaryIO + +from dissect.database.sqlite3.c_sqlite3 import c_sqlite3 +from dissect.database.sqlite3.exception import InvalidDatabase + +if TYPE_CHECKING: + from collections.abc import Iterator + +log = logging.getLogger(__name__) +log.setLevel(os.getenv("DISSECT_LOG_SQLITE3", "CRITICAL")) + +# See https://sqlite.org/fileformat2.html#wal_file_format +WAL_HEADER_MAGIC_LE = 0x377F0682 +WAL_HEADER_MAGIC_BE = 0x377F0683 +WAL_HEADER_MAGIC = {WAL_HEADER_MAGIC_LE, WAL_HEADER_MAGIC_BE} + + +class WAL: + def __init__(self, fh: WAL | Path | BinaryIO): + # Use the provided WAL file handle or try to open a sidecar WAL file. + if hasattr(fh, "read"): + name = getattr(fh, "name", None) + path = Path(name) if name else None + else: + if not isinstance(fh, Path): + fh = Path(fh) + path = fh + fh = path.open("rb") + + self.fh = fh + self.path = path + self.header = c_sqlite3.wal_header(fh) + + if self.header.magic not in WAL_HEADER_MAGIC: + raise InvalidDatabase("Invalid WAL header magic") + + self.checksum_endian = "<" if self.header.magic == WAL_HEADER_MAGIC_LE else ">" + + self.frame = lru_cache(1024)(self.frame) + + def frame(self, frame_idx: int) -> Frame: + frame_size = len(c_sqlite3.wal_frame) + self.header.page_size + offset = len(c_sqlite3.wal_header) + frame_idx * frame_size + return Frame(self, offset) + + def frames(self) -> Iterator[Frame]: + frame_idx = 0 + while True: + try: + yield self.frame(frame_idx) + frame_idx += 1 + except EOFError: # noqa: PERF203 + break + + @cached_property + def commits(self) -> list[Commit]: + """Return all commits in the WAL file. + + Commits are frames where ``header.page_count`` specifies the size of the + database file in pages after the commit. For all other frames it is 0. + + References: + - https://sqlite.org/fileformat2.html#wal_file_format + """ + commits = [] + frames = [] + + for frame in self.frames(): + frames.append(frame) + + # A commit record has a page_count header greater than zero + if frame.page_count > 0: + commits.append(Commit(self, frames)) + frames = [] + + if frames: + # TODO: Do we want to track these somewhere? + log.warning("Found leftover %d frames after the last WAL commit", len(frames)) + + return commits + + @cached_property + def checkpoints(self) -> list[Checkpoint]: + """Return deduplicated checkpoints, oldest first. + + Deduplicate commits by the ``salt1`` value of their first frame. Later + commits overwrite earlier ones so the returned list contains the most + recent commit for each ``salt1``, sorted ascending. + + References: + - https://sqlite.org/fileformat2.html#wal_file_format + - https://sqlite.org/wal.html#checkpointing + """ + checkpoints_map: dict[int, Checkpoint] = {} + for commit in self.commits: + if not commit.frames: + continue + salt1 = commit.frames[0].header.salt1 + # Keep the most recent commit for each salt1 (later commits overwrite). + checkpoints_map[salt1] = commit + + return [checkpoints_map[salt] for salt in sorted(checkpoints_map.keys())] + + +class Frame: + def __init__(self, wal: WAL, offset: int): + self.wal = wal + self.offset = offset + + self.fh = wal.fh + + self.fh.seek(offset) + self.header = c_sqlite3.wal_frame(self.fh) + + def __repr__(self) -> str: + return f"" + + @property + def valid(self) -> bool: + salt1_match = self.header.salt1 == self.wal.header.salt1 + salt2_match = self.header.salt2 == self.wal.header.salt2 + + return salt1_match and salt2_match + + @property + def data(self) -> bytes: + self.fh.seek(self.offset + len(c_sqlite3.wal_frame)) + return self.fh.read(self.wal.header.page_size) + + @property + def page_number(self) -> int: + return self.header.page_number + + @property + def page_count(self) -> int: + return self.header.page_count + + +class _FrameCollection: + """Convenience class to keep track of a collection of frames that were committed together.""" + + def __init__(self, wal: WAL, frames: list[Frame]): + self.wal = wal + self.frames = frames + + def __contains__(self, page: int) -> bool: + return page in self.page_map + + def __getitem__(self, page: int) -> Frame: + return self.page_map[page] + + def __repr__(self) -> str: + return f"<{self.__class__.__name__} frames={len(self.frames)}>" + + @cached_property + def page_map(self) -> dict[int, Frame]: + return {frame.page_number: frame for frame in self.frames} + + def get(self, page: int, default: Any = None) -> Frame: + return self.page_map.get(page, default) + + +class Checkpoint(_FrameCollection): + """A checkpoint is an operation that transfers all committed transactions from + the WAL file back into the main database file. + + References: + - https://sqlite.org/fileformat2.html#wal_file_format + """ + + +class Commit(_FrameCollection): + """A commit is a collection of frames that were committed together. + + References: + - https://sqlite.org/fileformat2.html#wal_file_format + """ + + +def checksum(buf: bytes, endian: str = ">") -> tuple[int, int]: + s0 = s1 = 0 + num_ints = len(buf) // 4 + arr = struct.unpack(f"{endian}{num_ints}I", buf) + + for int_num in range(0, num_ints, 2): + s0 = (s0 + (arr[int_num] + s1)) & 0xFFFFFFFF + s1 = (s1 + (arr[int_num + 1] + s0)) & 0xFFFFFFFF + + return s0, s1 diff --git a/tests/_data/sqlite3/test.sqlite b/tests/_data/sqlite3/test.sqlite index 9a30530..4b0d00a 100644 --- a/tests/_data/sqlite3/test.sqlite +++ b/tests/_data/sqlite3/test.sqlite @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b8a2b588f035eca3a644799c77902e9058e1e577392b2616ab08e9cd2c73c9fb -size 16384 +oid sha256:10c92d94ad110823f169c6221941309576dccacd459f8b627a1eb54f5a3f813c +size 20480 diff --git a/tests/_data/sqlite3/test.sqlite-wal b/tests/_data/sqlite3/test.sqlite-wal new file mode 100644 index 0000000..ae4b789 --- /dev/null +++ b/tests/_data/sqlite3/test.sqlite-wal @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:263c7b76fff7ece44363f5c40fd50718586583385a305b0b3bcca121d32f98de +size 70072 diff --git a/tests/_tools/sqlite3/__init__.py b/tests/_tools/sqlite3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/_tools/sqlite3/generate_sqlite.py b/tests/_tools/sqlite3/generate_sqlite.py new file mode 100644 index 0000000..7019a00 --- /dev/null +++ b/tests/_tools/sqlite3/generate_sqlite.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +conn = sqlite3.connect("db.sqlite", isolation_level=None) + +# Set WAL mode +conn.execute("PRAGMA journal_mode=WAL;") + +# Disable automatic checkpoints to keep all data in WAL for testing +conn.execute("PRAGMA wal_autocheckpoint=-1;") + + +def create_table() -> None: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS test ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + value INTEGER NOT NULL + ) + """ + ) + + +def insert_data(name: str, value: str | int) -> None: + conn.execute("INSERT INTO test (name, value) VALUES (?, ?)", (name, value)) + + +def delete_data(name: str, value: str | int) -> None: + conn.execute("DELETE FROM test WHERE name = ? AND value = ?", (name, value)) + + +def update_data(old_name: str, old_value: str | int, new_name: str, new_value: str | int) -> None: + conn.execute( + "UPDATE test SET name = ?, value = ? WHERE name = ? AND value = ?", + (new_name, new_value, old_name, old_value), + ) + + +def create_checkpoint() -> None: + conn.execute("PRAGMA wal_checkpoint(FULL);") + + +def move_files() -> None: + destination_dir = (Path(__file__).parent / "../../_data/sqlite3/").resolve() + + Path("db.sqlite").rename(destination_dir / "test.sqlite") + Path("db.sqlite-wal").rename(destination_dir / "test.sqlite-wal") + Path("db.sqlite-shm").rename(destination_dir / "test.sqlite-shm") + + # Remove this line if the shm file is needed as well + Path(destination_dir / "test.sqlite-shm").unlink() + + +if __name__ == "__main__": + create_table() + + # Initial data + insert_data("testing", 1337) + insert_data("omg", 7331) + insert_data("A" * 4100, 4100) + insert_data("B" * 4100, 4100) + insert_data("negative", -11644473429) + + create_checkpoint() + + # Insert extra data after the first checkpoint + insert_data("after checkpoint", 42) + insert_data("after checkpoint", 43) + insert_data("after checkpoint", 44) + insert_data("after checkpoint", 45) + + create_checkpoint() + + # More data after second checkpoint, fewer entries to ensure both checkpoints will be in WAL + insert_data("second checkpoint", 100) + insert_data("second checkpoint", 101) + + create_checkpoint() + + # Modify some data after third checkpoint + delete_data("after checkpoint", 43) + update_data("after checkpoint", 45, "wow", 1234) + + # Rename files to prevent automatic cleanup + move_files() + + conn.close() diff --git a/tests/sqlite3/conftest.py b/tests/sqlite3/conftest.py index a312084..9f86947 100644 --- a/tests/sqlite3/conftest.py +++ b/tests/sqlite3/conftest.py @@ -1,20 +1,25 @@ from __future__ import annotations -from typing import TYPE_CHECKING, BinaryIO +from typing import TYPE_CHECKING import pytest -from tests._util import open_file +from tests._util import absolute_path if TYPE_CHECKING: - from collections.abc import Iterator + from pathlib import Path @pytest.fixture -def sqlite_db() -> Iterator[BinaryIO]: - yield from open_file("_data/sqlite3/test.sqlite") +def sqlite_db() -> Path: + return absolute_path("_data/sqlite3/test.sqlite") @pytest.fixture -def empty_db() -> Iterator[BinaryIO]: - yield from open_file("_data/sqlite3/empty.sqlite") +def sqlite_wal() -> Path: + return absolute_path("_data/sqlite3/test.sqlite-wal") + + +@pytest.fixture +def empty_db() -> Path: + return absolute_path("_data/sqlite3/empty.sqlite") diff --git a/tests/sqlite3/test_sqlite3.py b/tests/sqlite3/test_sqlite3.py index 37e2db0..a0ce473 100644 --- a/tests/sqlite3/test_sqlite3.py +++ b/tests/sqlite3/test_sqlite3.py @@ -1,30 +1,37 @@ from __future__ import annotations from io import BytesIO -from typing import Any, BinaryIO +from typing import TYPE_CHECKING, Any, BinaryIO import pytest from dissect.database.sqlite3 import sqlite3 +if TYPE_CHECKING: + from pathlib import Path -def test_sqlite(sqlite_db: BinaryIO) -> None: - s = sqlite3.SQLite3(sqlite_db) - assert s.header.magic == sqlite3.SQLITE3_HEADER_MAGIC +@pytest.mark.parametrize( + ("db_as_path"), + [pytest.param(True, id="db_as_path"), pytest.param(False, id="db_as_fh")], +) +def test_sqlite(sqlite_db: Path, db_as_path: bool) -> None: + db = sqlite3.SQLite3(sqlite_db) if db_as_path else sqlite3.SQLite3(sqlite_db.open("rb")) + + assert db.header.magic == sqlite3.SQLITE3_HEADER_MAGIC - tables = list(s.tables()) - assert len(tables) == 1 + tables = list(db.tables()) + assert len(tables) == 2 table = tables[0] assert table.name == "test" assert table.page == 2 assert [column.name for column in table.columns] == ["id", "name", "value"] assert table.primary_key == "id" - assert s.table("test").__dict__ == table.__dict__ + assert db.table("test").__dict__ == table.__dict__ rows = list(table.rows()) - assert len(rows) == 5 + assert len(rows) == 10 assert rows[0].id == 1 assert rows[0].name == "testing" assert rows[0].value == 1337 @@ -40,6 +47,21 @@ def test_sqlite(sqlite_db: BinaryIO) -> None: assert rows[4].id == 5 assert rows[4].name == "negative" assert rows[4].value == -11644473429 + assert rows[5].id == 6 + assert rows[5].name == "after checkpoint" + assert rows[5].value == 42 + assert rows[6].id == 8 + assert rows[6].name == "after checkpoint" + assert rows[6].value == 44 + assert rows[7].id == 9 + assert rows[7].name == "wow" + assert rows[7].value == 1234 + assert rows[8].id == 10 + assert rows[8].name == "second checkpoint" + assert rows[8].value == 100 + assert rows[9].id == 11 + assert rows[9].name == "second checkpoint" + assert rows[9].value == 101 assert len(rows) == len(list(table)) assert table.row(0).__dict__ == rows[0].__dict__ diff --git a/tests/sqlite3/test_wal.py b/tests/sqlite3/test_wal.py new file mode 100644 index 0000000..cc01925 --- /dev/null +++ b/tests/sqlite3/test_wal.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import pytest + +from dissect.database.sqlite3 import sqlite3 + +if TYPE_CHECKING: + from pathlib import Path + + +@pytest.mark.parametrize( + ("db_as_path"), + [pytest.param(True, id="db_as_path"), pytest.param(False, id="db_as_fh")], +) +@pytest.mark.parametrize( + ("wal_as_path"), + [pytest.param(True, id="wal_as_path"), pytest.param(False, id="wal_as_fh")], +) +def test_sqlite_wal(sqlite_db: Path, sqlite_wal: Path, db_as_path: bool, wal_as_path: bool) -> None: + db = sqlite3.SQLite3( + sqlite_db if db_as_path else sqlite_db.open("rb"), + sqlite_wal if wal_as_path else sqlite_wal.open("rb"), + checkpoint=1, + ) + _assert_checkpoint_1(db) + + db = sqlite3.SQLite3( + sqlite_db if db_as_path else sqlite_db.open("rb"), + sqlite_wal if wal_as_path else sqlite_wal.open("rb"), + checkpoint=2, + ) + _assert_checkpoint_2(db) + + db = sqlite3.SQLite3( + sqlite_db if db_as_path else sqlite_db.open("rb"), + sqlite_wal if wal_as_path else sqlite_wal.open("rb"), + checkpoint=3, + ) + _assert_checkpoint_3(db) + + +def _assert_checkpoint_1(s: sqlite3.SQLite3) -> None: + # After the first checkpoint the "after checkpoint" entries are present + table = next(iter(s.tables())) + + rows = list(table.rows()) + assert len(rows) == 9 + + assert rows[0].id == 1 + assert rows[0].name == "testing" + assert rows[0].value == 1337 + assert rows[1].id == 2 + assert rows[1].name == "omg" + assert rows[1].value == 7331 + assert rows[2].id == 3 + assert rows[2].name == "A" * 4100 + assert rows[2].value == 4100 + assert rows[3].id == 4 + assert rows[3].name == "B" * 4100 + assert rows[3].value == 4100 + assert rows[4].id == 5 + assert rows[4].name == "negative" + assert rows[4].value == -11644473429 + assert rows[5].id == 6 + assert rows[5].name == "after checkpoint" + assert rows[5].value == 42 + assert rows[6].id == 7 + assert rows[6].name == "after checkpoint" + assert rows[6].value == 43 + assert rows[7].id == 8 + assert rows[7].name == "after checkpoint" + assert rows[7].value == 44 + assert rows[8].id == 9 + assert rows[8].name == "after checkpoint" + assert rows[8].value == 45 + + +def _assert_checkpoint_2(s: sqlite3.SQLite3) -> None: + # After the second checkpoint two more entries are present ("second checkpoint") + table = next(iter(s.tables())) + + rows = list(table.rows()) + assert len(rows) == 11 + + assert rows[0].id == 1 + assert rows[0].name == "testing" + assert rows[0].value == 1337 + assert rows[1].id == 2 + assert rows[1].name == "omg" + assert rows[1].value == 7331 + assert rows[2].id == 3 + assert rows[2].name == "A" * 4100 + assert rows[2].value == 4100 + assert rows[3].id == 4 + assert rows[3].name == "B" * 4100 + assert rows[3].value == 4100 + assert rows[4].id == 5 + assert rows[4].name == "negative" + assert rows[4].value == -11644473429 + assert rows[5].id == 6 + assert rows[5].name == "after checkpoint" + assert rows[5].value == 42 + assert rows[6].id == 7 + assert rows[6].name == "after checkpoint" + assert rows[6].value == 43 + assert rows[7].id == 8 + assert rows[7].name == "after checkpoint" + assert rows[7].value == 44 + assert rows[8].id == 9 + assert rows[8].name == "after checkpoint" + assert rows[8].value == 45 + assert rows[9].id == 10 + assert rows[9].name == "second checkpoint" + assert rows[9].value == 100 + assert rows[10].id == 11 + assert rows[10].name == "second checkpoint" + assert rows[10].value == 101 + + +def _assert_checkpoint_3(s: sqlite3.SQLite3) -> None: + # After the third checkpoint the deletion and update of one "after checkpoint" are reflected + table = next(iter(s.tables())) + rows = list(table.rows()) + + assert len(rows) == 10 + + assert rows[0].id == 1 + assert rows[0].name == "testing" + assert rows[0].value == 1337 + assert rows[1].id == 2 + assert rows[1].name == "omg" + assert rows[1].value == 7331 + assert rows[2].id == 3 + assert rows[2].name == "A" * 4100 + assert rows[2].value == 4100 + assert rows[3].id == 4 + assert rows[3].name == "B" * 4100 + assert rows[3].value == 4100 + assert rows[4].id == 5 + assert rows[4].name == "negative" + assert rows[4].value == -11644473429 + assert rows[5].id == 6 + assert rows[5].name == "after checkpoint" + assert rows[5].value == 42 + assert rows[6].id == 8 + assert rows[6].name == "after checkpoint" + assert rows[6].value == 44 + assert rows[7].id == 9 + assert rows[7].name == "wow" + assert rows[7].value == 1234 + assert rows[8].id == 10 + assert rows[8].name == "second checkpoint" + assert rows[8].value == 100 + assert rows[9].id == 11 + assert rows[9].name == "second checkpoint" + assert rows[9].value == 101