diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..9b4f63f --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +tests/_data/** filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore index 74cecaf..2e944b3 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,6 @@ dist/ *.pyc __pycache__/ .pytest_cache/ -tests/docs/api -tests/docs/build +tests/_docs/api +tests/_docs/build .tox/ diff --git a/dissect/ntfs/attr.py b/dissect/ntfs/attr.py index 8bb69a2..6c2a09a 100644 --- a/dissect/ntfs/attr.py +++ b/dissect/ntfs/attr.py @@ -6,15 +6,9 @@ from dissect.util.stream import RangeStream, RunlistStream from dissect.util.ts import wintimestamp -from dissect.ntfs.c_ntfs import ( - ATTRIBUTE_TYPE_CODE, - IO_REPARSE_TAG, - c_ntfs, - segment_reference, - varint, -) +from dissect.ntfs.c_ntfs import ATTRIBUTE_TYPE_CODE, IO_REPARSE_TAG, c_ntfs from dissect.ntfs.exceptions import MftNotAvailableError, VolumeNotAvailableError -from dissect.ntfs.util import ensure_volume, get_full_path, ts_to_ns +from dissect.ntfs.util import ensure_volume, get_full_path, segment_reference, ts_to_ns, varint if TYPE_CHECKING: from collections.abc import Iterator diff --git a/dissect/ntfs/c_ntfs.py b/dissect/ntfs/c_ntfs.py index 40924d4..cadee16 100644 --- a/dissect/ntfs/c_ntfs.py +++ b/dissect/ntfs/c_ntfs.py @@ -1,7 +1,5 @@ from __future__ import annotations -import struct - from dissect.cstruct import cstruct ntfs_def = """ @@ -276,6 +274,14 @@ USHORT PrintNameLength; } _MOUNT_POINT_REPARSE_BUFFER; +typedef struct _CLOUD_FILTER_REPARSE_BUFFER { + // ULONG Unknown_1; + // ULONG Unknown_2; + CHAR Guid[16]; + USHORT NameLength; + // WCHAR Name[NameLength]; +} _CLOUD_FILTER_REPARSE_BUFFER; + /* ================ Index ================ */ enum COLLATION : ULONG { @@ -613,45 +619,3 @@ INDEX_NODE = 0x01 INDEX_ENTRY_NODE = 0x01 INDEX_ENTRY_END = 0x02 - - -def segment_reference(reference: c_ntfs._MFT_SEGMENT_REFERENCE) -> int: - """Helper to calculate the complete segment number from a cstruct MFT segment reference. - - Args: - reference: A cstruct _MFT_SEGMENT_REFERENCE instance to return the complete segment number of. - """ - return reference.SegmentNumberLowPart | (reference.SegmentNumberHighPart << 32) - - -def varint(buf: bytes) -> int: - """Parse variable integers. - - Dataruns in NTFS are stored as a tuple of variable sized integers. The size of each integer is - stored in the first byte, 4 bits for each integer. This logic can be seen in - :func:`AttributeHeader.dataruns `. - - This function only parses those variable amount of bytes into actual integers. To do that, we - simply pad the bytes to 8 bytes long and parse it as a signed 64 bit integer. We pad with 0xff - if the number is negative and 0x00 otherwise. - - Args: - buf: The byte buffer to parse a varint from. - """ - if len(buf) < 8: - buf += (b"\xff" if buf[-1] & 0x80 else b"\x00") * (8 - len(buf)) - - return struct.unpack(" int: - """Count the number of trailing zero bits in an integer of a given size. - - Args: - value: The integer to count trailing zero bits in. - size: Integer size to limit to. - """ - for i in range(size): - if value & (1 << i): - return i - return 0 diff --git a/dissect/ntfs/c_ntfs.pyi b/dissect/ntfs/c_ntfs.pyi new file mode 100644 index 0000000..3a6a5cf --- /dev/null +++ b/dissect/ntfs/c_ntfs.pyi @@ -0,0 +1,926 @@ +# Generated by cstruct-stubgen +from typing import BinaryIO, Literal, overload + +import dissect.cstruct as __cs__ +from typing_extensions import TypeAlias + +class _c_ntfs(__cs__.cstruct): + class FILE_ATTRIBUTE(__cs__.Flag): + READONLY = ... + HIDDEN = ... + SYSTEM = ... + DIRECTORY = ... + ARCHIVE = ... + DEVICE = ... + NORMAL = ... + TEMPORARY = ... + SPARSE_FILE = ... + REPARSE_POINT = ... + COMPRESSED = ... + OFFLINE = ... + NOT_CONTENT_INDEXED = ... + ENCRYPTED = ... + INTEGRITY_STREAM = ... + VIRTUAL = ... + NO_SCRUB_DATA = ... + RECALL_ON_OPEN = ... + PINNED = ... + UNPINNED = ... + RECALL_ON_DATA_ACCESS = ... + + class _BIOS_PARAMETER_BLOCK(__cs__.Structure): + BytesPerSector: _c_ntfs.uint16 + SectorsPerCluster: _c_ntfs.int8 + ReservedSectors: _c_ntfs.uint16 + Fats: _c_ntfs.uint8 + RootEntries: _c_ntfs.uint16 + Sectors: _c_ntfs.uint16 + Media: _c_ntfs.uint8 + SectorsPerFat: _c_ntfs.uint16 + SectorsPerTrack: _c_ntfs.uint16 + Heads: _c_ntfs.uint16 + HiddenSectors: _c_ntfs.uint32 + LargeSectors: _c_ntfs.uint32 + @overload + def __init__( + self, + BytesPerSector: _c_ntfs.uint16 | None = ..., + SectorsPerCluster: _c_ntfs.int8 | None = ..., + ReservedSectors: _c_ntfs.uint16 | None = ..., + Fats: _c_ntfs.uint8 | None = ..., + RootEntries: _c_ntfs.uint16 | None = ..., + Sectors: _c_ntfs.uint16 | None = ..., + Media: _c_ntfs.uint8 | None = ..., + SectorsPerFat: _c_ntfs.uint16 | None = ..., + SectorsPerTrack: _c_ntfs.uint16 | None = ..., + Heads: _c_ntfs.uint16 | None = ..., + HiddenSectors: _c_ntfs.uint32 | None = ..., + LargeSectors: _c_ntfs.uint32 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + BIOS_PARAMETER_BLOCK: TypeAlias = _BIOS_PARAMETER_BLOCK + class _BOOT_SECTOR(__cs__.Structure): + Jump: __cs__.CharArray + Oem: __cs__.CharArray + Bpb: _c_ntfs._BIOS_PARAMETER_BLOCK + Unused0: __cs__.CharArray + NumberSectors: _c_ntfs.uint64 + MftStartLcn: _c_ntfs.uint64 + Mft2StartLcn: _c_ntfs.uint64 + ClustersPerFileRecordSegment: _c_ntfs.int8 + Reserved0: __cs__.CharArray + ClustersPerIndexBuffer: _c_ntfs.int8 + Reserved1: __cs__.CharArray + SerialNumber: _c_ntfs.uint64 + Checksum: _c_ntfs.uint32 + BootStrap: __cs__.CharArray + @overload + def __init__( + self, + Jump: __cs__.CharArray | None = ..., + Oem: __cs__.CharArray | None = ..., + Bpb: _c_ntfs._BIOS_PARAMETER_BLOCK | None = ..., + Unused0: __cs__.CharArray | None = ..., + NumberSectors: _c_ntfs.uint64 | None = ..., + MftStartLcn: _c_ntfs.uint64 | None = ..., + Mft2StartLcn: _c_ntfs.uint64 | None = ..., + ClustersPerFileRecordSegment: _c_ntfs.int8 | None = ..., + Reserved0: __cs__.CharArray | None = ..., + ClustersPerIndexBuffer: _c_ntfs.int8 | None = ..., + Reserved1: __cs__.CharArray | None = ..., + SerialNumber: _c_ntfs.uint64 | None = ..., + Checksum: _c_ntfs.uint32 | None = ..., + BootStrap: __cs__.CharArray | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + BOOT_SECTOR: TypeAlias = _BOOT_SECTOR + class _MFT_SEGMENT_REFERENCE(__cs__.Structure): + SegmentNumberLowPart: _c_ntfs.uint32 + SegmentNumberHighPart: _c_ntfs.uint16 + SequenceNumber: _c_ntfs.uint16 + @overload + def __init__( + self, + SegmentNumberLowPart: _c_ntfs.uint32 | None = ..., + SegmentNumberHighPart: _c_ntfs.uint16 | None = ..., + SequenceNumber: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + MFT_SEGMENT_REFERENCE: TypeAlias = _MFT_SEGMENT_REFERENCE + FILE_REFERENCE: TypeAlias = _MFT_SEGMENT_REFERENCE + class _MULTI_SECTOR_HEADER(__cs__.Structure): + Signature: __cs__.CharArray + UpdateSequenceArrayOffset: _c_ntfs.uint16 + UpdateSequenceArraySize: _c_ntfs.uint16 + @overload + def __init__( + self, + Signature: __cs__.CharArray | None = ..., + UpdateSequenceArrayOffset: _c_ntfs.uint16 | None = ..., + UpdateSequenceArraySize: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + MULTI_SECTOR_HEADER: TypeAlias = _MULTI_SECTOR_HEADER + class _FILE_RECORD_SEGMENT_HEADER(__cs__.Structure): + MultiSectorHeader: _c_ntfs._MULTI_SECTOR_HEADER + Lsn: _c_ntfs.uint64 + SequenceNumber: _c_ntfs.uint16 + ReferenceCount: _c_ntfs.uint16 + FirstAttributeOffset: _c_ntfs.uint16 + Flags: _c_ntfs.uint16 + BytesInUse: _c_ntfs.uint32 + BytesAllocated: _c_ntfs.uint32 + BaseFileRecordSegment: _c_ntfs._MFT_SEGMENT_REFERENCE + NextAttributeInstance: _c_ntfs.uint16 + @overload + def __init__( + self, + MultiSectorHeader: _c_ntfs._MULTI_SECTOR_HEADER | None = ..., + Lsn: _c_ntfs.uint64 | None = ..., + SequenceNumber: _c_ntfs.uint16 | None = ..., + ReferenceCount: _c_ntfs.uint16 | None = ..., + FirstAttributeOffset: _c_ntfs.uint16 | None = ..., + Flags: _c_ntfs.uint16 | None = ..., + BytesInUse: _c_ntfs.uint32 | None = ..., + BytesAllocated: _c_ntfs.uint32 | None = ..., + BaseFileRecordSegment: _c_ntfs._MFT_SEGMENT_REFERENCE | None = ..., + NextAttributeInstance: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + FILE_RECORD_SEGMENT_HEADER: TypeAlias = _FILE_RECORD_SEGMENT_HEADER + class ATTRIBUTE_TYPE_CODE(__cs__.Enum): + UNUSED = ... + STANDARD_INFORMATION = ... + ATTRIBUTE_LIST = ... + FILE_NAME = ... + OBJECT_ID = ... + SECURITY_DESCRIPTOR = ... + VOLUME_NAME = ... + VOLUME_INFORMATION = ... + DATA = ... + INDEX_ROOT = ... + INDEX_ALLOCATION = ... + BITMAP = ... + REPARSE_POINT = ... + EA_INFORMATION = ... + EA = ... + PROPERTY_SET = ... + LOGGED_UTILITY_STREAM = ... + END = ... + + class _ATTRIBUTE_RECORD_HEADER(__cs__.Structure): + TypeCode: _c_ntfs.ATTRIBUTE_TYPE_CODE + RecordLength: _c_ntfs.uint32 + FormCode: _c_ntfs.uint8 + NameLength: _c_ntfs.uint8 + NameOffset: _c_ntfs.uint16 + Flags: _c_ntfs.uint16 + Instance: _c_ntfs.uint16 + class __anonymous_2__(__cs__.Union): + class __anonymous_0__(__cs__.Structure): + ValueLength: _c_ntfs.uint32 + ValueOffset: _c_ntfs.uint16 + Flags: _c_ntfs.uint8 + Reserved: _c_ntfs.uint8 + @overload + def __init__( + self, + ValueLength: _c_ntfs.uint32 | None = ..., + ValueOffset: _c_ntfs.uint16 | None = ..., + Flags: _c_ntfs.uint8 | None = ..., + Reserved: _c_ntfs.uint8 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + Resident: __anonymous_0__ + class __anonymous_1__(__cs__.Structure): + LowestVcn: _c_ntfs.uint64 + HighestVcn: _c_ntfs.uint64 + MappingPairsOffset: _c_ntfs.uint16 + CompressionUnit: _c_ntfs.uint8 + Reserved: __cs__.Array[_c_ntfs.uint8] + AllocatedLength: _c_ntfs.int64 + FileSize: _c_ntfs.int64 + ValidDataLength: _c_ntfs.int64 + TotalAllocated: _c_ntfs.int64 + @overload + def __init__( + self, + LowestVcn: _c_ntfs.uint64 | None = ..., + HighestVcn: _c_ntfs.uint64 | None = ..., + MappingPairsOffset: _c_ntfs.uint16 | None = ..., + CompressionUnit: _c_ntfs.uint8 | None = ..., + Reserved: __cs__.Array[_c_ntfs.uint8] | None = ..., + AllocatedLength: _c_ntfs.int64 | None = ..., + FileSize: _c_ntfs.int64 | None = ..., + ValidDataLength: _c_ntfs.int64 | None = ..., + TotalAllocated: _c_ntfs.int64 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + Nonresident: __anonymous_1__ + @overload + def __init__(self, Resident: __anonymous_0__ | None = ..., Nonresident: __anonymous_1__ | None = ...): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + Form: __anonymous_2__ + @overload + def __init__( + self, + TypeCode: _c_ntfs.ATTRIBUTE_TYPE_CODE | None = ..., + RecordLength: _c_ntfs.uint32 | None = ..., + FormCode: _c_ntfs.uint8 | None = ..., + NameLength: _c_ntfs.uint8 | None = ..., + NameOffset: _c_ntfs.uint16 | None = ..., + Flags: _c_ntfs.uint16 | None = ..., + Instance: _c_ntfs.uint16 | None = ..., + Form: __anonymous_2__ | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + ATTRIBUTE_RECORD_HEADER: TypeAlias = _ATTRIBUTE_RECORD_HEADER + class _STANDARD_INFORMATION(__cs__.Structure): + CreationTime: _c_ntfs.int64 + LastModificationTime: _c_ntfs.int64 + LastChangeTime: _c_ntfs.int64 + LastAccessTime: _c_ntfs.int64 + FileAttributes: _c_ntfs.uint32 + MaximumVersions: _c_ntfs.uint32 + VersionNumber: _c_ntfs.uint32 + ClassId: _c_ntfs.uint32 + OwnerId: _c_ntfs.uint32 + SecurityId: _c_ntfs.uint32 + QuotaCharged: _c_ntfs.uint64 + Usn: _c_ntfs.uint64 + @overload + def __init__( + self, + CreationTime: _c_ntfs.int64 | None = ..., + LastModificationTime: _c_ntfs.int64 | None = ..., + LastChangeTime: _c_ntfs.int64 | None = ..., + LastAccessTime: _c_ntfs.int64 | None = ..., + FileAttributes: _c_ntfs.uint32 | None = ..., + MaximumVersions: _c_ntfs.uint32 | None = ..., + VersionNumber: _c_ntfs.uint32 | None = ..., + ClassId: _c_ntfs.uint32 | None = ..., + OwnerId: _c_ntfs.uint32 | None = ..., + SecurityId: _c_ntfs.uint32 | None = ..., + QuotaCharged: _c_ntfs.uint64 | None = ..., + Usn: _c_ntfs.uint64 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + STANDARD_INFORMATION_EX: TypeAlias = _STANDARD_INFORMATION + class _ATTRIBUTE_LIST_ENTRY(__cs__.Structure): + AttributeTypeCode: _c_ntfs.ATTRIBUTE_TYPE_CODE + RecordLength: _c_ntfs.uint16 + AttributeNameLength: _c_ntfs.uint8 + AttributeNameOffset: _c_ntfs.uint8 + LowestVcn: _c_ntfs.uint64 + SegmentReference: _c_ntfs._MFT_SEGMENT_REFERENCE + Reserved: _c_ntfs.uint16 + AttributeName: __cs__.WcharArray + @overload + def __init__( + self, + AttributeTypeCode: _c_ntfs.ATTRIBUTE_TYPE_CODE | None = ..., + RecordLength: _c_ntfs.uint16 | None = ..., + AttributeNameLength: _c_ntfs.uint8 | None = ..., + AttributeNameOffset: _c_ntfs.uint8 | None = ..., + LowestVcn: _c_ntfs.uint64 | None = ..., + SegmentReference: _c_ntfs._MFT_SEGMENT_REFERENCE | None = ..., + Reserved: _c_ntfs.uint16 | None = ..., + AttributeName: __cs__.WcharArray | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + ATTRIBUTE_LIST_ENTRY: TypeAlias = _ATTRIBUTE_LIST_ENTRY + class _FILE_NAME(__cs__.Structure): + ParentDirectory: _c_ntfs._MFT_SEGMENT_REFERENCE + CreationTime: _c_ntfs.int64 + LastModificationTime: _c_ntfs.int64 + LastChangeTime: _c_ntfs.int64 + LastAccessTime: _c_ntfs.int64 + AllocatedLength: _c_ntfs.int64 + FileSize: _c_ntfs.int64 + FileAttributes: _c_ntfs.uint32 + EaSize: _c_ntfs.uint16 + _: _c_ntfs.uint16 + ReparsePointTag: _c_ntfs.uint32 + FileNameLength: _c_ntfs.uint8 + Flags: _c_ntfs.uint8 + FileName: __cs__.WcharArray + @overload + def __init__( + self, + ParentDirectory: _c_ntfs._MFT_SEGMENT_REFERENCE | None = ..., + CreationTime: _c_ntfs.int64 | None = ..., + LastModificationTime: _c_ntfs.int64 | None = ..., + LastChangeTime: _c_ntfs.int64 | None = ..., + LastAccessTime: _c_ntfs.int64 | None = ..., + AllocatedLength: _c_ntfs.int64 | None = ..., + FileSize: _c_ntfs.int64 | None = ..., + FileAttributes: _c_ntfs.uint32 | None = ..., + EaSize: _c_ntfs.uint16 | None = ..., + _: _c_ntfs.uint16 | None = ..., + ReparsePointTag: _c_ntfs.uint32 | None = ..., + FileNameLength: _c_ntfs.uint8 | None = ..., + Flags: _c_ntfs.uint8 | None = ..., + FileName: __cs__.WcharArray | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + FILE_NAME: TypeAlias = _FILE_NAME + class IO_REPARSE_TAG(__cs__.Enum): + RESERVED_ZERO = ... + RESERVED_ONE = ... + RESERVED_TWO = ... + MOUNT_POINT = ... + HSM = ... + DRIVE_EXTENDER = ... + HSM2 = ... + SIS = ... + WIM = ... + CSV = ... + DFS = ... + FILTER_MANAGER = ... + SYMLINK = ... + IIS_CACHE = ... + DFSR = ... + DEDUP = ... + APPXSTRM = ... + NFS = ... + FILE_PLACEHOLDER = ... + DFM = ... + WOF = ... + WCI = ... + WCI_1 = ... + GLOBAL_REPARSE = ... + CLOUD = ... + CLOUD_1 = ... + CLOUD_2 = ... + CLOUD_3 = ... + CLOUD_4 = ... + CLOUD_5 = ... + CLOUD_6 = ... + CLOUD_7 = ... + CLOUD_8 = ... + CLOUD_9 = ... + CLOUD_A = ... + CLOUD_B = ... + CLOUD_C = ... + CLOUD_D = ... + CLOUD_E = ... + CLOUD_F = ... + APPEXECLINK = ... + PROJFS = ... + LX_SYMLINK = ... + STORAGE_SYNC = ... + WCI_TOMBSTONE = ... + UNHANDLED = ... + ONEDRIVE = ... + PROJFS_TOMBSTONE = ... + AF_UNIX = ... + LX_FIFO = ... + LX_CHR = ... + LX_BLK = ... + WCI_LINK = ... + WCI_LINK_1 = ... + + class _REPARSE_DATA_BUFFER(__cs__.Structure): + ReparseTag: _c_ntfs.IO_REPARSE_TAG + ReparseDataLength: _c_ntfs.uint16 + Reserved: _c_ntfs.uint16 + @overload + def __init__( + self, + ReparseTag: _c_ntfs.IO_REPARSE_TAG | None = ..., + ReparseDataLength: _c_ntfs.uint16 | None = ..., + Reserved: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + REPARSE_DATA_BUFFER: TypeAlias = _REPARSE_DATA_BUFFER + class SYMLINK_FLAG(__cs__.Enum): + ABSOLUTE = ... + RELATIVE = ... + + class _SYMBOLIC_LINK_REPARSE_BUFFER(__cs__.Structure): + SubstituteNameOffset: _c_ntfs.uint16 + SubstituteNameLength: _c_ntfs.uint16 + PrintNameOffset: _c_ntfs.uint16 + PrintNameLength: _c_ntfs.uint16 + Flags: _c_ntfs.SYMLINK_FLAG + @overload + def __init__( + self, + SubstituteNameOffset: _c_ntfs.uint16 | None = ..., + SubstituteNameLength: _c_ntfs.uint16 | None = ..., + PrintNameOffset: _c_ntfs.uint16 | None = ..., + PrintNameLength: _c_ntfs.uint16 | None = ..., + Flags: _c_ntfs.SYMLINK_FLAG | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + SYMBOLIC_LINK_REPARSE_BUFFER: TypeAlias = _SYMBOLIC_LINK_REPARSE_BUFFER + class _MOUNT_POINT_REPARSE_BUFFER(__cs__.Structure): + SubstituteNameOffset: _c_ntfs.uint16 + SubstituteNameLength: _c_ntfs.uint16 + PrintNameOffset: _c_ntfs.uint16 + PrintNameLength: _c_ntfs.uint16 + @overload + def __init__( + self, + SubstituteNameOffset: _c_ntfs.uint16 | None = ..., + SubstituteNameLength: _c_ntfs.uint16 | None = ..., + PrintNameOffset: _c_ntfs.uint16 | None = ..., + PrintNameLength: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + class COLLATION(__cs__.Enum): + BINARY = ... + FILE_NAME = ... + UNICODE_STRING = ... + NUMBER_RULES = ... + NTOFS_ULONG = ... + NTOFS_SID = ... + NTOFS_SECURITY_HASH = ... + NTOFS_ULONGS = ... + + COLLATION_RULE: TypeAlias = COLLATION + class _INDEX_HEADER(__cs__.Structure): + FirstEntryOffset: _c_ntfs.uint32 + TotalSizeOfEntries: _c_ntfs.uint32 + AllocatedSize: _c_ntfs.uint32 + Flags: _c_ntfs.uint8 + Reserved: __cs__.Array[_c_ntfs.uint8] + @overload + def __init__( + self, + FirstEntryOffset: _c_ntfs.uint32 | None = ..., + TotalSizeOfEntries: _c_ntfs.uint32 | None = ..., + AllocatedSize: _c_ntfs.uint32 | None = ..., + Flags: _c_ntfs.uint8 | None = ..., + Reserved: __cs__.Array[_c_ntfs.uint8] | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + INDEX_HEADER: TypeAlias = _INDEX_HEADER + class _INDEX_ROOT(__cs__.Structure): + AttributeType: _c_ntfs.ATTRIBUTE_TYPE_CODE + CollationRule: _c_ntfs.COLLATION + BytesPerIndexBuffer: _c_ntfs.uint32 + ClustersPerIndexBuffer: _c_ntfs.uint8 + Reserved: __cs__.Array[_c_ntfs.uint8] + IndexHeader: _c_ntfs._INDEX_HEADER + @overload + def __init__( + self, + AttributeType: _c_ntfs.ATTRIBUTE_TYPE_CODE | None = ..., + CollationRule: _c_ntfs.COLLATION | None = ..., + BytesPerIndexBuffer: _c_ntfs.uint32 | None = ..., + ClustersPerIndexBuffer: _c_ntfs.uint8 | None = ..., + Reserved: __cs__.Array[_c_ntfs.uint8] | None = ..., + IndexHeader: _c_ntfs._INDEX_HEADER | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + INDEX_ROOT: TypeAlias = _INDEX_ROOT + class _INDEX_ALLOCATION_BUFFER(__cs__.Structure): + MultiSectorHeader: _c_ntfs._MULTI_SECTOR_HEADER + Lsn: _c_ntfs.uint64 + Vcn: _c_ntfs.uint64 + IndexHeader: _c_ntfs._INDEX_HEADER + @overload + def __init__( + self, + MultiSectorHeader: _c_ntfs._MULTI_SECTOR_HEADER | None = ..., + Lsn: _c_ntfs.uint64 | None = ..., + Vcn: _c_ntfs.uint64 | None = ..., + IndexHeader: _c_ntfs._INDEX_HEADER | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + INDEX_ALLOCATION_BUFFER: TypeAlias = _INDEX_ALLOCATION_BUFFER + class _INDEX_ENTRY(__cs__.Structure): + FileReference: _c_ntfs._MFT_SEGMENT_REFERENCE + DataOffset: _c_ntfs.uint16 + DataLength: _c_ntfs.uint16 + _: _c_ntfs.uint32 + Length: _c_ntfs.uint16 + KeyLength: _c_ntfs.uint16 + Flags: _c_ntfs.uint16 + Reserved: _c_ntfs.uint16 + @overload + def __init__( + self, + FileReference: _c_ntfs._MFT_SEGMENT_REFERENCE | None = ..., + DataOffset: _c_ntfs.uint16 | None = ..., + DataLength: _c_ntfs.uint16 | None = ..., + _: _c_ntfs.uint32 | None = ..., + Length: _c_ntfs.uint16 | None = ..., + KeyLength: _c_ntfs.uint16 | None = ..., + Flags: _c_ntfs.uint16 | None = ..., + Reserved: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + INDEX_ENTRY: TypeAlias = _INDEX_ENTRY + class SECURITY_DESCRIPTOR_CONTROL(__cs__.Flag): + SE_OWNER_DEFAULTED = ... + SE_GROUP_DEFAULTED = ... + SE_DACL_PRESENT = ... + SE_DACL_DEFAULTED = ... + SE_SACL_PRESENT = ... + SE_SACL_DEFAULTED = ... + SE_DACL_AUTO_INHERIT_REQ = ... + SE_SACL_AUTO_INHERIT_REQ = ... + SE_DACL_AUTO_INHERITED = ... + SE_SACL_AUTO_INHERITED = ... + SE_DACL_PROTECTED = ... + SE_SACL_PROTECTED = ... + SE_RM_CONTROL_VALID = ... + SE_SELF_RELATIVE = ... + + class ACCESS_MASK(__cs__.Flag): + FILE_READ_DATA = ... + FILE_LIST_DIRECTORY = ... + FILE_WRITE_DATA = ... + FILE_ADD_FILE = ... + FILE_APPEND_DATA = ... + FILE_ADD_SUBDIRECTORY = ... + FILE_READ_EA = ... + FILE_WRITE_EA = ... + FILE_EXECUTE = ... + FILE_TRAVERSE = ... + FILE_DELETE_CHILD = ... + FILE_READ_ATTRIBUTES = ... + FILE_WRITE_ATTRIBUTES = ... + DELETE = ... + READ_CONTROL = ... + WRITE_DAC = ... + WRITE_OWNER = ... + SYNCHRONIZE = ... + STANDARD_RIGHTS_READ = ... + STANDARD_RIGHTS_WRITE = ... + STANDARD_RIGHTS_EXECUTE = ... + STANDARD_RIGHTS_REQUIRED = ... + STANDARD_RIGHTS_ALL = ... + ACCESS_SYSTEM_SECURITY = ... + MAXIMUM_ALLOWED = ... + GENERIC_ALL = ... + GENERIC_EXECUTE = ... + GENERIC_WRITE = ... + GENERIC_READ = ... + + class ACE_TYPE(__cs__.Enum): + ACCESS_ALLOWED = ... + ACCESS_DENIED = ... + SYSTEM_AUDIT = ... + SYSTEM_ALARM = ... + ACCESS_ALLOWED_COMPOUND = ... + ACCESS_ALLOWED_OBJECT = ... + ACCESS_DENIED_OBJECT = ... + SYSTEM_AUDIT_OBJECT = ... + SYSTEM_ALARM_OBJECT = ... + ACCESS_ALLOWED_CALLBACK = ... + ACCESS_DENIED_CALLBACK = ... + ACCESS_ALLOWED_CALLBACK_OBJECT = ... + ACCESS_DENIED_CALLBACK_OBJECT = ... + SYSTEM_AUDIT_CALLBACK = ... + SYSTEM_ALARM_CALLBACK = ... + SYSTEM_AUDIT_CALLBACK_OBJECT = ... + SYSTEM_ALARM_CALLBACK_OBJECT = ... + SYSTEM_MANDATORY_LABEL = ... + SYSTEM_RESOURCE_ATTRIBUTE = ... + SYSTEM_SCOPED_POLICY_ID = ... + SYSTEM_PROCESS_TRUST_LABEL = ... + SYSTEM_ACCESS_FILTER = ... + + class ACE_FLAGS(__cs__.Flag): + OBJECT_INHERIT_ACE = ... + CONTAINER_INHERIT_ACE = ... + NO_PROPAGATE_INHERIT_ACE = ... + INHERIT_ONLY_ACE = ... + INHERITED_ACE = ... + SUCCESSFUL_ACCESS_ACE_FLAG = ... + FAILED_ACCESS_ACE_FLAG = ... + + class ACE_OBJECT_FLAGS(__cs__.Flag): + ACE_OBJECT_TYPE_PRESENT = ... + ACE_INHERITED_OBJECT_TYPE_PRESENT = ... + + class COMPOUND_ACE_TYPE(__cs__.Enum): + COMPOUND_ACE_IMPERSONATION = ... + + class _ACL(__cs__.Structure): + AclRevision: _c_ntfs.uint8 + Sbz1: _c_ntfs.uint8 + AclSize: _c_ntfs.uint16 + AceCount: _c_ntfs.uint16 + Sbz2: _c_ntfs.uint16 + @overload + def __init__( + self, + AclRevision: _c_ntfs.uint8 | None = ..., + Sbz1: _c_ntfs.uint8 | None = ..., + AclSize: _c_ntfs.uint16 | None = ..., + AceCount: _c_ntfs.uint16 | None = ..., + Sbz2: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + ACL: TypeAlias = _ACL + class _ACE_HEADER(__cs__.Structure): + AceType: _c_ntfs.ACE_TYPE + AceFlags: _c_ntfs.ACE_FLAGS + AceSize: _c_ntfs.uint16 + @overload + def __init__( + self, + AceType: _c_ntfs.ACE_TYPE | None = ..., + AceFlags: _c_ntfs.ACE_FLAGS | None = ..., + AceSize: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + ACE_HEADER: TypeAlias = _ACE_HEADER + class _SECURITY_DESCRIPTOR_HEADER(__cs__.Structure): + HashId: _c_ntfs.uint32 + SecurityId: _c_ntfs.uint32 + Offset: _c_ntfs.uint64 + Length: _c_ntfs.uint32 + @overload + def __init__( + self, + HashId: _c_ntfs.uint32 | None = ..., + SecurityId: _c_ntfs.uint32 | None = ..., + Offset: _c_ntfs.uint64 | None = ..., + Length: _c_ntfs.uint32 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + SECURITY_DESCRIPTOR_HEADER: TypeAlias = _SECURITY_DESCRIPTOR_HEADER + class _SECURITY_DESCRIPTOR_RELATIVE(__cs__.Structure): + Revision: _c_ntfs.uint8 + Sbz1: _c_ntfs.uint8 + Control: _c_ntfs.SECURITY_DESCRIPTOR_CONTROL + Owner: _c_ntfs.uint32 + Group: _c_ntfs.uint32 + Sacl: _c_ntfs.uint32 + Dacl: _c_ntfs.uint32 + @overload + def __init__( + self, + Revision: _c_ntfs.uint8 | None = ..., + Sbz1: _c_ntfs.uint8 | None = ..., + Control: _c_ntfs.SECURITY_DESCRIPTOR_CONTROL | None = ..., + Owner: _c_ntfs.uint32 | None = ..., + Group: _c_ntfs.uint32 | None = ..., + Sacl: _c_ntfs.uint32 | None = ..., + Dacl: _c_ntfs.uint32 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + SECURITY_DESCRIPTOR_RELATIVE: TypeAlias = _SECURITY_DESCRIPTOR_RELATIVE + class USN_REASON(__cs__.Flag): + DATA_OVERWRITE = ... + DATA_EXTEND = ... + DATA_TRUNCATION = ... + NAMED_DATA_OVERWRITE = ... + NAMED_DATA_EXTEND = ... + NAMED_DATA_TRUNCATION = ... + FILE_CREATE = ... + FILE_DELETE = ... + EA_CHANGE = ... + SECURITY_CHANGE = ... + RENAME_OLD_NAME = ... + RENAME_NEW_NAME = ... + INDEXABLE_CHANGE = ... + BASIC_INFO_CHANGE = ... + HARD_LINK_CHANGE = ... + COMPRESSION_CHANGE = ... + ENCRYPTION_CHANGE = ... + OBJECT_ID_CHANGE = ... + REPARSE_POINT_CHANGE = ... + STREAM_CHANGE = ... + TRANSACTED_CHANGE = ... + INTEGRITY_CHANGE = ... + CLOSE = ... + + class USN_SOURCE(__cs__.Flag): + NORMAL = ... + DATA_MANAGEMENT = ... + AUXILIARY_DATA = ... + REPLICATION_MANAGEMENT = ... + CLIENT_REPLICATION_MANAGEMENT = ... + + class _FILE_ID_128(__cs__.Structure): + Identifier: __cs__.Array[_c_ntfs.uint8] + @overload + def __init__(self, Identifier: __cs__.Array[_c_ntfs.uint8] | None = ...): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + FILE_ID_128: TypeAlias = _FILE_ID_128 + class USN_RECORD_COMMON_HEADER(__cs__.Structure): + RecordLength: _c_ntfs.uint32 + MajorVersion: _c_ntfs.uint16 + MinorVersion: _c_ntfs.uint16 + @overload + def __init__( + self, + RecordLength: _c_ntfs.uint32 | None = ..., + MajorVersion: _c_ntfs.uint16 | None = ..., + MinorVersion: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + class USN_RECORD_V2(__cs__.Structure): + RecordLength: _c_ntfs.uint32 + MajorVersion: _c_ntfs.uint16 + MinorVersion: _c_ntfs.uint16 + FileReferenceNumber: _c_ntfs._MFT_SEGMENT_REFERENCE + ParentFileReferenceNumber: _c_ntfs._MFT_SEGMENT_REFERENCE + Usn: _c_ntfs.uint64 + TimeStamp: _c_ntfs.uint64 + Reason: _c_ntfs.USN_REASON + SourceInfo: _c_ntfs.USN_SOURCE + SecurityId: _c_ntfs.uint32 + FileAttributes: _c_ntfs.FILE_ATTRIBUTE + FileNameLength: _c_ntfs.uint16 + FileNameOffset: _c_ntfs.uint16 + @overload + def __init__( + self, + RecordLength: _c_ntfs.uint32 | None = ..., + MajorVersion: _c_ntfs.uint16 | None = ..., + MinorVersion: _c_ntfs.uint16 | None = ..., + FileReferenceNumber: _c_ntfs._MFT_SEGMENT_REFERENCE | None = ..., + ParentFileReferenceNumber: _c_ntfs._MFT_SEGMENT_REFERENCE | None = ..., + Usn: _c_ntfs.uint64 | None = ..., + TimeStamp: _c_ntfs.uint64 | None = ..., + Reason: _c_ntfs.USN_REASON | None = ..., + SourceInfo: _c_ntfs.USN_SOURCE | None = ..., + SecurityId: _c_ntfs.uint32 | None = ..., + FileAttributes: _c_ntfs.FILE_ATTRIBUTE | None = ..., + FileNameLength: _c_ntfs.uint16 | None = ..., + FileNameOffset: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + class USN_RECORD_V3(__cs__.Structure): + RecordLength: _c_ntfs.uint32 + MajorVersion: _c_ntfs.uint16 + MinorVersion: _c_ntfs.uint16 + FileReferenceNumber: _c_ntfs._FILE_ID_128 + ParentFileReferenceNumber: _c_ntfs._FILE_ID_128 + Usn: _c_ntfs.uint64 + TimeStamp: _c_ntfs.uint64 + Reason: _c_ntfs.USN_REASON + SourceInfo: _c_ntfs.USN_SOURCE + SecurityId: _c_ntfs.uint32 + FileAttributes: _c_ntfs.FILE_ATTRIBUTE + FileNameLength: _c_ntfs.uint16 + FileNameOffset: _c_ntfs.uint16 + @overload + def __init__( + self, + RecordLength: _c_ntfs.uint32 | None = ..., + MajorVersion: _c_ntfs.uint16 | None = ..., + MinorVersion: _c_ntfs.uint16 | None = ..., + FileReferenceNumber: _c_ntfs._FILE_ID_128 | None = ..., + ParentFileReferenceNumber: _c_ntfs._FILE_ID_128 | None = ..., + Usn: _c_ntfs.uint64 | None = ..., + TimeStamp: _c_ntfs.uint64 | None = ..., + Reason: _c_ntfs.USN_REASON | None = ..., + SourceInfo: _c_ntfs.USN_SOURCE | None = ..., + SecurityId: _c_ntfs.uint32 | None = ..., + FileAttributes: _c_ntfs.FILE_ATTRIBUTE | None = ..., + FileNameLength: _c_ntfs.uint16 | None = ..., + FileNameOffset: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + class USN_RECORD_EXTENT(__cs__.Structure): + Offset: _c_ntfs.int64 + Length: _c_ntfs.int64 + @overload + def __init__(self, Offset: _c_ntfs.int64 | None = ..., Length: _c_ntfs.int64 | None = ...): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + + class USN_RECORD_V4(__cs__.Structure): + RecordLength: _c_ntfs.uint32 + MajorVersion: _c_ntfs.uint16 + MinorVersion: _c_ntfs.uint16 + FileReferenceNumber: _c_ntfs._FILE_ID_128 + ParentFileReferenceNumber: _c_ntfs._FILE_ID_128 + Usn: _c_ntfs.uint64 + Reason: _c_ntfs.USN_REASON + SourceInfo: _c_ntfs.USN_SOURCE + RemainingExtents: _c_ntfs.uint32 + NumberOfExtents: _c_ntfs.uint16 + ExtentSize: _c_ntfs.uint16 + @overload + def __init__( + self, + RecordLength: _c_ntfs.uint32 | None = ..., + MajorVersion: _c_ntfs.uint16 | None = ..., + MinorVersion: _c_ntfs.uint16 | None = ..., + FileReferenceNumber: _c_ntfs._FILE_ID_128 | None = ..., + ParentFileReferenceNumber: _c_ntfs._FILE_ID_128 | None = ..., + Usn: _c_ntfs.uint64 | None = ..., + Reason: _c_ntfs.USN_REASON | None = ..., + SourceInfo: _c_ntfs.USN_SOURCE | None = ..., + RemainingExtents: _c_ntfs.uint32 | None = ..., + NumberOfExtents: _c_ntfs.uint16 | None = ..., + ExtentSize: _c_ntfs.uint16 | None = ..., + ): ... + @overload + def __init__(self, fh: bytes | memoryview | bytearray | BinaryIO, /): ... + +# Technically `c_ntfs` is an instance of `_c_ntfs`, but then we can't use it in type hints +c_ntfs: TypeAlias = _c_ntfs + +ATTRIBUTE_TYPE_CODE: TypeAlias = c_ntfs.ATTRIBUTE_TYPE_CODE +IO_REPARSE_TAG: TypeAlias = c_ntfs.IO_REPARSE_TAG +ACCESS_MASK: TypeAlias = c_ntfs.ACCESS_MASK +ACE_TYPE: TypeAlias = c_ntfs.ACE_TYPE +ACE_OBJECT_FLAGS: TypeAlias = c_ntfs.ACE_OBJECT_FLAGS +COLLATION: TypeAlias = c_ntfs.COLLATION + +NTFS_SIGNATURE: bytes = ... + +SECTOR_SIZE: int = ... +SECTOR_SHIFT: int = ... + +USN_PAGE_SIZE: int = ... + +DEFAULT_SECTOR_SIZE: int = ... +DEFAULT_CLUSTER_SIZE: int = ... +DEFAULT_RECORD_SIZE: int = ... +DEFAULT_INDEX_SIZE: int = ... + +FILE_NUMBER_MFT: int = ... +FILE_NUMBER_MFTMIRR: int = ... +FILE_NUMBER_LOGFILE: int = ... +FILE_NUMBER_VOLUME: int = ... +FILE_NUMBER_ATTRDEF: int = ... +FILE_NUMBER_ROOT: int = ... +FILE_NUMBER_BITMAP: int = ... +FILE_NUMBER_BOOT: int = ... +FILE_NUMBER_BADCLUS: int = ... +FILE_NUMBER_SECURE: int = ... +FILE_NUMBER_UPCASE: int = ... +FILE_NUMBER_EXTEND: int = ... + +FILE_RECORD_SEGMENT_IN_USE: int = ... +FILE_FILE_NAME_INDEX_PRESENT: int = ... + +ATTRIBUTE_FLAG_COMPRESSION_MASK: int = ... +ATTRIBUTE_FLAG_ENCRYPTED: int = ... +ATTRIBUTE_FLAG_SPARSE: int = ... + +FILE_NAME_NTFS: int = ... +FILE_NAME_DOS: int = ... + +COMPRESSION_FORMAT_NONE: int = ... +COMPRESSION_FORMAT_DEFAULT: int = ... +COMPRESSION_FORMAT_LZNT1: int = ... + +INDEX_NODE: int = ... +INDEX_ENTRY_NODE: int = ... +INDEX_ENTRY_END: int = ... diff --git a/dissect/ntfs/index.py b/dissect/ntfs/index.py index ec01ca3..2379d0c 100644 --- a/dissect/ntfs/index.py +++ b/dissect/ntfs/index.py @@ -6,22 +6,14 @@ from typing import TYPE_CHECKING, Any, BinaryIO, Callable from dissect.ntfs.attr import AttributeRecord -from dissect.ntfs.c_ntfs import ( - ATTRIBUTE_TYPE_CODE, - COLLATION, - INDEX_ENTRY_END, - INDEX_ENTRY_NODE, - SECTOR_SHIFT, - c_ntfs, - segment_reference, -) +from dissect.ntfs.c_ntfs import ATTRIBUTE_TYPE_CODE, COLLATION, INDEX_ENTRY_END, INDEX_ENTRY_NODE, SECTOR_SHIFT, c_ntfs from dissect.ntfs.exceptions import ( BrokenIndexError, Error, FileNotFoundError, MftNotAvailableError, ) -from dissect.ntfs.util import apply_fixup +from dissect.ntfs.util import apply_fixup, segment_reference if TYPE_CHECKING: from collections.abc import Iterator diff --git a/dissect/ntfs/mft.py b/dissect/ntfs/mft.py index b978268..dc4a517 100644 --- a/dissect/ntfs/mft.py +++ b/dissect/ntfs/mft.py @@ -16,7 +16,6 @@ FILE_NUMBER_ROOT, IO_REPARSE_TAG, c_ntfs, - segment_reference, ) from dissect.ntfs.exceptions import ( BrokenMftError, @@ -27,7 +26,7 @@ NotAReparsePointError, ) from dissect.ntfs.index import Index, IndexEntry -from dissect.ntfs.util import AttributeCollection, AttributeMap, apply_fixup +from dissect.ntfs.util import AttributeCollection, AttributeMap, apply_fixup, segment_reference if TYPE_CHECKING: from collections.abc import Iterator @@ -73,11 +72,11 @@ def _get_path(self, path: str, root: MftRecord | None = None) -> MftRecord: parts = search_path.split("\\") - for part_num, part in enumerate(parts): + for part in parts: if not part: continue - while node.is_reparse_point() and part_num < len(parts): + while node.is_symlink() or node.is_mount_point(): node = node.reparse_point_record if not node.is_dir(): @@ -331,6 +330,28 @@ def is_mount_point(self) -> bool: attr = self.attributes[ATTRIBUTE_TYPE_CODE.REPARSE_POINT] return bool(attr) and attr.tag == IO_REPARSE_TAG.MOUNT_POINT + def is_cloud_file(self) -> bool: + """Return whether this record is a cloud reparse point.""" + attr = self.attributes[ATTRIBUTE_TYPE_CODE.REPARSE_POINT] + return bool(attr) and attr.tag in ( + IO_REPARSE_TAG.CLOUD, + IO_REPARSE_TAG.CLOUD_1, + IO_REPARSE_TAG.CLOUD_2, + IO_REPARSE_TAG.CLOUD_3, + IO_REPARSE_TAG.CLOUD_4, + IO_REPARSE_TAG.CLOUD_5, + IO_REPARSE_TAG.CLOUD_6, + IO_REPARSE_TAG.CLOUD_7, + IO_REPARSE_TAG.CLOUD_8, + IO_REPARSE_TAG.CLOUD_9, + IO_REPARSE_TAG.CLOUD_A, + IO_REPARSE_TAG.CLOUD_B, + IO_REPARSE_TAG.CLOUD_C, + IO_REPARSE_TAG.CLOUD_D, + IO_REPARSE_TAG.CLOUD_E, + IO_REPARSE_TAG.CLOUD_F, + ) + @cached_property def reparse_point_name(self) -> str: """Return the (printable) name of this reparse point.""" diff --git a/dissect/ntfs/ntfs.py b/dissect/ntfs/ntfs.py index 2f44270..687ab5c 100644 --- a/dissect/ntfs/ntfs.py +++ b/dissect/ntfs/ntfs.py @@ -11,13 +11,13 @@ DEFAULT_SECTOR_SIZE, FILE_NUMBER_VOLUME, NTFS_SIGNATURE, - bsf, c_ntfs, ) from dissect.ntfs.exceptions import Error, FileNotFoundError, VolumeNotAvailableError from dissect.ntfs.mft import Mft, MftRecord from dissect.ntfs.secure import Secure from dissect.ntfs.usnjrnl import UsnJrnl +from dissect.ntfs.util import bsf if TYPE_CHECKING: from collections.abc import Iterator diff --git a/dissect/ntfs/usnjrnl.py b/dissect/ntfs/usnjrnl.py index 807c427..632c60c 100644 --- a/dissect/ntfs/usnjrnl.py +++ b/dissect/ntfs/usnjrnl.py @@ -6,9 +6,9 @@ from dissect.util.stream import RunlistStream from dissect.util.ts import wintimestamp -from dissect.ntfs.c_ntfs import USN_PAGE_SIZE, c_ntfs, segment_reference +from dissect.ntfs.c_ntfs import USN_PAGE_SIZE, c_ntfs from dissect.ntfs.exceptions import Error -from dissect.ntfs.util import ts_to_ns +from dissect.ntfs.util import segment_reference, ts_to_ns if TYPE_CHECKING: from collections.abc import Iterator diff --git a/dissect/ntfs/util.py b/dissect/ntfs/util.py index 4b0f302..dabb24d 100644 --- a/dissect/ntfs/util.py +++ b/dissect/ntfs/util.py @@ -14,7 +14,6 @@ SECTOR_SHIFT, SECTOR_SIZE, c_ntfs, - segment_reference, ) from dissect.ntfs.exceptions import FilenameNotAvailableError, VolumeNotAvailableError from dissect.ntfs.stream import CompressedRunlistStream @@ -283,3 +282,41 @@ def get_full_path(mft: Mft, name: str, parent: c_ntfs._MFT_SEGMENT_REFERENCE, se def ts_to_ns(ts: int) -> int: """Convert Windows timestamps to nanosecond timestamps.""" return (ts * 100) - 11644473600000000000 + + +def segment_reference(reference: c_ntfs._MFT_SEGMENT_REFERENCE) -> int: + """Helper to calculate the complete segment number from a cstruct MFT segment reference. + + Args: + reference: A cstruct _MFT_SEGMENT_REFERENCE instance to return the complete segment number of. + """ + return reference.SegmentNumberLowPart | (reference.SegmentNumberHighPart << 32) + + +def varint(buf: bytes) -> int: + """Parse variable integers. + + Dataruns in NTFS are stored as a tuple of variable sized integers. The size of each integer is + stored in the first byte, 4 bits for each integer. This logic can be seen in + :func:`AttributeHeader.dataruns `. + + This function only parses those variable amount of bytes into actual integers. To do that, we + simply pad the bytes to 8 bytes long and parse it as a signed 64 bit integer. We pad with 0xff + if the number is negative and 0x00 otherwise. + + Args: + buf: The byte buffer to parse a varint from. + """ + if len(buf) < 8: + buf += (b"\xff" if buf[-1] & 0x80 else b"\x00") * (8 - len(buf)) + + return struct.unpack(" int: + """Count the number of trailing zero bits in an integer of a given size. + + Args: + value: The integer to count trailing zero bits in. + """ + return (value & -value).bit_length() - 1 if value else 0 diff --git a/pyproject.toml b/pyproject.toml index 8cd8558..f55e9f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,8 @@ select = [ ignore = ["E203", "B904", "UP024", "ANN002", "ANN003", "ANN204", "ANN401", "SIM105", "TRY003"] [tool.ruff.lint.per-file-ignores] -"tests/docs/**" = ["INP001"] +"tests/_docs/**" = ["INP001"] +"*.pyi" = ["PYI042", "F401"] [tool.ruff.lint.isort] known-first-party = ["dissect.ntfs"] diff --git a/tests/_data/boot_2m.bin.gz b/tests/_data/boot_2m.bin.gz new file mode 100644 index 0000000..8485a78 --- /dev/null +++ b/tests/_data/boot_2m.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:360b5a85f95d358d630561cb5376b111e57266009214190c5ee1dd5ab2cbf25c +size 5647 diff --git a/tests/_data/mft.bin.gz b/tests/_data/mft.bin.gz new file mode 100644 index 0000000..c54168a --- /dev/null +++ b/tests/_data/mft.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ec5917559984b5bd23a41ec6fe6aa9892c8349e31012449fe38995ede4661e63 +size 4290 diff --git a/tests/_data/ntfs-cloud.bin.gz b/tests/_data/ntfs-cloud.bin.gz new file mode 100644 index 0000000..c4a76fa --- /dev/null +++ b/tests/_data/ntfs-cloud.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c44f83257be7a8e0ff1aecb61fe82b362c44256808da216d8824ea735576af5a +size 1234746 diff --git a/tests/_data/ntfs.bin.gz b/tests/_data/ntfs.bin.gz new file mode 100644 index 0000000..947ac05 --- /dev/null +++ b/tests/_data/ntfs.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cc1b5c665a80befed008b3db817325aac1555e1efe511de63203f421c2250080 +size 908628 diff --git a/tests/_data/ntfs_fragmented_mft.csv.gz b/tests/_data/ntfs_fragmented_mft.csv.gz new file mode 100644 index 0000000..75899c0 --- /dev/null +++ b/tests/_data/ntfs_fragmented_mft.csv.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a312c02c16173fc9550113878d7b1657e4b19902ff6c052207334d4ac7496970 +size 3999 diff --git a/tests/_data/sds.bin.gz b/tests/_data/sds.bin.gz new file mode 100644 index 0000000..9fe465a --- /dev/null +++ b/tests/_data/sds.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5a1c70deee11511da80a390d1c5021b50087e3278d9d2e4eb25c9c71238b623e +size 1243 diff --git a/tests/_data/sds_complex.bin.gz b/tests/_data/sds_complex.bin.gz new file mode 100644 index 0000000..773145f --- /dev/null +++ b/tests/_data/sds_complex.bin.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22e6e52207f98289d4d7731204f9140eee181ab30060aa37dad6dd0b6f7fe37d +size 970 diff --git a/tests/docs/Makefile b/tests/_docs/Makefile similarity index 91% rename from tests/docs/Makefile rename to tests/_docs/Makefile index 69e0098..e693b42 100644 --- a/tests/docs/Makefile +++ b/tests/_docs/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line, and also # from the environment for the first two. -SPHINXOPTS ?= -jauto +SPHINXOPTS ?= -jauto -w $(BUILDDIR)/warnings.log --fail-on-warning SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = build diff --git a/tests/_docs/__init__.py b/tests/_docs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/docs/conf.py b/tests/_docs/conf.py similarity index 83% rename from tests/docs/conf.py rename to tests/_docs/conf.py index 7ef62d3..741b830 100644 --- a/tests/docs/conf.py +++ b/tests/_docs/conf.py @@ -1,3 +1,5 @@ +project = "dissect.ntfs" + extensions = [ "autoapi.extension", "sphinx.ext.autodoc", @@ -32,3 +34,8 @@ autodoc_member_order = "groupwise" autosectionlabel_prefix_document = True + +suppress_warnings = [ + # https://github.com/readthedocs/sphinx-autoapi/issues/285 + "autoapi.python_import_resolution", +] diff --git a/tests/docs/index.rst b/tests/_docs/index.rst similarity index 100% rename from tests/docs/index.rst rename to tests/_docs/index.rst diff --git a/tests/conftest.py b/tests/conftest.py index 7a22967..fa738cc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -24,27 +24,32 @@ def open_file_gz(name: str, mode: str = "rb") -> Iterator[BinaryIO]: @pytest.fixture def ntfs_bin() -> Iterator[BinaryIO]: - yield from open_file_gz("data/ntfs.bin.gz") + yield from open_file_gz("_data/ntfs.bin.gz") @pytest.fixture def mft_bin() -> Iterator[BinaryIO]: - yield from open_file_gz("data/mft.bin.gz") + yield from open_file_gz("_data/mft.bin.gz") + + +@pytest.fixture +def ntfs_cloud_bin() -> Iterator[BinaryIO]: + yield from open_file_gz("_data/ntfs-cloud.bin.gz") @pytest.fixture def sds_bin() -> Iterator[BinaryIO]: - yield from open_file_gz("data/sds.bin.gz") + yield from open_file_gz("_data/sds.bin.gz") @pytest.fixture def sds_complex_bin() -> Iterator[BinaryIO]: - yield from open_file_gz("data/sds_complex.bin.gz") + yield from open_file_gz("_data/sds_complex.bin.gz") @pytest.fixture def boot_2m_bin() -> Iterator[BinaryIO]: - yield from open_file_gz("data/boot_2m.bin.gz") + yield from open_file_gz("_data/boot_2m.bin.gz") @pytest.fixture @@ -55,7 +60,7 @@ def ntfs_fragmented_mft_fh() -> BinaryIO: # We use a MappingStream to stitch everything together at the correct offsets stream = MappingStream(align=512) - with io.TextIOWrapper(gzip.open(absolute_path("data/ntfs_fragmented_mft.csv.gz"), "r")) as fh: + with io.TextIOWrapper(gzip.open(absolute_path("_data/ntfs_fragmented_mft.csv.gz"), "r")) as fh: for offset, data in csv.reader(fh): buf = bytes.fromhex(data) stream.add(int(offset), len(buf), io.BytesIO(buf), 0) diff --git a/tests/data/boot_2m.bin.gz b/tests/data/boot_2m.bin.gz deleted file mode 100644 index 75d18df..0000000 Binary files a/tests/data/boot_2m.bin.gz and /dev/null differ diff --git a/tests/data/mft.bin.gz b/tests/data/mft.bin.gz deleted file mode 100644 index fc87838..0000000 Binary files a/tests/data/mft.bin.gz and /dev/null differ diff --git a/tests/data/ntfs.bin.gz b/tests/data/ntfs.bin.gz deleted file mode 100644 index 4a3fb07..0000000 Binary files a/tests/data/ntfs.bin.gz and /dev/null differ diff --git a/tests/data/ntfs_fragmented_mft.csv.gz b/tests/data/ntfs_fragmented_mft.csv.gz deleted file mode 100644 index cecc006..0000000 Binary files a/tests/data/ntfs_fragmented_mft.csv.gz and /dev/null differ diff --git a/tests/data/sds.bin.gz b/tests/data/sds.bin.gz deleted file mode 100644 index 3e3154b..0000000 Binary files a/tests/data/sds.bin.gz and /dev/null differ diff --git a/tests/data/sds_complex.bin.gz b/tests/data/sds_complex.bin.gz deleted file mode 100644 index 6c18b98..0000000 Binary files a/tests/data/sds_complex.bin.gz and /dev/null differ diff --git a/tests/test_mft.py b/tests/test_mft.py index 6800775..e1e9b4f 100644 --- a/tests/test_mft.py +++ b/tests/test_mft.py @@ -96,3 +96,51 @@ def test_mft_records_segment_number(mft_bin: BinaryIO) -> None: assert len(records_backwards) == 5 assert records_backwards[0].filename == "$AttrDef" assert records_backwards[4].filename == "$MFT" + + +def test_mft_record_reparse_cloud(mft_bin: BinaryIO) -> None: + """Test if the ``MftRecord.is_cloud_file()`` method works correctly.""" + + fs = NTFS(mft=mft_bin) + + data = bytes.fromhex( + "46494c4530000300279c0086000000000b000100380001001003000000040000000000000000000007000000748a00000600" + "0108000000001000000060000000000000000000000048000000180000001084449ee606dc014e3201a4e606dc01d6857682" + "e706dc01b3012f82e706dc012004000000000000000000000000000000000000310900000000000000000000b860d4210000" + "0000300000007000000000000000000004005800000018000100e3a00100000002001084449ee606dc011084449ee606dc01" + "1084449ee606dc011084449ee606dc010000000000000000000000000000000020000000000000000b036500780061006d00" + "70006c0065002e00740078007400400000002800000000000000000005001000000018000000579007b6d972f011ba7f000c" + "296de6358000000050000000000018000000010031000000180000005468697320697320616e206578616d706c652066696c" + "6520696e20746865204f6e65447269766520666f6c646572210d0a00200044006f00c0000000880100000000000000000600" + "6c010000180000001a600090640100000180e8015db1004665527041e8879700e401000002000a0080070001006000000048" + "0804006400380600080082c8006c1100600168003c5b1508014ed0004e010ed8000e0a30000400e0000e0004004635006803" + "000660041e001400310306000006343632656230003432393832353439003566623337313062006263313465386632143530" + "0426002527373765003164303837356139003534356238623664003535373332653230203866396233054f39390066383435" + "3161306300333134393234393400383135653664613840363138313065861334003335353935303465006462346466636135" + "0030613037313162614061393830666685133000383466326434363308323766801535383338203561313537000b33628438" + "3005770038613680060037616331333136370062310018543d78730185220000a0a54fa67401809337c8f6bf2b214700b52e" + "a7965bd16b7cc0aff06cabfa05980b0630030725860400000000ffffffff8279471100000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" + "000000000000000000000000000000000000000000000600" + ) + record = MftRecord.from_bytes(data, fs) + + assert record.filename == "example.txt" + assert record.is_reparse_point() + assert record.is_cloud_file() + assert record.open().read() == b"This is an example file in the OneDrive folder!\r\n" + + +def test_mft_record_reparse_cloud_full(ntfs_cloud_bin: BinaryIO) -> None: + """Test if offline saved OneDrive MftRecords can be found and read succesfully.""" + + fs = NTFS(ntfs_cloud_bin) + + record = fs.mft.get("OneDrive/example.txt") + assert record.filename == "example.txt" + assert record.is_cloud_file() + assert record.is_file() + assert record.open().read() == b"This is an example file in the OneDrive folder!\r\n" diff --git a/tox.ini b/tox.ini index 17e3629..c4d1023 100644 --- a/tox.ini +++ b/tox.ini @@ -55,12 +55,12 @@ deps = sphinx-design furo commands = - make -C tests/docs clean - make -C tests/docs html + make -C tests/_docs clean + make -C tests/_docs html [testenv:docs-linkcheck] allowlist_externals = make deps = {[testenv:docs-build]deps} commands = - make -C tests/docs clean - make -C tests/docs linkcheck + make -C tests/_docs clean + make -C tests/_docs linkcheck