add initial support for $I30 index records

dtformats/indx_directory_entry.py

@@ -0,0 +1,144 @@
+# -*- coding: utf-8 -*-
+"""INDX entries """
+
+import os
+
+from dtformats import data_format
+from dtformats.errors import ParseError
+
+class INDXRecord(data_format.BinaryDataFile):
+  """ Class that represents an INDX record.
+
+  Note: currently only supports $I30 index records
+  """
+
+  _FABRIC = data_format.BinaryDataFile.ReadDefinitionFile(
+    'indx_directory_entry.yml')
+
+  _DEBUG_INDX_ENTRY_HEADER = [
+    ('signature', 'signature', ''),
+    ('fixup_value_offset', 'fixup_value_offset', '_FormatIntegerAsDecimal'),
+    ('num_fixup_values', 'num_fixup_values', '_FormatIntegerAsDecimal'),
+    ('logfile_sequence_number', 'logfile_sequence_number',
+     '_FormatIntegerAsDecimal'),
+    ('virtual_cluster_number', 'virtual_cluster_number',
+     '_FormatIntegerAsDecimal')]
+
+  _DEBUG_INDX_NODE_HEADER = [
+    ('index_values_offset', 'index_values_offset',
+     '_FormatIntegerAsDecimal'),
+    ('index_node_size', 'index_node_size', '_FormatIntegerAsDecimal'),
+    ('allocated_index_node_size', 'allocated_index_node_size',
+     '_FormatIntegerAsDecimal'),
+    ('index_node_flags', 'index_node_flags', '_FormatIntegerAsDecimal')]
+
+  _DEBUG_INDX_DIR_RECORD = [
+      ('file_reference', 'file_reference', '_FormatIntegerAsDecimal'),
+      ('index_value_size', 'index_value_size', '_FormatIntegerAsDecimal'),
+      ('index_key_data_size', 'index_key_data_size', '_FormatIntegerAsDecimal'),
+      ('index_value_flags', 'index_value_flags', '_FormatIntegerAsDecimal')]
+
+  _DEBUG_FILE_NAME_ATTR = [
+      ('parent_file_reference', 'parent_file_reference',
+       '_FormatIntegerAsDecimal'),
+      ('creation_time', 'creation_time', '_FormatIntegerAsDecimal'),
+      ('modification_time', 'modification_time', '_FormatIntegerAsDecimal'),
+      ('entry_modification_time', 'entry_modification_time',
+       '_FormatIntegerAsDecimal'),
+      ('access_time', 'access_time', '_FormatIntegerAsDecimal'),
+      ('allocated_file_size', 'allocated_file_size', '_FormatIntegerAsDecimal'),
+      ('file_size', 'file_size', '_FormatIntegerAsDecimal'),
+      ('file_attribute_flags', 'file_attribute_flags',
+       '_FormatIntegerAsDecimal'),
+      ('extended_data', 'extended_data', '_FormatIntegerAsDecimal'),
+      ('name_size', 'name_size', '_FormatIntegerAsDecimal'),
+      ('name_space', 'name_space', '_FormatIntegerAsDecimal'),
+      ('filename', 'filename', '_FormatString')]
+
+  def PrintRecord(self, record):
+    """
+    Prints a human readable version of the INDX record
+    to STDOUT.
+
+    Args:
+        record (index_dir_entry): An index_dir_entry structure.
+    """
+    if record is not None:
+      if self._debug:
+        self._DebugPrintStructureObject(
+          record.entry_header, self._DEBUG_INDX_ENTRY_HEADER)
+        self._DebugPrintStructureObject(
+          record.node_header, self._DEBUG_INDX_NODE_HEADER)
+        self._DebugPrintStructureObject(
+          record, self._DEBUG_INDX_DIR_RECORD)
+        self._DebugPrintStructureObject(
+          record.index_key_data, self._DEBUG_FILE_NAME_ATTR)
+
+
+  def _ParseIndexEntryHeader(self, file_object):
+    file_offset = file_object.tell()
+    data_type_map = self._GetDataTypeMap('index_entry_header')
+
+    indx_record, data_size = self._ReadStructureFromFileObject(
+        file_object, file_offset, data_type_map, 'INDX Entry Header')
+    return indx_record, data_size
+
+
+  def _ParseIndexNodeHeader(self, file_object):
+    file_offset = file_object.tell()
+    data_type_map = self._GetDataTypeMap('index_node_header')
+
+    indx_record, data_size = self._ReadStructureFromFileObject(
+        file_object, file_offset, data_type_map, 'INDX Node Header')
+    return indx_record, data_size
+
+
+  def _ParseIndexDirectoryEntry(self, file_object):
+    file_offset = file_object.tell()
+    data_type_map = self._GetDataTypeMap('index_dir_entry')
+
+    try:
+      indx_record, data_size = self._ReadStructureFromFileObject(
+          file_object, file_offset, data_type_map,
+          '4KB block with possible INDX Directory Entry')
+      return indx_record, data_size
+    except ParseError as e:
+      if self._debug:
+        print(e)
+      return None, None
+
+
+  def ReadFileObject(self, file_object):
+    """Reads a file-like object containing INDX records.
+
+    Args:
+      file_object (file): file-like object.
+
+    Raises:
+      ParseError: if the file cannot be read.
+    """
+    self._file_object = file_object
+
+  def ReadRecords(self):
+    """
+    Reads INDX records.
+
+    Yields:
+      index_dir_entry: An $I30 INDX record.
+
+    Raises:
+      ParseError: if a record cannot be read.
+    """
+    self._file_object.seek(0, os.SEEK_SET)
+    file_offset = 0
+
+    # INDX entries allocated in 4096-byte chunks
+    block_size = 4096
+
+    while file_offset < self._file_size:
+      self._file_object.seek(file_offset, os.SEEK_SET)
+      index_dir_entry, _ = self._ParseIndexDirectoryEntry(self._file_object)
+      if index_dir_entry:
+        yield index_dir_entry
+
+      file_offset += block_size

dtformats/indx_directory_entry.yml

@@ -0,0 +1,158 @@
+name: index_record
+type: format
+description: Index Directory Entry
+urls: ["https://github.com/libyal/libfsntfs/blob/83c2f4ce3d16b5535eae9de767adc93fff724004/documentation/New%20Technologies%20File%20System%20(NTFS).asciidoc#index"]
+metadata:
+  authors: ['Joachim Metz <[email protected]>', 'Juan Leaniz <[email protected]']
+  year: 2021
+---
+name: byte
+type: integer
+attributes:
+  format: unsigned
+  size: 1
+  units: bytes
+---
+name: uint16
+type: integer
+attributes:
+  format: unsigned
+  size: 2
+  units: bytes
+---
+name: uint32
+type: integer
+attributes:
+  format: unsigned
+  size: 4
+  units: bytes
+---
+name: wchar
+type: character
+attributes:
+  size: 1
+  units: bytes
+---
+name: wchar16
+type: character
+description: 16-bit wide character type
+attributes:
+  size: 2
+  units: bytes
+---
+name: wchar32
+type: character
+description: 32-bit wide character type
+attributes:
+  size: 4
+  units: bytes
+---
+name: int64
+type: integer
+description: 64-bit little-endian signed integer type
+attributes:
+  byte_order: little-endian
+  format: signed
+  size: 8
+  units: bytes
+---
+name: uint64
+type: integer
+description: 64-bit little-endian unsigned integer type
+attributes:
+  byte_order: little-endian
+  format: unsigned
+  size: 8
+  units: bytes
+---
+name: file_name_attribute
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: parent_file_reference
+  data_type: uint64
+- name: creation_time
+  data_type: uint64
+- name: modification_time
+  data_type: uint64
+- name: entry_modification_time
+  data_type: uint64
+- name: access_time
+  data_type: uint64
+- name: allocated_file_size
+  data_type: uint64
+- name: file_size
+  data_type: uint64
+- name: file_attribute_flags
+  data_type: uint32
+- name: extended_data
+  data_type: uint32
+- name: name_size
+  data_type: byte
+- name: name_space
+  data_type: byte
+- name: filename
+  type: string
+  encoding: utf-16-le
+  element_data_type: wchar16
+  number_of_elements: file_name_attribute.name_size
+  elements_terminator: "\x00\x00"
+---
+name: index_entry_header
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: signature
+  type: stream
+  element_data_type: byte
+  number_of_elements: 4
+  value: "INDX"
+- name: fixup_value_offset
+  data_type: uint16
+- name: num_fixup_values
+  data_type: uint16
+- name: logfile_sequence_number
+  data_type: int64
+- name: virtual_cluster_number
+  data_type: int64
+---
+name: index_node_header
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: index_values_offset
+  data_type: uint32
+- name: index_node_size
+  data_type: uint32
+- name: allocated_index_node_size
+  data_type: uint32
+- name: index_node_flags
+  data_type: uint32
+---
+name: index_dir_entry
+type: structure
+attributes:
+  byte_order: little-endian
+members:
+- name: entry_header
+  data_type: index_entry_header
+- name: node_header
+  data_type: index_node_header
+- name: values_offset
+  type: stream
+  element_data_type: byte
+  number_of_elements: index_dir_entry.node_header.index_values_offset - 16
+- name: file_reference
+  data_type: uint64
+- name: index_value_size
+  data_type: uint16
+- name: index_key_data_size
+  data_type: uint16
+- name: index_value_flags
+  data_type: uint32
+- name: index_key_data
+  condition: index_dir_entry.index_key_data_size > 64
+  data_type: file_name_attribute