diff --git a/README.md b/README.md index 6e4c44c..f4fe094 100755 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ This branch will probably remain seperate, as it is meant to be customized to ai ## Dependencies: -Since version v0.8.2, Python3 is required, with the following packages: +Python 2.7 or 3 with the following packages: $ sudo apt-get install liblzo2-dev $ sudo pip install python-lzo diff --git a/scripts/ubireader_display_blocks b/scripts/ubireader_display_blocks index defd10b..c2ff655 100755 --- a/scripts/ubireader_display_blocks +++ b/scripts/ubireader_display_blocks @@ -1,190 +1,25 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/scripts/ubireader_display_blocks -# (c) 2019 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -############################################################# -# Search by block parameters and display information about -# matching blocks. -############################################################# - -import os import sys -import argparse -from ubireader.ubi import ubi_base -from ubireader.ubi_io import ubi_file -from ubireader import settings -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -if __name__=='__main__': - - description = 'Search for specified blocks and display information.' - usage = """ - ubireader_display_blocks "{'block.attr': value,...}" path/to/image - Search for blocks by given parameters and display information about them. - This is block only, no volume or image information is created, which can - be used to debug file and image extraction. - Example: - "{'peb_num':[0, 1] + range(100, 102), 'ec_hdr.ec': 1, 'is_valid': True}" - This matches block.peb_num 0, 1, 100, 101, and 102 - with a block.ec_hdr.ec (erase count) of 1, that are valid PEB blocks. - For a full list of parameters check ubireader.ubi.block.description. - """ - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') +from ubireader.ui.scripts import DisplayBlocksUi +from ubireader.exceptions import UBIReaderParseError - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('block_search_params', - help=""" - Double quoted Dict of ubi.block.description attributes, which is run through eval(). - Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}" - """) - - parser.add_argument('filepath', help='File with blocks of interest.') +def main(): + ui = DisplayBlocksUi(True) + ui.usage='ubireader_display_blocks [options] filepath' if len(sys.argv) == 1: - parser.print_help() - - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - else: - parser.error('File path must be provided.') + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - if args.block_search_params: - try: - search_params = eval(args.block_search_params) - - if not isinstance(search_params, dict): - parser.error('Search Param Error: Params must be a Dict of block PEB object items:value pairs.') - - except NameError as e: - parser.error('Search Param Error: Dict key block attrs must be single quoted.') - - except Exception as e: - parser.error('Search Param Error: %s' % e) - - else: - parser.error('No search parameters given, -b arg is required.') - - - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - ubi_obj = ubi_base(ufile_obj) - blocks = [] - - for block in ubi_obj.blocks: - match = True - - for key in search_params: - b = ubi_obj.blocks[block] - - for attr in key.split('.'): - if hasattr(b, attr): - b = getattr(b, attr) - - if isinstance(search_params[key], list): - if isinstance(b, list): - for value in b: - if value in search_params[key]: - break - else: - match = False - elif b not in search_params[key]: - match = False - - elif b != search_params[key]: - match = False - break - - if match: - blocks.append(ubi_obj.blocks[block]) + func = args.pop('func') + filepath = args.pop('filepath') - print('\nBlock matches: %s' % len(blocks)) + try: + func(filepath, **args) + except UBIReaderParseError as e: + sys.exit("Error: " + str(e)) - for block in blocks: - print(block.display()) +if __name__ == "__main__": + main() diff --git a/scripts/ubireader_display_info b/scripts/ubireader_display_info index 5aec8b4..f8416a1 100755 --- a/scripts/ubireader_display_info +++ b/scripts/ubireader_display_info @@ -1,187 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os import sys -import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size -from ubireader.ubi_io import ubi_file, leb_virtual_file - - -if __name__=='__main__': - start = time.time() - description = 'Show information about UBI or UBIFS image.' - usage = 'ubireader_display_info [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-u', '--ubifs-info', action='store_true', dest='ubifs_info', - help='Get UBIFS information from inside a UBI image. (default: false)') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') +from ubireader.ui.scripts import DisplayInfoUi +from ubireader.exceptions import UBIReaderParseError - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('filepath', help='File to extract contents of.') +def main(): + ui = DisplayInfoUi(True) + ui.usage='ubireader_display_info [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) - - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - ubifs_info = args.ubifs_info - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - if filetype == UBI_EC_HDR_MAGIC: - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Display UBI info if not UBIFS request. - if not ubifs_info: - print(ubi_obj.display()) - - # Loop through found images in file. - for image in ubi_obj.images: - # Display image information if not UBIFS request. - if not ubifs_info: - print('%s' % image.display('\t')) - - # Loop through volumes in each image. - for volume in image.volumes: - # Show UBI or UBIFS info. - if not ubifs_info: - - # Display volume information. - print(image.volumes[volume].display('\t\t')) - - else: - # Get blocks associated with this volume. - vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - - # Skip volume if empty. - if not len(vol_blocks): - continue - # Create LEB backed virtual file with volume blocks. - # Necessary to prevent having to load entire UBI image - # into memory. - lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - - # Create UBIFS object and print info. - ubifs_obj = ubifs(lebv_file) - print(ubifs_obj.display()) - print(ubifs_obj.superblock_node.display('\t')) - print(ubifs_obj.master_node.display('\t')) - try: - print(ubifs_obj.master_node2.display('\t')) - except: - print('Master Node Error only one valid node.') + args = vars(ui.parser.parse_args()) + + func = args.pop('func') + filepath = args.pop('filepath') - elif filetype == UBIFS_NODE_MAGIC: - # Create UBIFS object - ubifs_obj = ubifs(ufile_obj) - print(ubifs_obj.display()) - print(ubifs_obj.superblock_node.display('\t')) - print(ubifs_obj.master_node.display('\t')) - try: - print(ubifs_obj.master_node2.display('\t')) - except: - print('Master Node Error only one valid node.') + try: + func(filepath, **args) + except UBIReaderParseError as e: + sys.exit("Error: " + str(e)) - else: - print('Something went wrong to get here.') +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/ubireader_extract_files b/scripts/ubireader_extract_files index 4ba3c17..fbc61f9 100755 --- a/scripts/ubireader_extract_files +++ b/scripts/ubireader_extract_files @@ -1,200 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os import sys -import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.output import extract_files -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -def create_output_dir(outpath): - if os.path.exists(outpath): - if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) - else: - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - - -if __name__=='__main__': - start = time.time() - description = 'Extract contents of a UBI or UBIFS image.' - usage = 'ubireader_extract_files [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-k', '--keep-permissions', action='store_true', dest='permissions', - help='Maintain file permissions, requires running as root. (default: False)') - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') +from ubireader.ui.scripts import ExtractFilesUi +from ubireader.exceptions import UBIReaderParseError - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') - - parser.add_argument('filepath', help='File to extract contents of.') +def main(): + ui = ExtractFilesUi(True) + ui.usage='ubireader_extract_files [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - if args.outpath: - outpath = args.outpath - else: - outpath = settings.output_dir - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - perms = args.permissions - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - if filetype == UBI_EC_HDR_MAGIC: - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Loop through found images in file. - for image in ubi_obj.images: - - # Create path for specific image - # In case multiple images in data - img_outpath = os.path.join(outpath, '%s' % image.image_seq) - - # Loop through volumes in each image. - for volume in image.volumes: - - # Get blocks associated with this volume. - vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - - # Create volume data output path. - vol_outpath = os.path.join(img_outpath, volume) - - # Create volume output path directory. - create_output_dir(vol_outpath) - - # Skip volume if empty. - if not len(vol_blocks): - continue - - # Create LEB backed virtual file with volume blocks. - # Necessary to prevent having to load entire UBI image - # into memory. - lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - - # Extract files from UBI image. - ubifs_obj = ubifs(lebv_file) - print('Extracting files to: %s' % vol_outpath) - extract_files(ubifs_obj, vol_outpath, perms) - - - elif filetype == UBIFS_NODE_MAGIC: - # Create UBIFS object - ubifs_obj = ubifs(ufile_obj) - - # Create directory for files. - create_output_dir(outpath) + func = args.pop('func') + filepath = args.pop('filepath') - # Extract files from UBIFS image. - print('Extracting files to: %s' % outpath) - extract_files(ubifs_obj, outpath, perms) + try: + func(filepath, **args) + except UBIReaderParseError as e: + sys.exit("Error: " + str(e)) - else: - print('Something went wrong to get here.') +if __name__ == "__main__": + main() diff --git a/scripts/ubireader_extract_images b/scripts/ubireader_extract_images index 9e7400f..6e1bb8d 100755 --- a/scripts/ubireader_extract_images +++ b/scripts/ubireader_extract_images @@ -1,169 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os import sys -import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubi_io import ubi_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size - -def create_output_dir(outpath): - if not os.path.exists(outpath): - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - - -if __name__=='__main__': - start = time.time() - description = 'Extract UBI or UBIFS images from file containing UBI data in it.' - usage = 'ubireader_extract_images [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') - - parser.add_argument('-u', '--image-type', dest='image_type', - help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') +from ubireader.ui.scripts import ExtractImagesUi +from ubireader.exceptions import UBIReaderParseError - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') - - parser.add_argument('filepath', help='File to extract contents of.') +def main(): + ui = ExtractImagesUi(True) + ui.usage='ubireader_extract_images [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if filetype != UBI_EC_HDR_MAGIC: - parser.error('File does not look like UBI data.') - - img_name = os.path.basename(path) - if args.outpath: - outpath = os.path.abspath(os.path.join(args.outpath, img_name)) - else: - outpath = os.path.join(settings.output_dir, img_name) - - if args.block_size: - block_size = args.block_size - else: - block_size = guess_peb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - if args.image_type: - image_type = args.image_type.upper() - else: - image_type = 'UBIFS' - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Loop through found images in file. - for image in ubi_obj.images: - if image_type == 'UBI': - # Create output path and open file. - img_outpath = os.path.join(outpath, 'img-%s.ubi' % image.image_seq) - create_output_dir(outpath) - f = open(img_outpath, 'wb') - - # Loop through UBI image blocks - for block in image.get_blocks(ubi_obj.blocks): - if ubi_obj.blocks[block].is_valid: - # Write block (PEB) to file - f.write(ubi_obj.file.read_block(ubi_obj.blocks[block])) + func = args.pop('func') + filepath = args.pop('filepath') - elif image_type == 'UBIFS': - # Loop through image volumes - for volume in image.volumes: - # Create output path and open file. - vol_outpath = os.path.join(outpath, 'img-%s_vol-%s.ubifs' % (image.image_seq, volume)) - create_output_dir(outpath) - f = open(vol_outpath, 'wb') + try: + func(filepath, **args) + except UBIReaderParseError as e: + sys.exit("Error: " + str(e)) - # Loop through and write volume block data (LEB) to file. - for block in image.volumes[volume].reader(ubi_obj): - f.write(block) +if __name__ == "__main__": + main() diff --git a/scripts/ubireader_list_files b/scripts/ubireader_list_files index 0571704..71009a9 100755 --- a/scripts/ubireader_list_files +++ b/scripts/ubireader_list_files @@ -1,174 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (C) Collin Mulliner based on Jason Pruitt's ubireader_extract_images -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os import sys -import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.list import list_files, copy_file -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -if __name__=='__main__': - start = time.time() - description = 'List and Extract files of a UBI or UBIFS image.' - usage = 'ubireader_list_files [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') +from ubireader.ui.scripts import ListFilesUi +from ubireader.exceptions import UBIReaderParseError - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-P', '--path', dest='listpath', - help='Path to list.') - - parser.add_argument('-C', '--copy', dest='copyfile', - help='File to Copy.') - - parser.add_argument('-D', '--copy-dest', dest='copyfiledest', - help='Copy Destination.') - - parser.add_argument('filepath', help='UBI/UBIFS image file.') +def main(): + ui = ListFilesUi(True) + ui.usage='ubireader_list_files [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - if filetype == UBI_EC_HDR_MAGIC: - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Loop through found images in file. - for image in ubi_obj.images: - - # Loop through volumes in each image. - for volume in image.volumes: - - # Get blocks associated with this volume. - vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - - # Skip volume if empty. - if not len(vol_blocks): - continue - - # Create LEB backed virtual file with volume blocks. - # Necessary to prevent having to load entire UBI image - # into memory. - lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - - # Create UBIFS object. - ubifs_obj = ubifs(lebv_file) - - if args.listpath: - list_files(ubifs_obj, args.listpath) - if args.copyfile and args.copyfiledest: - copy_file(ubifs_obj, args.copyfile, args.copyfiledest) - - elif filetype == UBIFS_NODE_MAGIC: - # Create UBIFS object - ubifs_obj = ubifs(ufile_obj) + func = args.pop('func') + filepath = args.pop('filepath') - if args.listpath: - list_files(ubifs_obj, args.listpath) - if args.copyfile and args.copyfiledest: - copy_file(ubifs_obj, args.copyfile, args.copyfiledest) + try: + func(filepath, **args) + except UBIReaderParseError as e: + sys.exit("Error: " + str(e)) - else: - print('Something went wrong to get here.') +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/ubireader_utils_info b/scripts/ubireader_utils_info index 40277b2..3ba3622 100755 --- a/scripts/ubireader_utils_info +++ b/scripts/ubireader_utils_info @@ -1,345 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os import sys -import time -import argparse -if (sys.version_info > (3, 0)): - import configparser -else: - import ConfigParser as configparser -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, PRINT_VOL_TYPE_LIST, UBI_VTBL_AUTORESIZE_FLG -from ubireader.ubifs import ubifs -from ubireader.ubifs.defines import PRINT_UBIFS_KEY_HASH, PRINT_UBIFS_COMPR -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size - -def create_output_dir(outpath): - if os.path.exists(outpath): - if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) - else: - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - - -def get_ubi_params(ubi_obj): - """Get ubi_obj utils params - - Arguments: - Obj:ubi -- UBI object. - - Returns: - Dict -- Dict keyed to volume with Dict of args and flags. - """ - ubi_flags = {'min_io_size':'-m', - 'max_bud_bytes':'-j', - 'leb_size':'-e', - 'default_compr':'-x', - 'sub_page_size':'-s', - 'fanout':'-f', - 'key_hash':'-k', - 'orph_lebs':'-p', - 'log_lebs':'-l', - 'max_leb_cnt': '-c', - 'peb_size':'-p', - 'sub_page_size':'-s', - 'vid_hdr_offset':'-O', - 'version':'-x', - 'image_seq':'-Q', - 'alignment':'-a', - 'vol_id':'-n', - 'name':'-N'} - - ubi_params = {} - ubi_args = {} - ini_params = {} - - for image in ubi_obj.images: - img_seq = image.image_seq - ubi_params[img_seq] = {} - ubi_args[img_seq] = {} - ini_params[img_seq] = {} - - for volume in image.volumes: - ubi_args[img_seq][volume] = {} - ini_params[img_seq][volume] = {} - - # Get ubinize.ini settings - ini_params[img_seq][volume]['vol_type'] = PRINT_VOL_TYPE_LIST[image.volumes[volume].vol_rec.vol_type] - - if image.volumes[volume].vol_rec.flags == UBI_VTBL_AUTORESIZE_FLG: - ini_params[img_seq][volume]['vol_flags'] = 'autoresize' - else: - ini_params[img_seq][volume]['vol_flags'] = image.volumes[volume].vol_rec.flags - - ini_params[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id - ini_params[img_seq][volume]['vol_name'] = image.volumes[volume].name.rstrip(b'\x00').decode('utf-8') - ini_params[img_seq][volume]['vol_alignment'] = image.volumes[volume].vol_rec.alignment - - ini_params[img_seq][volume]['vol_size'] = image.volumes[volume].vol_rec.reserved_pebs * ubi_obj.leb_size - - # Create file object backed by UBI blocks. - lebv_file = leb_virtual_file(ubi_obj, image.volumes[volume].get_blocks(ubi_obj.blocks)) - # Create UBIFS object - ubifs_obj = ubifs(lebv_file) - - for key, value in ubifs_obj.superblock_node: - if key == 'key_hash': - value = PRINT_UBIFS_KEY_HASH[value] - elif key == 'default_compr': - value = PRINT_UBIFS_COMPR[value] - - if key in ubi_flags: - ubi_args[img_seq][volume][key] = value - - for key, value in image.volumes[volume].vol_rec: - if key == 'name': - value = value.rstrip(b'\x00').decode('utf-8') - - if key in ubi_flags: - ubi_args[img_seq][volume][key] = value - - ubi_args[img_seq][volume]['version'] = image.version - ubi_args[img_seq][volume]['vid_hdr_offset'] = image.vid_hdr_offset - ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] - ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] - ubi_args[img_seq][volume]['image_seq'] = image.image_seq - ubi_args[img_seq][volume]['peb_size'] = ubi_obj.peb_size - ubi_args[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id - - ubi_params[img_seq][volume] = {'flags':ubi_flags, 'args':ubi_args[img_seq][volume], 'ini':ini_params[img_seq][volume]} - - return ubi_params - - -def print_ubi_params(ubi_obj): - ubi_params = get_ubi_params(ubi_obj) - for img_params in ubi_params: - for volume in ubi_params[img_params]: - ubi_flags = ubi_params[img_params][volume]['flags'] - ubi_args = ubi_params[img_params][volume]['args'] - ini_params = ubi_params[img_params][volume]['ini'] - sorted_keys = sorted(ubi_params[img_params][volume]['args']) - - print('\nVolume %s' % volume) - for key in sorted_keys: - if len(key)< 8: - name = '%s\t' % key - else: - name = key - print('\t%s\t%s %s' % (name, ubi_flags[key], ubi_args[key])) - - print('\n\t#ubinize.ini#') - print('\t[%s]' % ini_params['vol_name']) - for key in ini_params: - if key != 'name': - print('\t%s=%s' % (key, ini_params[key])) - - -def make_files(ubi, outpath): - ubi_params = get_ubi_params(ubi) - - for img_params in ubi_params: - config = configparser.ConfigParser() - img_outpath = os.path.join(outpath, 'img-%s' % img_params) - - if not os.path.exists(img_outpath): - os.mkdir(img_outpath) - - ini_path = os.path.join(img_outpath, 'img-%s.ini' % img_params) - ubi_file = os.path.join('img-%s.ubi' % img_params) - script_path = os.path.join(img_outpath, 'create_ubi_img-%s.sh' % img_params) - ubifs_files =[] - buf = '#!/bin/sh\n' - print('Writing to: %s' % script_path) - - with open(script_path, 'w') as fscr: - with open(ini_path, 'w') as fini: - print('Writing to: %s' % ini_path) - vol_idx = 0 - - for volume in ubi_params[img_params]: - ubifs_files.append(os.path.join('img-%s_%s.ubifs' % (img_params, vol_idx))) - ini_params = ubi_params[img_params][volume]['ini'] - ini_file = 'img-%s.ini' % img_params - config.add_section(volume) - config.set(volume, 'mode', 'ubi') - config.set(volume, 'image', ubifs_files[vol_idx]) - - for i in ini_params: - config.set(volume, i, str(ini_params[i])) - - ubi_flags = ubi_params[img_params][volume]['flags'] - ubi_args = ubi_params[img_params][volume]['args'] - mkfs_flags = ['min_io_size', - 'leb_size', - 'max_leb_cnt', - 'default_compr', - 'fanout', - 'key_hash', - 'orph_lebs', - 'log_lebs'] - - argstr = '' - for flag in mkfs_flags: - argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) - #leb = '%s %s' % (ubi_flags['leb_size'], ubi_args['leb_size']) - peb = '%s %s' % (ubi_flags['peb_size'], ubi_args['peb_size']) - min_io = '%s %s' % (ubi_flags['min_io_size'], ubi_args['min_io_size']) - #leb_cnt = '%s %s' % (ubi_flags['max_leb_cnt'], ubi_args['max_leb_cnt']) - vid_hdr = '%s %s' % (ubi_flags['vid_hdr_offset'], ubi_args['vid_hdr_offset']) - sub_page = '%s %s' % (ubi_flags['sub_page_size'], ubi_args['sub_page_size']) +from ubireader.ui.scripts import UtilsInfoUi +from ubireader.exceptions import UBIReaderParseError - buf += '/usr/sbin/mkfs.ubifs%s -r $%s %s\n' % (argstr, (vol_idx+1), ubifs_files[vol_idx]) - - vol_idx += 1 - - config.write(fini) - - ubinize_flags = ['peb_size', - 'min_io_size', - 'vid_hdr_offset', - 'sub_page_size', - 'version', - 'image_seq'] - - argstr = '' - for flag in ubinize_flags: - argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) - - buf += '/usr/sbin/ubinize%s -o %s %s\n' % (argstr, ubi_file, ini_file) - fscr.write(buf) - os.chmod(script_path, 0o755) - -if __name__=='__main__': - start = time.time() - description = 'Determine settings for recreating UBI image.' - usage = 'ubireader_utils_info [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-r', '--show-only', action='store_true', dest='show_only', - help='Print parameters to screen only. (default: false)') - - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') - - parser.add_argument('filepath', help='File to extract contents of.') +def main(): + ui = UtilsInfoUi(True) + ui.usage='ubireader_utils_info [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if filetype != UBI_EC_HDR_MAGIC: - parser.error('File does not look like UBI data.') - - img_name = os.path.basename(path) - if args.outpath: - outpath = os.path.abspath(os.path.join(args.outpath, img_name)) - else: - outpath = os.path.join(settings.output_dir, img_name) - - if args.block_size: - block_size = args.block_size - else: - block_size = guess_peb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Print info. - print_ubi_params(ubi_obj) + func = args.pop('func') + filepath = args.pop('filepath') - if not args.show_only: - create_output_dir(outpath) - # Create build scripts. - make_files(ubi_obj, outpath) + try: + func(filepath, **args) + except UBIReaderParseError as e: + sys.exit("Error: " + str(e)) +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/setup.py b/setup.py index eef9d99..2d47a31 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = '0.8.3' +version = '0.8.5' setup( name='ubi_reader', @@ -25,4 +25,9 @@ 'scripts/ubireader_utils_info', 'scripts/ubireader_display_blocks', ], + entry_points={ + 'console_scripts': [ + 'ubireader = ubireader.__main__:main', + ] + } ) diff --git a/ubireader/__main__.py b/ubireader/__main__.py new file mode 100644 index 0000000..8ebc2ce --- /dev/null +++ b/ubireader/__main__.py @@ -0,0 +1,33 @@ +import sys +import inspect +import argparse + +import ubireader.ui.scripts +from ubireader.exceptions import UBIReaderParseError + +def main(): + parser = argparse.ArgumentParser(description='UBI and UBIFS tools.') + subparsers = parser.add_subparsers() + + for name, obj in inspect.getmembers(sys.modules[ubireader.ui.scripts.__name__]): + if inspect.isclass(obj) and name.endswith('Ui'): + subp = obj() + subparsers.add_parser( + subp.cmd, + usage=subp.usage, + description=subp.description, + parents=[subp.parser] + ) + + args = vars(parser.parse_args()) + print(args) + func = args.pop('func') + filepath = args.pop('filepath') + try: + func(filepath, **args) + except UBIReaderParseError as e: + sys.exit("Error: " + str(e)) + + +if __name__ == "__main__": + main() diff --git a/ubireader/exceptions.py b/ubireader/exceptions.py new file mode 100644 index 0000000..df8d980 --- /dev/null +++ b/ubireader/exceptions.py @@ -0,0 +1,2 @@ +class UBIReaderParseError(Exception): + pass \ No newline at end of file diff --git a/ubireader/parsers/__init__.py b/ubireader/parsers/__init__.py new file mode 100644 index 0000000..7411b3e --- /dev/null +++ b/ubireader/parsers/__init__.py @@ -0,0 +1,111 @@ +import os + +from ubireader import settings +from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size, guess_leb_size +from ubireader.exceptions import UBIReaderParseError +from ubireader.debug import error, log +from ubireader.ubifs.defines import UBIFS_NODE_MAGIC +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC + +class ArgHandler(): + # Processing variables. + + def __init__(self, filepath, *args, **kwargs): + settings.logging_on = kwargs['log'] + settings.logging_on_verbose = kwargs['verbose'] + settings.warn_only_block_read_errors = kwargs['warn_only_block_read_errors'] + settings.ignore_block_header_errors = kwargs['ignore_block_header_errors'] + settings.uboot_fix = kwargs['uboot_fix'] + + self._filepath = filepath + if not os.path.exists(self.filepath): + raise UBIReaderParseError("File path doesn't exist.") + + if kwargs['start_offset']: + self._start_offset = kwargs['start_offset'] + + if kwargs['guess_offset']: + self._start_offset = guess_start_offset(self.filepath, kwargs['guess_offset']) + + else: + self._start_offset = guess_start_offset(self.filepath) + + self._end_offset = kwargs['end_offset'] + + + self._filetype = guess_filetype(self.filepath, self.start_offset) + if self.filetype not in [UBI_EC_HDR_MAGIC, UBIFS_NODE_MAGIC]: + raise UBIReaderParseError('Could not determine file type.') + + if kwargs['block_size']: + self._block_size = kwargs['block_size'] + else: + if self.filetype == UBI_EC_HDR_MAGIC: + self._block_size = guess_peb_size(self.filepath) + + elif self.filetype == UBIFS_NODE_MAGIC: + self._block_size = guess_leb_size(self.filepath) + + if not self.block_size: + raise UBIReaderParseError('PEB/LEB size could not be determined.') + + if 'image_type' in kwargs: + if kwargs['image_type']: + self._image_type = kwargs['image_type'].upper() + else: + self._image_type = 'UBIFS' + + if self._image_type not in ['UBI', 'UBIFS']: + raise UBIReaderParseError('Image type must be UBI or UBIFS.') + + if 'output_dir' in kwargs: + if kwargs['output_dir']: + self._output_dir = kwargs['output_dir'] + else: + self._output_dir = settings.output_dir + + if 'permissions' in kwargs: + self._permissions = kwargs['permissions'] + + @property + def output_dir(self): + return self._output_dir + + @property + def block_size(self): + return self._block_size + + @property + def start_offset(self): + return self._start_offset + + @property + def end_offset(self): + return self._end_offset + + @property + def permissions(self): + return self._permissions + + @property + def filepath(self): + return self._filepath + + @property + def filetype(self): + return self._filetype + + @property + def image_type(self): + return self._image_type + +def makedir(outpath, overwrite=True): + if os.path.exists(outpath): + if os.listdir(outpath) and not overwrite: + error(makedir, 'Fatal', 'Output directory is not empty. %s' % outpath) + else: + try: + os.makedirs(outpath, exist_ok=overwrite) + log(makedir, 'Created output path: %s' % outpath) + except Exception as e: + error(makedir, 'Fatal', '%s' % e) \ No newline at end of file diff --git a/ubireader/parsers/display_blocks.py b/ubireader/parsers/display_blocks.py new file mode 100644 index 0000000..1a4636b --- /dev/null +++ b/ubireader/parsers/display_blocks.py @@ -0,0 +1,61 @@ +from ubireader.parsers import ArgHandler +from ubireader.exceptions import UBIReaderParseError +from ubireader.ubi import ubi_base +from ubireader.ubi_io import ubi_file + +def parse(*args, **kwargs): + args = ArgHandler(*args, **kwargs) + + if 'block_search_params' in kwargs: + if kwargs['block_search_params']: + try: + search_params = eval(kwargs['block_search_params']) + + if not isinstance(search_params, dict): + raise UBIReaderParseError('Search Param Error: Params must be a Dict of block PEB object items:value pairs.') + + except NameError as e: + raise UBIReaderParseError('Search Param Error: Dict key block attrs must be single quoted.') + + except Exception as e: + raise UBIReaderParseError('Search Param Error: %s' % e) + + else: + raise UBIReaderParseError('No search parameters given, -b arg is required.') + + + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) + ubi_obj = ubi_base(ufile_obj) + blocks = [] + + for block in ubi_obj.blocks: + match = True + + for key in search_params: + b = ubi_obj.blocks[block] + + for attr in key.split('.'): + if hasattr(b, attr): + b = getattr(b, attr) + + if isinstance(search_params[key], list): + if isinstance(b, list): + for value in b: + if value in search_params[key]: + break + else: + match = False + elif b not in search_params[key]: + match = False + + elif b != search_params[key]: + match = False + break + + if match: + blocks.append(ubi_obj.blocks[block]) + + print('\nBlock matches: %s' % len(blocks)) + + for block in blocks: + print(block.display()) \ No newline at end of file diff --git a/ubireader/parsers/display_info.py b/ubireader/parsers/display_info.py new file mode 100644 index 0000000..2b72d23 --- /dev/null +++ b/ubireader/parsers/display_info.py @@ -0,0 +1,70 @@ +from ubireader.parsers import ArgHandler +from ubireader.ubi import ubi +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubifs import ubifs +from ubireader.ubifs.defines import UBIFS_NODE_MAGIC +from ubireader.ubi_io import ubi_file, leb_virtual_file + + +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) + # Create file object. + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) + + if args.filetype == UBI_EC_HDR_MAGIC: + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Display UBI info if not UBIFS request. + if args.image_type == 'UBI': + print(ubi_obj.display()) + + # Loop through found images in file. + for image in ubi_obj.images: + # Display image information if not UBIFS request. + if args.image_type == 'UBI': + print('%s' % image.display('\t')) + + # Loop through volumes in each image. + for volume in image.volumes: + # Show UBI or UBIFS info. + if args.image_type == 'UBI': + # Display volume information. + print(image.volumes[volume].display('\t\t')) + + else: + # Get blocks associated with this volume. + vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) + + # Skip volume if empty. + if not len(vol_blocks): + continue + + # Create LEB backed virtual file with volume blocks. + # Necessary to prevent having to load entire UBI image + # into memory. + lebv_file = leb_virtual_file(ubi_obj, vol_blocks) + + # Create UBIFS object and print info. + ubifs_obj = ubifs(lebv_file) + print(ubifs_obj.display()) + print(ubifs_obj.superblock_node.display('\t')) + print(ubifs_obj.master_node.display('\t')) + try: + print(ubifs_obj.master_node2.display('\t')) + except: + print('Master Node Error only one valid node.') + + elif args.filetype == UBIFS_NODE_MAGIC: + # Create UBIFS object + ubifs_obj = ubifs(ufile_obj) + print(ubifs_obj.display()) + print(ubifs_obj.superblock_node.display('\t')) + print(ubifs_obj.master_node.display('\t')) + try: + print(ubifs_obj.master_node2.display('\t')) + except: + print('Master Node Error only one valid node.') + + else: + print('Something went wrong to get here.') \ No newline at end of file diff --git a/ubireader/ubifs/output.py b/ubireader/parsers/extract_files.py old mode 100755 new mode 100644 similarity index 70% rename from ubireader/ubifs/output.py rename to ubireader/parsers/extract_files.py index f7f5250..0bd9053 --- a/ubireader/ubifs/output.py +++ b/ubireader/parsers/extract_files.py @@ -1,37 +1,83 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubifs -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - import os import struct from ubireader import settings -from ubireader.ubifs.defines import * -from ubireader.ubifs import walk -from ubireader.ubifs.misc import decompress from ubireader.debug import error, log, verbose_log +from ubireader.parsers import ArgHandler, makedir +from ubireader.ubi import ubi +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubifs import ubifs, walk +from ubireader.ubifs.misc import decompress +from ubireader.ubifs.defines import * +from ubireader.ubi_io import ubi_file, leb_virtual_file + +# Process file. +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) + + # Create file object. + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) + + if args.filetype == UBI_EC_HDR_MAGIC: + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Loop through found images in file. + for image in ubi_obj.images: + + # Create path for specific image + # In case multiple images in data + img_outpath = os.path.join(args.output_dir, '%s' % image.image_seq) + + # Loop through volumes in each image. + for volume in image.volumes: + + # Get blocks associated with this volume. + vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) + + # Create volume data output path. + vol_outpath = os.path.join(img_outpath, volume) + + # Create volume output path directory. + makedir(vol_outpath, False) + + # Skip volume if empty. + if not len(vol_blocks): + continue + + # Create LEB backed virtual file with volume blocks. + # Necessary to prevent having to load entire UBI image + # into memory. + lebv_file = leb_virtual_file(ubi_obj, vol_blocks) + + # Extract files from UBI image. + ubifs_obj = ubifs(lebv_file) + print('Extracting files to: %s' % vol_outpath) + extract(ubifs_obj, vol_outpath, args.permissions) + + + elif args.filetype == UBIFS_NODE_MAGIC: + # Create UBIFS object + ubifs_obj = ubifs(ufile_obj) + + # Create directory for files. + makedir(args.output_dir, False) + + # Extract files from UBIFS image. + print('Extracting files to: %s' % args.output_dir) + extract(ubifs_obj, args.output_dir, args.permissions) + + else: + print('Something went wrong to get here.') + def is_safe_path(basedir, path): basedir = os.path.realpath(basedir) path = os.path.realpath(os.path.join(basedir, path)) - return basedir == os.path.commonpath((basedir, path)) + return True if path.startswith(basedir) else False + -def extract_files(ubifs, out_path, perms=False): +def extract(ubifs, out_path, perms=False): """Extract UBIFS contents to_path/ Arguments: @@ -51,10 +97,10 @@ def extract_files(ubifs, out_path, perms=False): extract_dents(ubifs, inodes, dent, out_path, perms) if len(bad_blocks): - error(extract_files, 'Warn', 'Data may be missing or corrupted, bad blocks, LEB [%s]' % ','.join(map(str, bad_blocks))) + error(extract, 'Warn', 'Data may be missing or corrupted, bad blocks, LEB [%s]' % ','.join(map(str, bad_blocks))) except Exception as e: - error(extract_files, 'Error', '%s' % e) + error(extract, 'Error', '%s' % e) def extract_dents(ubifs, inodes, dent_node, path='', perms=False): @@ -65,7 +111,7 @@ def extract_dents(ubifs, inodes, dent_node, path='', perms=False): inode = inodes[dent_node.inum] if not is_safe_path(path, dent_node.name): - error(extract_dents, 'Warn', 'Path traversal attempt: %s, discarding' % (dent_node.name)) + error(extract_dents, 'Warn', 'Path traversal attempt: %s, discarding.' % (dent_node.name)) return dent_path = os.path.realpath(os.path.join(path, dent_node.name)) @@ -196,7 +242,7 @@ def _process_reg_file(ubifs, inode, path): verbose_log(_process_reg_file, 'ino num: %s, compression: %s, path: %s' % (inode['ino'].key['ino_num'], compr_type, path)) except Exception as e: - error(_process_reg_file, 'Warn', 'inode num:%s :%s' % (inode['ino'].key['ino_num'], e)) + error(_process_reg_file, 'Warn', 'inode num:%s path:%s :%s' % (inode['ino'].key['ino_num'], path, e)) # Pad end of file with \x00 if needed. if inode['ino'].size > len(buf): diff --git a/ubireader/parsers/extract_images.py b/ubireader/parsers/extract_images.py new file mode 100644 index 0000000..92e6fe1 --- /dev/null +++ b/ubireader/parsers/extract_images.py @@ -0,0 +1,48 @@ +import os + +from ubireader.parsers import ArgHandler, makedir +from ubireader.ubi import ubi +from ubireader.ubi_io import ubi_file +from ubireader import settings + + +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) + + img_name = os.path.basename(args.filepath) + if args.output_dir: + outpath = os.path.abspath(args.output_dir) + else: + outpath = settings.output_dir + + # Create file object. + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) + + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Loop through found images in file. + for image in ubi_obj.images: + if args.image_type == 'UBI': + # Create output path and open file. + img_outpath = os.path.join(outpath, f'{img_name}-{image.image_seq}.ubi') + makedir(outpath) + f = open(img_outpath, 'wb') + + # Loop through UBI image blocks + for block in image.get_blocks(ubi_obj.blocks): + if ubi_obj.blocks[block].is_valid: + # Write block (PEB) to file + f.write(ubi_obj.file.read_block(ubi_obj.blocks[block])) + + elif args.image_type == 'UBIFS': + # Loop through image volumes + for volume in image.volumes: + # Create output path and open file. + vol_outpath = os.path.join(outpath, f'{img_name}-{image.image_seq}-{volume}.ubifs') + makedir(outpath) + f = open(vol_outpath, 'wb') + + # Loop through and write volume block data (LEB) to file. + for block in image.volumes[volume].reader(ubi_obj): + f.write(block) \ No newline at end of file diff --git a/ubireader/parsers/list_files.py b/ubireader/parsers/list_files.py new file mode 100644 index 0000000..80b277d --- /dev/null +++ b/ubireader/parsers/list_files.py @@ -0,0 +1,68 @@ +from ubireader.parsers import ArgHandler +from ubireader.ubi import ubi +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubifs import ubifs +from ubireader.ubifs.list import list_files, copy_file +from ubireader.ubifs.defines import UBIFS_NODE_MAGIC +from ubireader.ubi_io import ubi_file, leb_virtual_file + +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) + + copyfile = False + if 'copyfile' in kwargs: + copyfile = kwargs['copyfile'] + + copyfiledest = None + if 'copyfiledest' in kwargs: + copyfiledest = kwargs['copyfiledest'] + + listpath = None + if 'listpath' in kwargs: + listpath = kwargs['listpath'] + + # Create file object. + ufile_obj = ubi_file(args.path, args.block_size, args.start_offset, args.end_offset) + + if args.filetype == UBI_EC_HDR_MAGIC: + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Loop through found images in file. + for image in ubi_obj.images: + + # Loop through volumes in each image. + for volume in image.volumes: + + # Get blocks associated with this volume. + vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) + + # Skip volume if empty. + if not len(vol_blocks): + continue + + # Create LEB backed virtual file with volume blocks. + # Necessary to prevent having to load entire UBI image + # into memory. + lebv_file = leb_virtual_file(ubi_obj, vol_blocks) + + # Create UBIFS object. + ubifs_obj = ubifs(lebv_file) + + if listpath: + list_files(ubifs_obj, listpath) + if copyfile and copyfiledest: + copy_file(ubifs_obj, copyfile, copyfiledest) + + elif args.filetype == UBIFS_NODE_MAGIC: + # Create UBIFS object + ubifs_obj = ubifs(ufile_obj) + + if listpath: + list_files(ubifs_obj, listpath) + + if copyfile and copyfiledest: + copy_file(ubifs_obj, copyfile, copyfiledest) + + else: + print('Something went wrong to get here.') \ No newline at end of file diff --git a/ubireader/parsers/utils_info.py b/ubireader/parsers/utils_info.py new file mode 100644 index 0000000..8c42e04 --- /dev/null +++ b/ubireader/parsers/utils_info.py @@ -0,0 +1,221 @@ +import os +import sys +if (sys.version_info > (3, 0)): + import configparser +else: + import ConfigParser as configparser + +from ubireader.parsers import ArgHandler, makedir +from ubireader.ubi import ubi +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, PRINT_VOL_TYPE_LIST, UBI_VTBL_AUTORESIZE_FLG +from ubireader.ubifs import ubifs +from ubireader.ubifs.defines import PRINT_UBIFS_KEY_HASH, PRINT_UBIFS_COMPR +from ubireader.ubi_io import ubi_file, leb_virtual_file + +def get_ubi_params(ubi_obj): + """Get ubi_obj utils params + + Arguments: + Obj:ubi -- UBI object. + + Returns: + Dict -- Dict keyed to volume with Dict of args and flags. + """ + ubi_flags = {'min_io_size':'-m', + 'max_bud_bytes':'-j', + 'leb_size':'-e', + 'default_compr':'-x', + 'sub_page_size':'-s', + 'fanout':'-f', + 'key_hash':'-k', + 'orph_lebs':'-p', + 'log_lebs':'-l', + 'max_leb_cnt': '-c', + 'peb_size':'-p', + 'sub_page_size':'-s', + 'vid_hdr_offset':'-O', + 'version':'-x', + 'image_seq':'-Q', + 'alignment':'-a', + 'vol_id':'-n', + 'name':'-N'} + + ubi_params = {} + ubi_args = {} + ini_params = {} + + for image in ubi_obj.images: + img_seq = image.image_seq + ubi_params[img_seq] = {} + ubi_args[img_seq] = {} + ini_params[img_seq] = {} + + for volume in image.volumes: + ubi_args[img_seq][volume] = {} + ini_params[img_seq][volume] = {} + + # Get ubinize.ini settings + ini_params[img_seq][volume]['vol_type'] = PRINT_VOL_TYPE_LIST[image.volumes[volume].vol_rec.vol_type] + + if image.volumes[volume].vol_rec.flags == UBI_VTBL_AUTORESIZE_FLG: + ini_params[img_seq][volume]['vol_flags'] = 'autoresize' + else: + ini_params[img_seq][volume]['vol_flags'] = image.volumes[volume].vol_rec.flags + + ini_params[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id + ini_params[img_seq][volume]['vol_name'] = image.volumes[volume].name.rstrip(b'\x00').decode('utf-8') + ini_params[img_seq][volume]['vol_alignment'] = image.volumes[volume].vol_rec.alignment + + ini_params[img_seq][volume]['vol_size'] = image.volumes[volume].vol_rec.reserved_pebs * ubi_obj.leb_size + + # Create file object backed by UBI blocks. + lebv_file = leb_virtual_file(ubi_obj, image.volumes[volume].get_blocks(ubi_obj.blocks)) + # Create UBIFS object + ubifs_obj = ubifs(lebv_file) + + for key, value in ubifs_obj.superblock_node: + if key == 'key_hash': + value = PRINT_UBIFS_KEY_HASH[value] + elif key == 'default_compr': + value = PRINT_UBIFS_COMPR[value] + + if key in ubi_flags: + ubi_args[img_seq][volume][key] = value + + for key, value in image.volumes[volume].vol_rec: + if key == 'name': + value = value.rstrip(b'\x00').decode('utf-8') + + if key in ubi_flags: + ubi_args[img_seq][volume][key] = value + + ubi_args[img_seq][volume]['version'] = image.version + ubi_args[img_seq][volume]['vid_hdr_offset'] = image.vid_hdr_offset + ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] + ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] + ubi_args[img_seq][volume]['image_seq'] = image.image_seq + ubi_args[img_seq][volume]['peb_size'] = ubi_obj.peb_size + ubi_args[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id + + ubi_params[img_seq][volume] = {'flags':ubi_flags, 'args':ubi_args[img_seq][volume], 'ini':ini_params[img_seq][volume]} + + return ubi_params + + +def print_ubi_params(ubi_obj): + ubi_params = get_ubi_params(ubi_obj) + for img_params in ubi_params: + for volume in ubi_params[img_params]: + ubi_flags = ubi_params[img_params][volume]['flags'] + ubi_args = ubi_params[img_params][volume]['args'] + ini_params = ubi_params[img_params][volume]['ini'] + sorted_keys = sorted(ubi_params[img_params][volume]['args']) + + print('\nVolume %s' % volume) + for key in sorted_keys: + if len(key)< 8: + name = '%s\t' % key + else: + name = key + print('\t%s\t%s %s' % (name, ubi_flags[key], ubi_args[key])) + + print('\n\t#ubinize.ini#') + print('\t[%s]' % ini_params['vol_name']) + for key in ini_params: + if key != 'name': + print('\t%s=%s' % (key, ini_params[key])) + + +def make_files(ubi, outpath): + ubi_params = get_ubi_params(ubi) + + for img_params in ubi_params: + config = configparser.ConfigParser() + img_outpath = os.path.join(outpath, 'img-%s' % img_params) + + if not os.path.exists(img_outpath): + os.mkdir(img_outpath) + + ini_path = os.path.join(img_outpath, 'img-%s.ini' % img_params) + ubi_file = os.path.join('img-%s.ubi' % img_params) + script_path = os.path.join(img_outpath, 'create_ubi_img-%s.sh' % img_params) + ubifs_files =[] + buf = '#!/bin/sh\n' + print('Writing to: %s' % script_path) + + with open(script_path, 'w') as fscr: + with open(ini_path, 'w') as fini: + print('Writing to: %s' % ini_path) + vol_idx = 0 + + for volume in ubi_params[img_params]: + ubifs_files.append(os.path.join('img-%s_%s.ubifs' % (img_params, vol_idx))) + ini_params = ubi_params[img_params][volume]['ini'] + ini_file = 'img-%s.ini' % img_params + config.add_section(volume) + config.set(volume, 'mode', 'ubi') + config.set(volume, 'image', ubifs_files[vol_idx]) + + for i in ini_params: + config.set(volume, i, str(ini_params[i])) + + ubi_flags = ubi_params[img_params][volume]['flags'] + ubi_args = ubi_params[img_params][volume]['args'] + mkfs_flags = ['min_io_size', + 'leb_size', + 'max_leb_cnt', + 'default_compr', + 'fanout', + 'key_hash', + 'orph_lebs', + 'log_lebs'] + + argstr = '' + for flag in mkfs_flags: + argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) + + #leb = '%s %s' % (ubi_flags['leb_size'], ubi_args['leb_size']) + peb = '%s %s' % (ubi_flags['peb_size'], ubi_args['peb_size']) + min_io = '%s %s' % (ubi_flags['min_io_size'], ubi_args['min_io_size']) + #leb_cnt = '%s %s' % (ubi_flags['max_leb_cnt'], ubi_args['max_leb_cnt']) + vid_hdr = '%s %s' % (ubi_flags['vid_hdr_offset'], ubi_args['vid_hdr_offset']) + sub_page = '%s %s' % (ubi_flags['sub_page_size'], ubi_args['sub_page_size']) + + buf += '/usr/sbin/mkfs.ubifs%s -r $%s %s\n' % (argstr, (vol_idx+1), ubifs_files[vol_idx]) + + vol_idx += 1 + + config.write(fini) + + ubinize_flags = ['peb_size', + 'min_io_size', + 'vid_hdr_offset', + 'sub_page_size', + 'version', + 'image_seq'] + + argstr = '' + for flag in ubinize_flags: + argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) + + buf += '/usr/sbin/ubinize%s -o %s %s\n' % (argstr, ubi_file, ini_file) + fscr.write(buf) + os.chmod(script_path, 0o755) + + +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) + + # Create file object. + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) + + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Print info. + print_ubi_params(ubi_obj) + + if not kwargs['show_only']: + makedir(args.output_dir) + # Create build scripts. + make_files(ubi_obj, args.output_dir) \ No newline at end of file diff --git a/ubireader/settings.py b/ubireader/settings.py index e4bc9ff..3ad9726 100755 --- a/ubireader/settings.py +++ b/ubireader/settings.py @@ -17,8 +17,6 @@ # along with this program. If not, see . ############################################################# -import os - output_dir = 'ubifs-root' error_action = True # if 'exit' on any error exit program. diff --git a/ubireader/ubi/__init__.py b/ubireader/ubi/__init__.py index 39e48fc..cf2cb88 100755 --- a/ubireader/ubi/__init__.py +++ b/ubireader/ubi/__init__.py @@ -17,13 +17,11 @@ # along with this program. If not, see . ############################################################# - -from ubireader.debug import error, log -from ubireader.ubi.block import sort, extract_blocks, rm_old_blocks -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, FILE_CHUNK_SZ +from ubireader.debug import error +from ubireader.ubi.block import sort, extract_blocks from ubireader.ubi import display from ubireader.ubi.image import description as image -from ubireader.ubi.block import layout +from ubireader.ubi.block import layout, rm_old_blocks class ubi_base(object): """UBI Base object @@ -148,22 +146,20 @@ def __init__(self, ubi_file): super(ubi, self).__init__(ubi_file) layout_list, data_list, int_vol_list, unknown_list = sort.by_type(self.blocks) - layout_list = rm_old_blocks(self.blocks, layout_list) - data_list = rm_old_blocks(self.blocks, data_list) - int_vol_list = rm_old_blocks(self.blocks, int_vol_list) - unknown_list = rm_old_blocks(self.blocks, unknown_list) - - if len(layout_list) < 2: - error(self, 'Fatal', 'Less than 2 layout blocks found.') - self._layout_blocks_list = layout.get_newest(self.blocks, layout_list) + self._layout_blocks_list = layout_list self._data_blocks_list = data_list self._int_vol_blocks_list = int_vol_list self._unknown_blocks_list = unknown_list + + newest_layout_list = rm_old_blocks(self.blocks, self.layout_blocks_list) + + if len(newest_layout_list) < 2: + error(self, 'Fatal', 'Less than 2 layout blocks found.') - layout_pairs = layout.group_pairs(self.blocks, self.layout_blocks_list) + layout_pairs = layout.group_pairs(self.blocks, newest_layout_list) - layout_infos = layout.associate_blocks(self.blocks, layout_pairs, self.first_peb_num) + layout_infos = layout.associate_blocks(self.blocks, layout_pairs) self._images = [] for i in range(0, len(layout_infos)): diff --git a/ubireader/ubi/block/__init__.py b/ubireader/ubi/block/__init__.py index defd003..ce80f96 100755 --- a/ubireader/ubi/block/__init__.py +++ b/ubireader/ubi/block/__init__.py @@ -17,7 +17,6 @@ # along with this program. If not, see . ############################################################# -import re from zlib import crc32 from ubireader import settings from ubireader.debug import error, log, verbose_display, verbose_log @@ -129,7 +128,7 @@ def extract_blocks(ubi): blk.file_offset = i blk.peb_num = ubi.first_peb_num + peb_count blk.size = ubi.file.block_size - blk.data_crc = (~crc32(buf[blk.ec_hdr.data_offset:blk.ec_hdr.data_offset+blk.vid_hdr.data_size]) & 0xFFFFFFFF) + blk.data_crc = (~crc32(buf[blk.ec_hdr.data_offset:blk.ec_hdr.data_offset+blk.vid_hdr.data_size]) & UBI_CRC32_INIT) blocks[blk.peb_num] = blk peb_count += 1 log(extract_blocks, blk) @@ -152,7 +151,7 @@ def extract_blocks(ubi): else: cur_offset += ubi.file.block_size - ubi.first_peb_num = cur_offset/ubi.file.block_size + ubi.first_peb_num = cur_offset//ubi.file.block_size ubi.file.start_offset = cur_offset return blocks @@ -162,7 +161,11 @@ def rm_old_blocks(blocks, block_list): del_blocks = [] for i in block_list: - if i in del_blocks or blocks[i].is_valid is not True: + if i in del_blocks: + continue + + if blocks[i].is_valid is not True: + del_blocks.append(i) continue for k in block_list: @@ -172,6 +175,10 @@ def rm_old_blocks(blocks, block_list): if k in del_blocks: continue + if blocks[k].is_valid is not True: + del_blocks.append(k) + continue + if blocks[i].leb_num != blocks[k].leb_num: continue @@ -179,26 +186,52 @@ def rm_old_blocks(blocks, block_list): continue second_newer = blocks[k].vid_hdr.sqnum > blocks[i].vid_hdr.sqnum - + del_block = None + use_block = None + if second_newer: if blocks[k].vid_hdr.copy_flag == 0: - log(rm_old_blocks, 'Old block removed (copy_flag): PEB %s, LEB %s' % (blocks[i].peb_num, blocks[i].leb_num)) - del_blocks.append(i) - break + del_block = i + use_block = k + else: if blocks[i].vid_hdr.copy_flag == 0: - log(rm_old_blocks, 'Old block removed (copy_flag): PEB %s, LEB %s' % (blocks[k].peb_num, blocks[k].leb_num)) - del_blocks.append(k) - break + del_block = k + use_block = i - if blocks[k].data_crc != blocks[k].vid_hdr.data_crc: - log(rm_old_blocks, 'Old block removed (data_crc): PEB %s, LEB %s' % (blocks[k].peb_num, blocks[k].leb_num)) - del_blocks.append(k) + if del_block is not None: + del_blocks.append(del_block) + log(rm_old_blocks, 'Old block removed (copy_flag): PEB %s, LEB %s, Using PEB%s' % (blocks[del_block].peb_num, blocks[del_block].leb_num, use_block)) break - elif blocks[i].data_crc != blocks[i].vid_hdr.data_crc: - log(rm_old_blocks, 'Old block removed (data_crc): PEB %s, LEB %s' % (blocks[i].peb_num, blocks[i].leb_num)) - del_blocks.append(i) - break - + if second_newer: + if blocks[k].data_crc != blocks[k].vid_hdr.data_crc: + del_block = k + use_block = i + else: + del_block = i + use_block = k + else: + if blocks[i].data_crc != blocks[i].vid_hdr.data_crc: + del_block = i + use_block = k + else: + del_block = k + use_block = i + + if del_block is not None: + del_blocks.append(del_block) + log(rm_old_blocks, 'Old block removed (data_crc): PEB %s, LEB %s, vid_hdr.data_crc %s / %s, Using PEB %s' % (blocks[del_block].peb_num, + blocks[del_block].leb_num, + blocks[del_block].vid_hdr.data_crc, + blocks[del_block].data_crc, + use_block)) + + else: + use_block = min(k, i) + del_blocks.append(use_block) + error('Warn', rm_old_blocks, 'Multiple PEB [%s] for LEB %s: Using first.' % (', '.join(i, k), blocks[i].leb_num, use_block)) + + break + return [j for j in block_list if j not in del_blocks] diff --git a/ubireader/ubi/block/layout.py b/ubireader/ubi/block/layout.py index de56ce9..67388d9 100755 --- a/ubireader/ubi/block/layout.py +++ b/ubireader/ubi/block/layout.py @@ -17,36 +17,9 @@ # along with this program. If not, see . ############################################################# -from ubireader.debug import error, log +from ubireader.debug import log from ubireader.ubi.block import sort -def get_newest(blocks, layout_blocks): - """Filter out old layout blocks from list - - Arguments: - List:blocks -- List of block objects - List:layout_blocks -- List of layout block indexes - - Returns: - List -- Newest layout blocks in list - """ - layout_temp = list(layout_blocks) - - for i in range(0, len(layout_temp)): - for k in range(0, len(layout_blocks)): - if blocks[layout_temp[i]].ec_hdr.image_seq != blocks[layout_blocks[k]].ec_hdr.image_seq: - continue - - if blocks[layout_temp[i]].leb_num != blocks[layout_blocks[k]].leb_num: - continue - - if blocks[layout_temp[i]].vid_hdr.sqnum > blocks[layout_blocks[k]].vid_hdr.sqnum: - del layout_blocks[k] - break - - return layout_blocks - - def group_pairs(blocks, layout_blocks_list): """Sort a list of layout blocks into pairs @@ -71,21 +44,21 @@ def group_pairs(blocks, layout_blocks_list): return list(image_dict.values()) -def associate_blocks(blocks, layout_pairs, start_peb_num): +def associate_blocks(blocks, layout_pairs): """Group block indexes with appropriate layout pairs Arguments: List:blocks -- List of block objects List:layout_pairs -- List of grouped layout blocks - Int:start_peb_num -- Number of the PEB to start from. Returns: List -- Layout block pairs grouped with associated block ranges. """ + seq_blocks = [] for layout_pair in layout_pairs: seq_blocks = sort.by_image_seq(blocks, blocks[layout_pair[0]].ec_hdr.image_seq) - + seq_blocks = [b for b in seq_blocks if b not in layout_pair] layout_pair.append(seq_blocks) return layout_pairs diff --git a/ubireader/ubi/block/sort.py b/ubireader/ubi/block/sort.py index eb26e62..e9e115b 100644 --- a/ubireader/ubi/block/sort.py +++ b/ubireader/ubi/block/sort.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . ############################################################# + from ubireader import settings def by_image_seq(blocks, image_seq): diff --git a/ubireader/ubi/image.py b/ubireader/ubi/image.py index 0a40c78..f634b28 100755 --- a/ubireader/ubi/image.py +++ b/ubireader/ubi/image.py @@ -38,7 +38,7 @@ def __repr__(self): def get_blocks(self, blocks): - return get_blocks_in_list(blocks, self._block_list) #range(self._start_peb, self._end_peb+1)) + return get_blocks_in_list(blocks, self._block_list) def _get_peb_range(self): diff --git a/ubireader/ubi/volume.py b/ubireader/ubi/volume.py index 633e95e..f02834b 100755 --- a/ubireader/ubi/volume.py +++ b/ubireader/ubi/volume.py @@ -19,7 +19,7 @@ from ubireader.debug import log from ubireader.ubi import display -from ubireader.ubi.block import sort, get_blocks_in_list +from ubireader.ubi.block import sort, get_blocks_in_list, rm_old_blocks class description(object): """UBI Volume object @@ -110,11 +110,12 @@ def get_volumes(blocks, layout_info): volumes = {} vol_blocks_lists = sort.by_vol_id(blocks, layout_info[2]) - for vol_rec in blocks[layout_info[0]].vtbl_recs: vol_name = vol_rec.name.strip(b'\x00').decode('utf-8') if vol_rec.rec_index not in vol_blocks_lists: vol_blocks_lists[vol_rec.rec_index] = [] + + vol_blocks_lists[vol_rec.rec_index] = rm_old_blocks(blocks, vol_blocks_lists[vol_rec.rec_index]) volumes[vol_name] = description(vol_rec.rec_index, vol_rec, vol_blocks_lists[vol_rec.rec_index]) return volumes diff --git a/ubireader/ubi_io.py b/ubireader/ubi_io.py index d1c0c13..2a03c68 100755 --- a/ubireader/ubi_io.py +++ b/ubireader/ubi_io.py @@ -17,7 +17,6 @@ # along with this program. If not, see . ############################################################# -from ubireader import settings from ubireader.debug import error, log, verbose_log from ubireader.ubi.block import sort diff --git a/ubireader/ubifs/__init__.py b/ubireader/ubifs/__init__.py index ba634a1..3e2bbca 100755 --- a/ubireader/ubifs/__init__.py +++ b/ubireader/ubifs/__init__.py @@ -16,8 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . ############################################################# -import re -import struct from ubireader.debug import error, log, verbose_display from ubireader.ubifs.defines import * diff --git a/ubireader/ubifs/display.py b/ubireader/ubifs/display.py index b7fa57e..c7241f0 100644 --- a/ubireader/ubifs/display.py +++ b/ubireader/ubifs/display.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . ############################################################# + from ubireader.ubifs.defines import PRINT_UBIFS_FLGS, PRINT_UBIFS_MST def ubifs(ubifs, tab=''): diff --git a/ubireader/ubifs/list.py b/ubireader/ubifs/list.py index 70bae65..166ea83 100755 --- a/ubireader/ubifs/list.py +++ b/ubireader/ubifs/list.py @@ -18,10 +18,7 @@ ############################################################# import os -import struct - import time -from ubireader import settings from ubireader.ubifs.defines import * from ubireader.ubifs import walk from ubireader.ubifs.misc import decompress diff --git a/ubireader/ubifs/misc.py b/ubireader/ubifs/misc.py index d99b260..c6ce0ea 100755 --- a/ubireader/ubifs/misc.py +++ b/ubireader/ubifs/misc.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . ############################################################# + import lzo import struct import zlib diff --git a/ubireader/utils.py b/ubireader/utils.py index a7dbbf3..1793285 100755 --- a/ubireader/utils.py +++ b/ubireader/utils.py @@ -60,7 +60,6 @@ def guess_start_offset(path, guess_offset=0): f.close() - def guess_filetype(path, start_offset=0): log(guess_filetype, 'Looking for file type at %s' % start_offset) @@ -82,7 +81,6 @@ def guess_filetype(path, start_offset=0): return ftype - def guess_leb_size(path): """Get LEB size from superblock @@ -127,7 +125,6 @@ def guess_leb_size(path): return block_size - def guess_peb_size(path): """Determine the most likely block size @@ -163,24 +160,24 @@ def guess_peb_size(path): file_offset += FILE_CHUNK_SZ f.close() - occurances = {} + occurrences = {} for i in range(0, len(offsets)): try: diff = offsets[i] - offsets[i-1] except: diff = offsets[i] - if diff not in occurances: - occurances[diff] = 0 + if diff not in occurrences: + occurrences[diff] = 0 - occurances[diff] += 1 + occurrences[diff] += 1 most_frequent = 0 block_size = None - for offset in occurances: - if occurances[offset] > most_frequent: - most_frequent = occurances[offset] + for offset in occurrences: + if occurrences[offset] > most_frequent: + most_frequent = occurrences[offset] block_size = offset - return block_size + return block_size \ No newline at end of file