From 25d65d53aba53cf72ec79fe4a65e4ebd3d427ea1 Mon Sep 17 00:00:00 2001 From: AT0myks Date: Mon, 28 Nov 2022 19:31:14 +0100 Subject: [PATCH 01/15] Add UBIReaderParseError exception --- ubireader/exceptions.py | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ubireader/exceptions.py diff --git a/ubireader/exceptions.py b/ubireader/exceptions.py new file mode 100644 index 0000000..df8d980 --- /dev/null +++ b/ubireader/exceptions.py @@ -0,0 +1,2 @@ +class UBIReaderParseError(Exception): + pass \ No newline at end of file From 9beb62bf2982ee782688471ef00590d2ee1ddfc7 Mon Sep 17 00:00:00 2001 From: AT0myks Date: Mon, 28 Nov 2022 19:44:34 +0100 Subject: [PATCH 02/15] Merge common arguments --- scripts/ubireader_display_blocks | 29 ---------------------------- scripts/ubireader_display_info | 29 ---------------------------- scripts/ubireader_extract_files | 29 ---------------------------- scripts/ubireader_extract_images | 29 ---------------------------- scripts/ubireader_list_files | 29 ---------------------------- scripts/ubireader_utils_info | 29 ---------------------------- ubireader/parsers/__init__.py | 33 ++++++++++++++++++++++++++++++++ 7 files changed, 33 insertions(+), 174 deletions(-) create mode 100644 ubireader/parsers/__init__.py diff --git a/scripts/ubireader_display_blocks b/scripts/ubireader_display_blocks index defd10b..2e46223 100755 --- a/scripts/ubireader_display_blocks +++ b/scripts/ubireader_display_blocks @@ -48,44 +48,15 @@ if __name__=='__main__': """ parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', help='Specify LEB size. (UBIFS Only)') - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - parser.add_argument('block_search_params', help=""" Double quoted Dict of ubi.block.description attributes, which is run through eval(). Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}" """) - parser.add_argument('filepath', help='File with blocks of interest.') - if len(sys.argv) == 1: parser.print_help() diff --git a/scripts/ubireader_display_info b/scripts/ubireader_display_info index 5aec8b4..5844588 100755 --- a/scripts/ubireader_display_info +++ b/scripts/ubireader_display_info @@ -38,41 +38,12 @@ if __name__=='__main__': usage = 'ubireader_display_info [options] filepath' parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - parser.add_argument('-u', '--ubifs-info', action='store_true', dest='ubifs_info', help='Get UBIFS information from inside a UBI image. (default: false)') - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', help='Specify LEB size. (UBIFS Only)') - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - parser.add_argument('filepath', help='File to extract contents of.') - if len(sys.argv) == 1: parser.print_help() sys.exit(1) diff --git a/scripts/ubireader_extract_files b/scripts/ubireader_extract_files index 4ba3c17..91c75f8 100755 --- a/scripts/ubireader_extract_files +++ b/scripts/ubireader_extract_files @@ -54,41 +54,12 @@ if __name__=='__main__': parser.add_argument('-k', '--keep-permissions', action='store_true', dest='permissions', help='Maintain file permissions, requires running as root. (default: False)') - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', help='Specify LEB size. (UBIFS Only)') - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - parser.add_argument('-o', '--output-dir', dest='outpath', help='Specify output directory path.') - parser.add_argument('filepath', help='File to extract contents of.') - if len(sys.argv) == 1: parser.print_help() sys.exit(1) diff --git a/scripts/ubireader_extract_images b/scripts/ubireader_extract_images index 9e7400f..5718e7c 100755 --- a/scripts/ubireader_extract_images +++ b/scripts/ubireader_extract_images @@ -45,41 +45,12 @@ if __name__=='__main__': usage = 'ubireader_extract_images [options] filepath' parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') - parser.add_argument('-u', '--image-type', dest='image_type', help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - parser.add_argument('-o', '--output-dir', dest='outpath', help='Specify output directory path.') - parser.add_argument('filepath', help='File to extract contents of.') - if len(sys.argv) == 1: parser.print_help() sys.exit(1) diff --git a/scripts/ubireader_list_files b/scripts/ubireader_list_files index 0571704..4ec3648 100755 --- a/scripts/ubireader_list_files +++ b/scripts/ubireader_list_files @@ -39,36 +39,9 @@ if __name__=='__main__': usage = 'ubireader_list_files [options] filepath' parser = argparse.ArgumentParser(usage=usage, description=description) - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size. (UBI Only)') - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', help='Specify LEB size. (UBIFS Only)') - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - parser.add_argument('-P', '--path', dest='listpath', help='Path to list.') @@ -78,8 +51,6 @@ if __name__=='__main__': parser.add_argument('-D', '--copy-dest', dest='copyfiledest', help='Copy Destination.') - parser.add_argument('filepath', help='UBI/UBIFS image file.') - if len(sys.argv) == 1: parser.print_help() sys.exit(1) diff --git a/scripts/ubireader_utils_info b/scripts/ubireader_utils_info index 40277b2..2ed83bd 100755 --- a/scripts/ubireader_utils_info +++ b/scripts/ubireader_utils_info @@ -246,38 +246,9 @@ if __name__=='__main__': parser.add_argument('-r', '--show-only', action='store_true', dest='show_only', help='Print parameters to screen only. (default: false)') - parser.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - parser.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - parser.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') - - parser.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI data in file. (default: 0)') - - parser.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI data in file.') - - parser.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - parser.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - parser.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - parser.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - parser.add_argument('-o', '--output-dir', dest='outpath', help='Specify output directory path.') - parser.add_argument('filepath', help='File to extract contents of.') - if len(sys.argv) == 1: parser.print_help() sys.exit(1) diff --git a/ubireader/parsers/__init__.py b/ubireader/parsers/__init__.py new file mode 100644 index 0000000..9aa9059 --- /dev/null +++ b/ubireader/parsers/__init__.py @@ -0,0 +1,33 @@ +import argparse + + +parser_base = argparse.ArgumentParser(add_help=False) + +parser_base.add_argument('filepath', help='UBI/UBIFS image file.') + +parser_base.add_argument('-l', '--log', action='store_true', dest='log', + help='Print extraction information to screen.') + +parser_base.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', + help='Prints nearly everything about anything to screen.') + +parser_base.add_argument('-p', '--peb-size', type=int, dest='block_size', + help='Specify PEB size.') + +parser_base.add_argument('-s', '--start-offset', type=int, dest='start_offset', + help='Specify offset of UBI/UBIFS data in file. (default: 0)') + +parser_base.add_argument('-n', '--end-offset', type=int, dest='end_offset', + help='Specify end offset of UBI/UBIFS data in file.') + +parser_base.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', + help='Specify offset to start guessing where UBI data is in file. (default: 0)') + +parser_base.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', + help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') + +parser_base.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', + help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') + +parser_base.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', + help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') \ No newline at end of file From 281ccec541460ff883c379ef9b7c44a72cf1110e Mon Sep 17 00:00:00 2001 From: AT0myks Date: Mon, 28 Nov 2022 20:08:44 +0100 Subject: [PATCH 03/15] Move scripts internals inside ubireader package Create a file for each script and move their remaining code there. Add the base parser as a parent of each moved parser. Adapt the scripts to the new layout. --- scripts/ubireader_display_blocks | 144 ++----------- scripts/ubireader_display_info | 141 ++----------- scripts/ubireader_extract_files | 152 ++------------ scripts/ubireader_extract_images | 123 ++--------- scripts/ubireader_list_files | 130 ++---------- scripts/ubireader_utils_info | 299 ++------------------------ ubireader/parsers/display_blocks.py | 152 ++++++++++++++ ubireader/parsers/display_info.py | 150 +++++++++++++ ubireader/parsers/extract_files.py | 165 +++++++++++++++ ubireader/parsers/extract_images.py | 134 ++++++++++++ ubireader/parsers/list_files.py | 140 +++++++++++++ ubireader/parsers/utils_info.py | 312 ++++++++++++++++++++++++++++ 12 files changed, 1138 insertions(+), 904 deletions(-) create mode 100644 ubireader/parsers/display_blocks.py create mode 100644 ubireader/parsers/display_info.py create mode 100644 ubireader/parsers/extract_files.py create mode 100644 ubireader/parsers/extract_images.py create mode 100644 ubireader/parsers/list_files.py create mode 100644 ubireader/parsers/utils_info.py diff --git a/scripts/ubireader_display_blocks b/scripts/ubireader_display_blocks index 2e46223..49475f3 100755 --- a/scripts/ubireader_display_blocks +++ b/scripts/ubireader_display_blocks @@ -22,140 +22,26 @@ # matching blocks. ############################################################# -import os -import sys import argparse -from ubireader.ubi import ubi_base -from ubireader.ubi_io import ubi_file -from ubireader import settings -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -if __name__=='__main__': +import sys - description = 'Search for specified blocks and display information.' - usage = """ - ubireader_display_blocks "{'block.attr': value,...}" path/to/image - Search for blocks by given parameters and display information about them. - This is block only, no volume or image information is created, which can - be used to debug file and image extraction. - Example: - "{'peb_num':[0, 1] + range(100, 102), 'ec_hdr.ec': 1, 'is_valid': True}" - This matches block.peb_num 0, 1, 100, 101, and 102 - with a block.ec_hdr.ec (erase count) of 1, that are valid PEB blocks. - For a full list of parameters check ubireader.ubi.block.description. - """ - parser = argparse.ArgumentParser(usage=usage, description=description) +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import display_blocks - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - parser.add_argument('block_search_params', - help=""" - Double quoted Dict of ubi.block.description attributes, which is run through eval(). - Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}" - """) +if __name__=='__main__': + parser = argparse.ArgumentParser( + usage=display_blocks.usage, + description=display_blocks.description, + parents=[display_blocks.parser] + ) if len(sys.argv) == 1: parser.print_help() - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - else: - parser.error('File path must be provided.') - sys.exit(1) - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - if args.block_search_params: - try: - search_params = eval(args.block_search_params) - - if not isinstance(search_params, dict): - parser.error('Search Param Error: Params must be a Dict of block PEB object items:value pairs.') - - except NameError as e: - parser.error('Search Param Error: Dict key block attrs must be single quoted.') - - except Exception as e: - parser.error('Search Param Error: %s' % e) - - else: - parser.error('No search parameters given, -b arg is required.') - - - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - ubi_obj = ubi_base(ufile_obj) - blocks = [] - - for block in ubi_obj.blocks: - match = True - - for key in search_params: - b = ubi_obj.blocks[block] - - for attr in key.split('.'): - if hasattr(b, attr): - b = getattr(b, attr) - - if isinstance(search_params[key], list): - if isinstance(b, list): - for value in b: - if value in search_params[key]: - break - else: - match = False - elif b not in search_params[key]: - match = False - - elif b != search_params[key]: - match = False - break - - if match: - blocks.append(ubi_obj.blocks[block]) - - print('\nBlock matches: %s' % len(blocks)) - - for block in blocks: - print(block.display()) + args = vars(parser.parse_args()) + func = args.pop("func") + try: + func(**args) + except UBIReaderParseError as e: + parser.error(str(e)) diff --git a/scripts/ubireader_display_info b/scripts/ubireader_display_info index 5844588..30ecf71 100755 --- a/scripts/ubireader_display_info +++ b/scripts/ubireader_display_info @@ -18,141 +18,28 @@ # along with this program. If not, see . ############################################################# -import os +import argparse import sys import time -import argparse -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size -from ubireader.ubi_io import ubi_file, leb_virtual_file +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import display_info if __name__=='__main__': - start = time.time() - description = 'Show information about UBI or UBIFS image.' - usage = 'ubireader_display_info [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-u', '--ubifs-info', action='store_true', dest='ubifs_info', - help='Get UBIFS information from inside a UBI image. (default: false)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') + parser = argparse.ArgumentParser( + usage=display_info.usage, + description=display_info.description, + parents=[display_info.parser] + ) if len(sys.argv) == 1: parser.print_help() sys.exit(1) - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - ubifs_info = args.ubifs_info - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - if filetype == UBI_EC_HDR_MAGIC: - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Display UBI info if not UBIFS request. - if not ubifs_info: - print(ubi_obj.display()) - - # Loop through found images in file. - for image in ubi_obj.images: - # Display image information if not UBIFS request. - if not ubifs_info: - print('%s' % image.display('\t')) - - # Loop through volumes in each image. - for volume in image.volumes: - # Show UBI or UBIFS info. - if not ubifs_info: - - # Display volume information. - print(image.volumes[volume].display('\t\t')) - - else: - # Get blocks associated with this volume. - vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - - # Skip volume if empty. - if not len(vol_blocks): - continue - - # Create LEB backed virtual file with volume blocks. - # Necessary to prevent having to load entire UBI image - # into memory. - lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - - # Create UBIFS object and print info. - ubifs_obj = ubifs(lebv_file) - print(ubifs_obj.display()) - print(ubifs_obj.superblock_node.display('\t')) - print(ubifs_obj.master_node.display('\t')) - try: - print(ubifs_obj.master_node2.display('\t')) - except: - print('Master Node Error only one valid node.') - - elif filetype == UBIFS_NODE_MAGIC: - # Create UBIFS object - ubifs_obj = ubifs(ufile_obj) - print(ubifs_obj.display()) - print(ubifs_obj.superblock_node.display('\t')) - print(ubifs_obj.master_node.display('\t')) - try: - print(ubifs_obj.master_node2.display('\t')) - except: - print('Master Node Error only one valid node.') - - else: - print('Something went wrong to get here.') + args = vars(parser.parse_args()) + func = args.pop("func") + try: + func(**args) + except UBIReaderParseError as e: + parser.error(str(e)) diff --git a/scripts/ubireader_extract_files b/scripts/ubireader_extract_files index 91c75f8..3873809 100755 --- a/scripts/ubireader_extract_files +++ b/scripts/ubireader_extract_files @@ -18,154 +18,28 @@ # along with this program. If not, see . ############################################################# -import os import sys import time import argparse -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.output import extract_files -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -def create_output_dir(outpath): - if os.path.exists(outpath): - if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) - else: - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import extract_files if __name__=='__main__': - start = time.time() - description = 'Extract contents of a UBI or UBIFS image.' - usage = 'ubireader_extract_files [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-k', '--keep-permissions', action='store_true', dest='permissions', - help='Maintain file permissions, requires running as root. (default: False)') - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') + parser = argparse.ArgumentParser( + usage=extract_files.usage, + description=extract_files.description, + parents=[extract_files.parser] + ) if len(sys.argv) == 1: parser.print_help() sys.exit(1) - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - if args.outpath: - outpath = args.outpath - else: - outpath = settings.output_dir - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - perms = args.permissions - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - if filetype == UBI_EC_HDR_MAGIC: - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Loop through found images in file. - for image in ubi_obj.images: - - # Create path for specific image - # In case multiple images in data - img_outpath = os.path.join(outpath, '%s' % image.image_seq) - - # Loop through volumes in each image. - for volume in image.volumes: - - # Get blocks associated with this volume. - vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - - # Create volume data output path. - vol_outpath = os.path.join(img_outpath, volume) - - # Create volume output path directory. - create_output_dir(vol_outpath) - - # Skip volume if empty. - if not len(vol_blocks): - continue - - # Create LEB backed virtual file with volume blocks. - # Necessary to prevent having to load entire UBI image - # into memory. - lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - - # Extract files from UBI image. - ubifs_obj = ubifs(lebv_file) - print('Extracting files to: %s' % vol_outpath) - extract_files(ubifs_obj, vol_outpath, perms) - - - elif filetype == UBIFS_NODE_MAGIC: - # Create UBIFS object - ubifs_obj = ubifs(ufile_obj) - - # Create directory for files. - create_output_dir(outpath) - - # Extract files from UBIFS image. - print('Extracting files to: %s' % outpath) - extract_files(ubifs_obj, outpath, perms) - - else: - print('Something went wrong to get here.') + args = vars(parser.parse_args()) + func = args.pop("func") + try: + func(**args) + except UBIReaderParseError as e: + parser.error(str(e)) diff --git a/scripts/ubireader_extract_images b/scripts/ubireader_extract_images index 5718e7c..ccefa95 100755 --- a/scripts/ubireader_extract_images +++ b/scripts/ubireader_extract_images @@ -18,123 +18,28 @@ # along with this program. If not, see . ############################################################# -import os +import argparse import sys import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubi_io import ubi_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size -def create_output_dir(outpath): - if not os.path.exists(outpath): - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import extract_images if __name__=='__main__': - start = time.time() - description = 'Extract UBI or UBIFS images from file containing UBI data in it.' - usage = 'ubireader_extract_images [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-u', '--image-type', dest='image_type', - help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') + parser = argparse.ArgumentParser( + usage=extract_images.usage, + description=extract_images.description, + parents=[extract_images.parser] + ) if len(sys.argv) == 1: parser.print_help() sys.exit(1) - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if filetype != UBI_EC_HDR_MAGIC: - parser.error('File does not look like UBI data.') - - img_name = os.path.basename(path) - if args.outpath: - outpath = os.path.abspath(os.path.join(args.outpath, img_name)) - else: - outpath = os.path.join(settings.output_dir, img_name) - - if args.block_size: - block_size = args.block_size - else: - block_size = guess_peb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - if args.image_type: - image_type = args.image_type.upper() - else: - image_type = 'UBIFS' - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Loop through found images in file. - for image in ubi_obj.images: - if image_type == 'UBI': - # Create output path and open file. - img_outpath = os.path.join(outpath, 'img-%s.ubi' % image.image_seq) - create_output_dir(outpath) - f = open(img_outpath, 'wb') - - # Loop through UBI image blocks - for block in image.get_blocks(ubi_obj.blocks): - if ubi_obj.blocks[block].is_valid: - # Write block (PEB) to file - f.write(ubi_obj.file.read_block(ubi_obj.blocks[block])) - - elif image_type == 'UBIFS': - # Loop through image volumes - for volume in image.volumes: - # Create output path and open file. - vol_outpath = os.path.join(outpath, 'img-%s_vol-%s.ubifs' % (image.image_seq, volume)) - create_output_dir(outpath) - f = open(vol_outpath, 'wb') - - # Loop through and write volume block data (LEB) to file. - for block in image.volumes[volume].reader(ubi_obj): - f.write(block) + args = vars(parser.parse_args()) + func = args.pop("func") + try: + func(**args) + except UBIReaderParseError as e: + parser.error(str(e)) diff --git a/scripts/ubireader_list_files b/scripts/ubireader_list_files index 4ec3648..0d52ad4 100755 --- a/scripts/ubireader_list_files +++ b/scripts/ubireader_list_files @@ -18,128 +18,28 @@ # along with this program. If not, see . ############################################################# -import os +import argparse import sys import time -import argparse - -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.list import list_files, copy_file -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -if __name__=='__main__': - start = time.time() - description = 'List and Extract files of a UBI or UBIFS image.' - usage = 'ubireader_list_files [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - parser.add_argument('-P', '--path', dest='listpath', - help='Path to list.') +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import list_files - parser.add_argument('-C', '--copy', dest='copyfile', - help='File to Copy.') - parser.add_argument('-D', '--copy-dest', dest='copyfiledest', - help='Copy Destination.') +if __name__=='__main__': + parser = argparse.ArgumentParser( + usage=list_files.usage, + description=list_files.description, + parents=[list_files.parser] + ) if len(sys.argv) == 1: parser.print_help() sys.exit(1) - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - parser.error('Could not determine file type.') - - if args.block_size: - block_size = args.block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - if filetype == UBI_EC_HDR_MAGIC: - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Loop through found images in file. - for image in ubi_obj.images: - - # Loop through volumes in each image. - for volume in image.volumes: - - # Get blocks associated with this volume. - vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) - - # Skip volume if empty. - if not len(vol_blocks): - continue - - # Create LEB backed virtual file with volume blocks. - # Necessary to prevent having to load entire UBI image - # into memory. - lebv_file = leb_virtual_file(ubi_obj, vol_blocks) - - # Create UBIFS object. - ubifs_obj = ubifs(lebv_file) - - if args.listpath: - list_files(ubifs_obj, args.listpath) - if args.copyfile and args.copyfiledest: - copy_file(ubifs_obj, args.copyfile, args.copyfiledest) - - elif filetype == UBIFS_NODE_MAGIC: - # Create UBIFS object - ubifs_obj = ubifs(ufile_obj) - - if args.listpath: - list_files(ubifs_obj, args.listpath) - if args.copyfile and args.copyfiledest: - copy_file(ubifs_obj, args.copyfile, args.copyfiledest) - - else: - print('Something went wrong to get here.') + args = vars(parser.parse_args()) + func = args.pop("func") + try: + func(**args) + except UBIReaderParseError as e: + parser.error(str(e)) diff --git a/scripts/ubireader_utils_info b/scripts/ubireader_utils_info index 2ed83bd..e8e4496 100755 --- a/scripts/ubireader_utils_info +++ b/scripts/ubireader_utils_info @@ -18,299 +18,28 @@ # along with this program. If not, see . ############################################################# -import os +import argparse import sys import time -import argparse -if (sys.version_info > (3, 0)): - import configparser -else: - import ConfigParser as configparser -from ubireader import settings -from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, PRINT_VOL_TYPE_LIST, UBI_VTBL_AUTORESIZE_FLG -from ubireader.ubifs import ubifs -from ubireader.ubifs.defines import PRINT_UBIFS_KEY_HASH, PRINT_UBIFS_COMPR -from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size - -def create_output_dir(outpath): - if os.path.exists(outpath): - if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) - else: - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - - -def get_ubi_params(ubi_obj): - """Get ubi_obj utils params - - Arguments: - Obj:ubi -- UBI object. - - Returns: - Dict -- Dict keyed to volume with Dict of args and flags. - """ - ubi_flags = {'min_io_size':'-m', - 'max_bud_bytes':'-j', - 'leb_size':'-e', - 'default_compr':'-x', - 'sub_page_size':'-s', - 'fanout':'-f', - 'key_hash':'-k', - 'orph_lebs':'-p', - 'log_lebs':'-l', - 'max_leb_cnt': '-c', - 'peb_size':'-p', - 'sub_page_size':'-s', - 'vid_hdr_offset':'-O', - 'version':'-x', - 'image_seq':'-Q', - 'alignment':'-a', - 'vol_id':'-n', - 'name':'-N'} - - ubi_params = {} - ubi_args = {} - ini_params = {} - - for image in ubi_obj.images: - img_seq = image.image_seq - ubi_params[img_seq] = {} - ubi_args[img_seq] = {} - ini_params[img_seq] = {} - - for volume in image.volumes: - ubi_args[img_seq][volume] = {} - ini_params[img_seq][volume] = {} - - # Get ubinize.ini settings - ini_params[img_seq][volume]['vol_type'] = PRINT_VOL_TYPE_LIST[image.volumes[volume].vol_rec.vol_type] - - if image.volumes[volume].vol_rec.flags == UBI_VTBL_AUTORESIZE_FLG: - ini_params[img_seq][volume]['vol_flags'] = 'autoresize' - else: - ini_params[img_seq][volume]['vol_flags'] = image.volumes[volume].vol_rec.flags - - ini_params[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id - ini_params[img_seq][volume]['vol_name'] = image.volumes[volume].name.rstrip(b'\x00').decode('utf-8') - ini_params[img_seq][volume]['vol_alignment'] = image.volumes[volume].vol_rec.alignment - - ini_params[img_seq][volume]['vol_size'] = image.volumes[volume].vol_rec.reserved_pebs * ubi_obj.leb_size - - # Create file object backed by UBI blocks. - lebv_file = leb_virtual_file(ubi_obj, image.volumes[volume].get_blocks(ubi_obj.blocks)) - # Create UBIFS object - ubifs_obj = ubifs(lebv_file) - - for key, value in ubifs_obj.superblock_node: - if key == 'key_hash': - value = PRINT_UBIFS_KEY_HASH[value] - elif key == 'default_compr': - value = PRINT_UBIFS_COMPR[value] - - if key in ubi_flags: - ubi_args[img_seq][volume][key] = value - - for key, value in image.volumes[volume].vol_rec: - if key == 'name': - value = value.rstrip(b'\x00').decode('utf-8') - - if key in ubi_flags: - ubi_args[img_seq][volume][key] = value - - ubi_args[img_seq][volume]['version'] = image.version - ubi_args[img_seq][volume]['vid_hdr_offset'] = image.vid_hdr_offset - ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] - ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] - ubi_args[img_seq][volume]['image_seq'] = image.image_seq - ubi_args[img_seq][volume]['peb_size'] = ubi_obj.peb_size - ubi_args[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id - - ubi_params[img_seq][volume] = {'flags':ubi_flags, 'args':ubi_args[img_seq][volume], 'ini':ini_params[img_seq][volume]} - - return ubi_params - - -def print_ubi_params(ubi_obj): - ubi_params = get_ubi_params(ubi_obj) - for img_params in ubi_params: - for volume in ubi_params[img_params]: - ubi_flags = ubi_params[img_params][volume]['flags'] - ubi_args = ubi_params[img_params][volume]['args'] - ini_params = ubi_params[img_params][volume]['ini'] - sorted_keys = sorted(ubi_params[img_params][volume]['args']) - - print('\nVolume %s' % volume) - for key in sorted_keys: - if len(key)< 8: - name = '%s\t' % key - else: - name = key - print('\t%s\t%s %s' % (name, ubi_flags[key], ubi_args[key])) - - print('\n\t#ubinize.ini#') - print('\t[%s]' % ini_params['vol_name']) - for key in ini_params: - if key != 'name': - print('\t%s=%s' % (key, ini_params[key])) - - -def make_files(ubi, outpath): - ubi_params = get_ubi_params(ubi) - - for img_params in ubi_params: - config = configparser.ConfigParser() - img_outpath = os.path.join(outpath, 'img-%s' % img_params) - - if not os.path.exists(img_outpath): - os.mkdir(img_outpath) - - ini_path = os.path.join(img_outpath, 'img-%s.ini' % img_params) - ubi_file = os.path.join('img-%s.ubi' % img_params) - script_path = os.path.join(img_outpath, 'create_ubi_img-%s.sh' % img_params) - ubifs_files =[] - buf = '#!/bin/sh\n' - print('Writing to: %s' % script_path) - with open(script_path, 'w') as fscr: - with open(ini_path, 'w') as fini: - print('Writing to: %s' % ini_path) - vol_idx = 0 +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import utils_info - for volume in ubi_params[img_params]: - ubifs_files.append(os.path.join('img-%s_%s.ubifs' % (img_params, vol_idx))) - ini_params = ubi_params[img_params][volume]['ini'] - ini_file = 'img-%s.ini' % img_params - config.add_section(volume) - config.set(volume, 'mode', 'ubi') - config.set(volume, 'image', ubifs_files[vol_idx]) - - for i in ini_params: - config.set(volume, i, str(ini_params[i])) - - ubi_flags = ubi_params[img_params][volume]['flags'] - ubi_args = ubi_params[img_params][volume]['args'] - mkfs_flags = ['min_io_size', - 'leb_size', - 'max_leb_cnt', - 'default_compr', - 'fanout', - 'key_hash', - 'orph_lebs', - 'log_lebs'] - - argstr = '' - for flag in mkfs_flags: - argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) - - #leb = '%s %s' % (ubi_flags['leb_size'], ubi_args['leb_size']) - peb = '%s %s' % (ubi_flags['peb_size'], ubi_args['peb_size']) - min_io = '%s %s' % (ubi_flags['min_io_size'], ubi_args['min_io_size']) - #leb_cnt = '%s %s' % (ubi_flags['max_leb_cnt'], ubi_args['max_leb_cnt']) - vid_hdr = '%s %s' % (ubi_flags['vid_hdr_offset'], ubi_args['vid_hdr_offset']) - sub_page = '%s %s' % (ubi_flags['sub_page_size'], ubi_args['sub_page_size']) - - buf += '/usr/sbin/mkfs.ubifs%s -r $%s %s\n' % (argstr, (vol_idx+1), ubifs_files[vol_idx]) - - vol_idx += 1 - - config.write(fini) - - ubinize_flags = ['peb_size', - 'min_io_size', - 'vid_hdr_offset', - 'sub_page_size', - 'version', - 'image_seq'] - - argstr = '' - for flag in ubinize_flags: - argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) - - buf += '/usr/sbin/ubinize%s -o %s %s\n' % (argstr, ubi_file, ini_file) - fscr.write(buf) - os.chmod(script_path, 0o755) if __name__=='__main__': - start = time.time() - description = 'Determine settings for recreating UBI image.' - usage = 'ubireader_utils_info [options] filepath' - parser = argparse.ArgumentParser(usage=usage, description=description) - - parser.add_argument('-r', '--show-only', action='store_true', dest='show_only', - help='Print parameters to screen only. (default: false)') - - parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') + parser = argparse.ArgumentParser( + usage=utils_info.usage, + description=utils_info.description, + parents=[utils_info.parser] + ) if len(sys.argv) == 1: parser.print_help() sys.exit(1) - args = parser.parse_args() - - settings.logging_on = args.log - - settings.logging_on_verbose = args.verbose - - settings.warn_only_block_read_errors = args.warn_only_block_read_errors - - settings.ignore_block_header_errors = args.ignore_block_header_errors - - settings.uboot_fix = args.uboot_fix - - if args.filepath: - path = args.filepath - if not os.path.exists(path): - parser.error("File path doesn't exist.") - - if args.start_offset: - start_offset = args.start_offset - elif args.guess_offset: - start_offset = guess_start_offset(path, args.guess_offset) - else: - start_offset = guess_start_offset(path) - - if args.end_offset: - end_offset = args.end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if filetype != UBI_EC_HDR_MAGIC: - parser.error('File does not look like UBI data.') - - img_name = os.path.basename(path) - if args.outpath: - outpath = os.path.abspath(os.path.join(args.outpath, img_name)) - else: - outpath = os.path.join(settings.output_dir, img_name) - - if args.block_size: - block_size = args.block_size - else: - block_size = guess_peb_size(path) - - if not block_size: - parser.error('Block size could not be determined.') - - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) - - # Create UBI object - ubi_obj = ubi(ufile_obj) - - # Print info. - print_ubi_params(ubi_obj) - - if not args.show_only: - create_output_dir(outpath) - # Create build scripts. - make_files(ubi_obj, outpath) - + args = vars(parser.parse_args()) + func = args.pop("func") + try: + func(**args) + except UBIReaderParseError as e: + parser.error(str(e)) diff --git a/ubireader/parsers/display_blocks.py b/ubireader/parsers/display_blocks.py new file mode 100644 index 0000000..558ef84 --- /dev/null +++ b/ubireader/parsers/display_blocks.py @@ -0,0 +1,152 @@ +import argparse +import os + +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import parser_base +from ubireader.ubi import ubi_base +from ubireader.ubi_io import ubi_file +from ubireader import settings +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubifs.defines import UBIFS_NODE_MAGIC +from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size + + +def display_blocks( + filepath, + block_search_params, + log=False, + verbose=False, + block_size=None, + start_offset=None, + end_offset=None, + guess_offset=None, + warn_only_block_read_errors=False, + ignore_block_header_errors=False, + uboot_fix=False + ): + settings.logging_on = log + + settings.logging_on_verbose = verbose + + settings.warn_only_block_read_errors = warn_only_block_read_errors + + settings.ignore_block_header_errors = ignore_block_header_errors + + settings.uboot_fix = uboot_fix + + if filepath: + path = filepath + if not os.path.exists(path): + raise UBIReaderParseError("File path doesn't exist.") + else: + raise UBIReaderParseError('File path must be provided.') + + if start_offset: + start_offset = start_offset + elif guess_offset: + start_offset = guess_start_offset(path, guess_offset) + else: + start_offset = guess_start_offset(path) + + if end_offset: + end_offset = end_offset + else: + end_offset = None + + filetype = guess_filetype(path, start_offset) + if not filetype: + raise UBIReaderParseError('Could not determine file type.') + + if block_size: + block_size = block_size + else: + if filetype == UBI_EC_HDR_MAGIC: + block_size = guess_peb_size(path) + elif filetype == UBIFS_NODE_MAGIC: + block_size = guess_leb_size(path) + + if not block_size: + raise UBIReaderParseError('Block size could not be determined.') + + if block_search_params: + try: + search_params = eval(block_search_params) + + if not isinstance(search_params, dict): + raise UBIReaderParseError('Search Param Error: Params must be a Dict of block PEB object items:value pairs.') + + except NameError as e: + raise UBIReaderParseError('Search Param Error: Dict key block attrs must be single quoted.') + + except Exception as e: + raise UBIReaderParseError('Search Param Error: %s' % e) + + else: + raise UBIReaderParseError('No search parameters given, -b arg is required.') + + + ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + ubi_obj = ubi_base(ufile_obj) + blocks = [] + + for block in ubi_obj.blocks: + match = True + + for key in search_params: + b = ubi_obj.blocks[block] + + for attr in key.split('.'): + if hasattr(b, attr): + b = getattr(b, attr) + + if isinstance(search_params[key], list): + if isinstance(b, list): + for value in b: + if value in search_params[key]: + break + else: + match = False + elif b not in search_params[key]: + match = False + + elif b != search_params[key]: + match = False + break + + if match: + blocks.append(ubi_obj.blocks[block]) + + print('\nBlock matches: %s' % len(blocks)) + + for block in blocks: + print(block.display()) + + +description = 'Search for specified blocks and display information.' +usage = """ +ubireader display-blocks "{'block.attr': value,...}" path/to/image + Search for blocks by given parameters and display information about them. + This is block only, no volume or image information is created, which can + be used to debug file and image extraction. +Example: + "{'peb_num':[0, 1] + range(100, 102), 'ec_hdr.ec': 1, 'is_valid': True}" + This matches block.peb_num 0, 1, 100, 101, and 102 + with a block.ec_hdr.ec (erase count) of 1, that are valid PEB blocks. + For a full list of parameters check ubireader.ubi.block.description. +""" +parser = argparse.ArgumentParser( + usage=usage, + description=description, + parents=[parser_base], + add_help=False +) +parser.set_defaults(func=display_blocks) + +parser.add_argument('-e', '--leb-size', type=int, dest='block_size', + help='Specify LEB size. (UBIFS Only)') + +parser.add_argument('block_search_params', + help=""" + Double quoted Dict of ubi.block.description attributes, which is run through eval(). + Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}" + """) diff --git a/ubireader/parsers/display_info.py b/ubireader/parsers/display_info.py new file mode 100644 index 0000000..7a3fcd5 --- /dev/null +++ b/ubireader/parsers/display_info.py @@ -0,0 +1,150 @@ +import argparse +import os + +from ubireader import settings +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import parser_base +from ubireader.ubi import ubi +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubifs import ubifs +from ubireader.ubifs.defines import UBIFS_NODE_MAGIC +from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size +from ubireader.ubi_io import ubi_file, leb_virtual_file + + +def display_info( + filepath, + log=False, + verbose=False, + ubifs_info=False, + block_size=None, + start_offset=None, + end_offset=None, + guess_offset=None, + warn_only_block_read_errors=False, + ignore_block_header_errors=False, + uboot_fix=False + ): + settings.logging_on = log + + settings.logging_on_verbose = verbose + + settings.warn_only_block_read_errors = warn_only_block_read_errors + + settings.ignore_block_header_errors = ignore_block_header_errors + + settings.uboot_fix = uboot_fix + + if filepath: + path = filepath + if not os.path.exists(path): + raise UBIReaderParseError("File path doesn't exist.") + + if start_offset: + start_offset = start_offset + elif guess_offset: + start_offset = guess_start_offset(path, guess_offset) + else: + start_offset = guess_start_offset(path) + + if end_offset: + end_offset = end_offset + else: + end_offset = None + + filetype = guess_filetype(path, start_offset) + if not filetype: + raise UBIReaderParseError('Could not determine file type.') + + ubifs_info = ubifs_info + + if block_size: + block_size = block_size + else: + if filetype == UBI_EC_HDR_MAGIC: + block_size = guess_peb_size(path) + elif filetype == UBIFS_NODE_MAGIC: + block_size = guess_leb_size(path) + + if not block_size: + raise UBIReaderParseError('Block size could not be determined.') + + + # Create file object. + ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + + if filetype == UBI_EC_HDR_MAGIC: + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Display UBI info if not UBIFS request. + if not ubifs_info: + print(ubi_obj.display()) + + # Loop through found images in file. + for image in ubi_obj.images: + # Display image information if not UBIFS request. + if not ubifs_info: + print('%s' % image.display('\t')) + + # Loop through volumes in each image. + for volume in image.volumes: + # Show UBI or UBIFS info. + if not ubifs_info: + + # Display volume information. + print(image.volumes[volume].display('\t\t')) + + else: + # Get blocks associated with this volume. + vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) + + # Skip volume if empty. + if not len(vol_blocks): + continue + + # Create LEB backed virtual file with volume blocks. + # Necessary to prevent having to load entire UBI image + # into memory. + lebv_file = leb_virtual_file(ubi_obj, vol_blocks) + + # Create UBIFS object and print info. + ubifs_obj = ubifs(lebv_file) + print(ubifs_obj.display()) + print(ubifs_obj.superblock_node.display('\t')) + print(ubifs_obj.master_node.display('\t')) + try: + print(ubifs_obj.master_node2.display('\t')) + except: + print('Master Node Error only one valid node.') + + elif filetype == UBIFS_NODE_MAGIC: + # Create UBIFS object + ubifs_obj = ubifs(ufile_obj) + print(ubifs_obj.display()) + print(ubifs_obj.superblock_node.display('\t')) + print(ubifs_obj.master_node.display('\t')) + try: + print(ubifs_obj.master_node2.display('\t')) + except: + print('Master Node Error only one valid node.') + + else: + print('Something went wrong to get here.') + + +description = 'Show information about UBI or UBIFS image.' +usage = 'ubireader_display_info [options] filepath' +parser = argparse.ArgumentParser( + usage=usage, + description=description, + parents=[parser_base], + add_help=False +) +parser.set_defaults(func=display_info) + +parser.add_argument('-u', '--ubifs-info', action='store_true', dest='ubifs_info', + help='Get UBIFS information from inside a UBI image. (default: false)') + +parser.add_argument('-e', '--leb-size', type=int, dest='block_size', + help='Specify LEB size. (UBIFS Only)') diff --git a/ubireader/parsers/extract_files.py b/ubireader/parsers/extract_files.py new file mode 100644 index 0000000..5b6bc1d --- /dev/null +++ b/ubireader/parsers/extract_files.py @@ -0,0 +1,165 @@ +import argparse +import os + +from ubireader import settings +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import parser_base +from ubireader.ubi import ubi +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubifs import ubifs +from ubireader.ubifs.output import extract_files as _extract_files +from ubireader.ubifs.defines import UBIFS_NODE_MAGIC +from ubireader.ubi_io import ubi_file, leb_virtual_file +from ubireader.debug import error, log +from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size + + +def create_output_dir(outpath): + if os.path.exists(outpath): + if os.listdir(outpath): + error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) + else: + try: + os.makedirs(outpath) + log(create_output_dir, 'Created output path: %s' % outpath) + except Exception as e: + error(create_output_dir, 'Fatal', '%s' % e) + + +def extract_files( + filepath, + permissions=False, + log=False, + verbose=False, + block_size=None, + start_offset=None, + end_offset=None, + guess_offset=None, + warn_only_block_read_errors=False, + ignore_block_header_errors=False, + uboot_fix=False, + outpath=None + ): + settings.logging_on = log + + settings.logging_on_verbose = verbose + + settings.warn_only_block_read_errors = warn_only_block_read_errors + + settings.ignore_block_header_errors = ignore_block_header_errors + + settings.uboot_fix = uboot_fix + + if filepath: + path = filepath + if not os.path.exists(path): + raise UBIReaderParseError("File path doesn't exist.") + + if start_offset: + start_offset = start_offset + elif guess_offset: + start_offset = guess_start_offset(path, guess_offset) + else: + start_offset = guess_start_offset(path) + + if end_offset: + end_offset = end_offset + else: + end_offset = None + + filetype = guess_filetype(path, start_offset) + if not filetype: + raise UBIReaderParseError('Could not determine file type.') + + if outpath: + outpath = outpath + else: + outpath = settings.output_dir + + if block_size: + block_size = block_size + else: + if filetype == UBI_EC_HDR_MAGIC: + block_size = guess_peb_size(path) + elif filetype == UBIFS_NODE_MAGIC: + block_size = guess_leb_size(path) + + if not block_size: + raise UBIReaderParseError('Block size could not be determined.') + + perms = permissions + + # Create file object. + ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + + if filetype == UBI_EC_HDR_MAGIC: + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Loop through found images in file. + for image in ubi_obj.images: + + # Create path for specific image + # In case multiple images in data + img_outpath = os.path.join(outpath, '%s' % image.image_seq) + + # Loop through volumes in each image. + for volume in image.volumes: + + # Get blocks associated with this volume. + vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) + + # Create volume data output path. + vol_outpath = os.path.join(img_outpath, volume) + + # Create volume output path directory. + create_output_dir(vol_outpath) + + # Skip volume if empty. + if not len(vol_blocks): + continue + + # Create LEB backed virtual file with volume blocks. + # Necessary to prevent having to load entire UBI image + # into memory. + lebv_file = leb_virtual_file(ubi_obj, vol_blocks) + + # Extract files from UBI image. + ubifs_obj = ubifs(lebv_file) + print('Extracting files to: %s' % vol_outpath) + _extract_files(ubifs_obj, vol_outpath, perms) + + + elif filetype == UBIFS_NODE_MAGIC: + # Create UBIFS object + ubifs_obj = ubifs(ufile_obj) + + # Create directory for files. + create_output_dir(outpath) + + # Extract files from UBIFS image. + print('Extracting files to: %s' % outpath) + _extract_files(ubifs_obj, outpath, perms) + + else: + print('Something went wrong to get here.') + + +description = 'Extract contents of a UBI or UBIFS image.' +usage = 'ubireader_extract_files [options] filepath' +parser = argparse.ArgumentParser( + usage=usage, + description=description, + parents=[parser_base], + add_help=False +) +parser.set_defaults(func=extract_files) + +parser.add_argument('-k', '--keep-permissions', action='store_true', dest='permissions', + help='Maintain file permissions, requires running as root. (default: False)') + +parser.add_argument('-e', '--leb-size', type=int, dest='block_size', + help='Specify LEB size. (UBIFS Only)') + +parser.add_argument('-o', '--output-dir', dest='outpath', + help='Specify output directory path.') diff --git a/ubireader/parsers/extract_images.py b/ubireader/parsers/extract_images.py new file mode 100644 index 0000000..81901de --- /dev/null +++ b/ubireader/parsers/extract_images.py @@ -0,0 +1,134 @@ +import argparse +import os + +from ubireader import settings +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import parser_base +from ubireader.ubi import ubi +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubi_io import ubi_file +from ubireader.debug import error, log +from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size + + +def create_output_dir(outpath): + if not os.path.exists(outpath): + try: + os.makedirs(outpath) + log(create_output_dir, 'Created output path: %s' % outpath) + except Exception as e: + error(create_output_dir, 'Fatal', '%s' % e) + + +def extract_images( + filepath, + log=False, + verbose=False, + block_size=None, + image_type=None, + start_offset=None, + end_offset=None, + guess_offset=None, + warn_only_block_read_errors=False, + ignore_block_header_errors=False, + uboot_fix=False, + outpath=None + ): + settings.logging_on = log + + settings.logging_on_verbose = verbose + + settings.warn_only_block_read_errors = warn_only_block_read_errors + + settings.ignore_block_header_errors = ignore_block_header_errors + + settings.uboot_fix = uboot_fix + + if filepath: + path = filepath + if not os.path.exists(path): + raise UBIReaderParseError("File path doesn't exist.") + + if start_offset: + start_offset = start_offset + elif guess_offset: + start_offset = guess_start_offset(path, guess_offset) + else: + start_offset = guess_start_offset(path) + + if end_offset: + end_offset = end_offset + else: + end_offset = None + + filetype = guess_filetype(path, start_offset) + if filetype != UBI_EC_HDR_MAGIC: + raise UBIReaderParseError('File does not look like UBI data.') + + img_name = os.path.basename(path) + if outpath: + outpath = os.path.abspath(os.path.join(outpath, img_name)) + else: + outpath = os.path.join(settings.output_dir, img_name) + + if block_size: + block_size = block_size + else: + block_size = guess_peb_size(path) + + if not block_size: + raise UBIReaderParseError('Block size could not be determined.') + + if image_type: + image_type = image_type.upper() + else: + image_type = 'UBIFS' + + # Create file object. + ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Loop through found images in file. + for image in ubi_obj.images: + if image_type == 'UBI': + # Create output path and open file. + img_outpath = os.path.join(outpath, 'img-%s.ubi' % image.image_seq) + create_output_dir(outpath) + f = open(img_outpath, 'wb') + + # Loop through UBI image blocks + for block in image.get_blocks(ubi_obj.blocks): + if ubi_obj.blocks[block].is_valid: + # Write block (PEB) to file + f.write(ubi_obj.file.read_block(ubi_obj.blocks[block])) + + elif image_type == 'UBIFS': + # Loop through image volumes + for volume in image.volumes: + # Create output path and open file. + vol_outpath = os.path.join(outpath, 'img-%s_vol-%s.ubifs' % (image.image_seq, volume)) + create_output_dir(outpath) + f = open(vol_outpath, 'wb') + + # Loop through and write volume block data (LEB) to file. + for block in image.volumes[volume].reader(ubi_obj): + f.write(block) + + +description = 'Extract UBI or UBIFS images from file containing UBI data in it.' +usage = 'ubireader_extract_images [options] filepath' +parser = argparse.ArgumentParser( + usage=usage, + description=description, + parents=[parser_base], + add_help=False +) +parser.set_defaults(func=extract_images) + +parser.add_argument('-u', '--image-type', dest='image_type', + help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') + +parser.add_argument('-o', '--output-dir', dest='outpath', + help='Specify output directory path.') diff --git a/ubireader/parsers/list_files.py b/ubireader/parsers/list_files.py new file mode 100644 index 0000000..3af2d02 --- /dev/null +++ b/ubireader/parsers/list_files.py @@ -0,0 +1,140 @@ +import argparse +import os + +from ubireader import settings +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import parser_base +from ubireader.ubi import ubi +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +from ubireader.ubifs import ubifs +from ubireader.ubifs.list import list_files, copy_file +from ubireader.ubifs.defines import UBIFS_NODE_MAGIC +from ubireader.ubi_io import ubi_file, leb_virtual_file +from ubireader.debug import error, log +from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size + + +def list_files( + filepath, + log=False, + verbose=False, + block_size=None, + start_offset=None, + end_offset=None, + guess_offset=None, + warn_only_block_read_errors=False, + ignore_block_header_errors=False, + uboot_fix=False, + listpath=None, + copyfile=None, + copyfiledest=None + ): + settings.logging_on = log + + settings.logging_on_verbose = verbose + + settings.warn_only_block_read_errors = warn_only_block_read_errors + + settings.ignore_block_header_errors = ignore_block_header_errors + + settings.uboot_fix = uboot_fix + + if filepath: + path = filepath + if not os.path.exists(path): + raise UBIReaderParseError("File path doesn't exist.") + + if start_offset: + start_offset = start_offset + elif guess_offset: + start_offset = guess_start_offset(path, guess_offset) + else: + start_offset = guess_start_offset(path) + + if end_offset: + end_offset = end_offset + else: + end_offset = None + + filetype = guess_filetype(path, start_offset) + if not filetype: + raise UBIReaderParseError('Could not determine file type.') + + if block_size: + block_size = block_size + else: + if filetype == UBI_EC_HDR_MAGIC: + block_size = guess_peb_size(path) + elif filetype == UBIFS_NODE_MAGIC: + block_size = guess_leb_size(path) + + if not block_size: + raise UBIReaderParseError('Block size could not be determined.') + + # Create file object. + ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + + if filetype == UBI_EC_HDR_MAGIC: + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Loop through found images in file. + for image in ubi_obj.images: + + # Loop through volumes in each image. + for volume in image.volumes: + + # Get blocks associated with this volume. + vol_blocks = image.volumes[volume].get_blocks(ubi_obj.blocks) + + # Skip volume if empty. + if not len(vol_blocks): + continue + + # Create LEB backed virtual file with volume blocks. + # Necessary to prevent having to load entire UBI image + # into memory. + lebv_file = leb_virtual_file(ubi_obj, vol_blocks) + + # Create UBIFS object. + ubifs_obj = ubifs(lebv_file) + + if listpath: + list_files(ubifs_obj, listpath) + if copyfile and copyfiledest: + copy_file(ubifs_obj, copyfile, copyfiledest) + + elif filetype == UBIFS_NODE_MAGIC: + # Create UBIFS object + ubifs_obj = ubifs(ufile_obj) + + if listpath: + list_files(ubifs_obj, listpath) + if copyfile and copyfiledest: + copy_file(ubifs_obj, copyfile, copyfiledest) + + else: + print('Something went wrong to get here.') + + +description = 'List and Extract files of a UBI or UBIFS image.' +usage = 'ubireader_list_files [options] filepath' +parser = argparse.ArgumentParser( + usage=usage, + description=description, + parents=[parser_base], + add_help=False +) +parser.set_defaults(func=list_files) + +parser.add_argument('-e', '--leb-size', type=int, dest='block_size', + help='Specify LEB size. (UBIFS Only)') + +parser.add_argument('-P', '--path', dest='listpath', + help='Path to list.') + +parser.add_argument('-C', '--copy', dest='copyfile', + help='File to Copy.') + +parser.add_argument('-D', '--copy-dest', dest='copyfiledest', + help='Copy Destination.') diff --git a/ubireader/parsers/utils_info.py b/ubireader/parsers/utils_info.py new file mode 100644 index 0000000..2b9c1eb --- /dev/null +++ b/ubireader/parsers/utils_info.py @@ -0,0 +1,312 @@ +import argparse +import os +import sys +if (sys.version_info > (3, 0)): + import configparser +else: + import ConfigParser as configparser + +from ubireader import settings +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import parser_base +from ubireader.ubi import ubi +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, PRINT_VOL_TYPE_LIST, UBI_VTBL_AUTORESIZE_FLG +from ubireader.ubifs import ubifs +from ubireader.ubifs.defines import PRINT_UBIFS_KEY_HASH, PRINT_UBIFS_COMPR +from ubireader.ubi_io import ubi_file, leb_virtual_file +from ubireader.debug import error, log +from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size + + +def create_output_dir(outpath): + if os.path.exists(outpath): + if os.listdir(outpath): + error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) + else: + try: + os.makedirs(outpath) + log(create_output_dir, 'Created output path: %s' % outpath) + except Exception as e: + error(create_output_dir, 'Fatal', '%s' % e) + + +def get_ubi_params(ubi_obj): + """Get ubi_obj utils params + + Arguments: + Obj:ubi -- UBI object. + + Returns: + Dict -- Dict keyed to volume with Dict of args and flags. + """ + ubi_flags = {'min_io_size':'-m', + 'max_bud_bytes':'-j', + 'leb_size':'-e', + 'default_compr':'-x', + 'sub_page_size':'-s', + 'fanout':'-f', + 'key_hash':'-k', + 'orph_lebs':'-p', + 'log_lebs':'-l', + 'max_leb_cnt': '-c', + 'peb_size':'-p', + 'sub_page_size':'-s', + 'vid_hdr_offset':'-O', + 'version':'-x', + 'image_seq':'-Q', + 'alignment':'-a', + 'vol_id':'-n', + 'name':'-N'} + + ubi_params = {} + ubi_args = {} + ini_params = {} + + for image in ubi_obj.images: + img_seq = image.image_seq + ubi_params[img_seq] = {} + ubi_args[img_seq] = {} + ini_params[img_seq] = {} + + for volume in image.volumes: + ubi_args[img_seq][volume] = {} + ini_params[img_seq][volume] = {} + + # Get ubinize.ini settings + ini_params[img_seq][volume]['vol_type'] = PRINT_VOL_TYPE_LIST[image.volumes[volume].vol_rec.vol_type] + + if image.volumes[volume].vol_rec.flags == UBI_VTBL_AUTORESIZE_FLG: + ini_params[img_seq][volume]['vol_flags'] = 'autoresize' + else: + ini_params[img_seq][volume]['vol_flags'] = image.volumes[volume].vol_rec.flags + + ini_params[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id + ini_params[img_seq][volume]['vol_name'] = image.volumes[volume].name.rstrip(b'\x00').decode('utf-8') + ini_params[img_seq][volume]['vol_alignment'] = image.volumes[volume].vol_rec.alignment + + ini_params[img_seq][volume]['vol_size'] = image.volumes[volume].vol_rec.reserved_pebs * ubi_obj.leb_size + + # Create file object backed by UBI blocks. + lebv_file = leb_virtual_file(ubi_obj, image.volumes[volume].get_blocks(ubi_obj.blocks)) + # Create UBIFS object + ubifs_obj = ubifs(lebv_file) + + for key, value in ubifs_obj.superblock_node: + if key == 'key_hash': + value = PRINT_UBIFS_KEY_HASH[value] + elif key == 'default_compr': + value = PRINT_UBIFS_COMPR[value] + + if key in ubi_flags: + ubi_args[img_seq][volume][key] = value + + for key, value in image.volumes[volume].vol_rec: + if key == 'name': + value = value.rstrip(b'\x00').decode('utf-8') + + if key in ubi_flags: + ubi_args[img_seq][volume][key] = value + + ubi_args[img_seq][volume]['version'] = image.version + ubi_args[img_seq][volume]['vid_hdr_offset'] = image.vid_hdr_offset + ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] + ubi_args[img_seq][volume]['sub_page_size'] = ubi_args[img_seq][volume]['vid_hdr_offset'] + ubi_args[img_seq][volume]['image_seq'] = image.image_seq + ubi_args[img_seq][volume]['peb_size'] = ubi_obj.peb_size + ubi_args[img_seq][volume]['vol_id'] = image.volumes[volume].vol_id + + ubi_params[img_seq][volume] = {'flags':ubi_flags, 'args':ubi_args[img_seq][volume], 'ini':ini_params[img_seq][volume]} + + return ubi_params + + +def print_ubi_params(ubi_obj): + ubi_params = get_ubi_params(ubi_obj) + for img_params in ubi_params: + for volume in ubi_params[img_params]: + ubi_flags = ubi_params[img_params][volume]['flags'] + ubi_args = ubi_params[img_params][volume]['args'] + ini_params = ubi_params[img_params][volume]['ini'] + sorted_keys = sorted(ubi_params[img_params][volume]['args']) + + print('\nVolume %s' % volume) + for key in sorted_keys: + if len(key)< 8: + name = '%s\t' % key + else: + name = key + print('\t%s\t%s %s' % (name, ubi_flags[key], ubi_args[key])) + + print('\n\t#ubinize.ini#') + print('\t[%s]' % ini_params['vol_name']) + for key in ini_params: + if key != 'name': + print('\t%s=%s' % (key, ini_params[key])) + + +def make_files(ubi, outpath): + ubi_params = get_ubi_params(ubi) + + for img_params in ubi_params: + config = configparser.ConfigParser() + img_outpath = os.path.join(outpath, 'img-%s' % img_params) + + if not os.path.exists(img_outpath): + os.mkdir(img_outpath) + + ini_path = os.path.join(img_outpath, 'img-%s.ini' % img_params) + ubi_file = os.path.join('img-%s.ubi' % img_params) + script_path = os.path.join(img_outpath, 'create_ubi_img-%s.sh' % img_params) + ubifs_files =[] + buf = '#!/bin/sh\n' + print('Writing to: %s' % script_path) + + with open(script_path, 'w') as fscr: + with open(ini_path, 'w') as fini: + print('Writing to: %s' % ini_path) + vol_idx = 0 + + for volume in ubi_params[img_params]: + ubifs_files.append(os.path.join('img-%s_%s.ubifs' % (img_params, vol_idx))) + ini_params = ubi_params[img_params][volume]['ini'] + ini_file = 'img-%s.ini' % img_params + config.add_section(volume) + config.set(volume, 'mode', 'ubi') + config.set(volume, 'image', ubifs_files[vol_idx]) + + for i in ini_params: + config.set(volume, i, str(ini_params[i])) + + ubi_flags = ubi_params[img_params][volume]['flags'] + ubi_args = ubi_params[img_params][volume]['args'] + mkfs_flags = ['min_io_size', + 'leb_size', + 'max_leb_cnt', + 'default_compr', + 'fanout', + 'key_hash', + 'orph_lebs', + 'log_lebs'] + + argstr = '' + for flag in mkfs_flags: + argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) + + #leb = '%s %s' % (ubi_flags['leb_size'], ubi_args['leb_size']) + peb = '%s %s' % (ubi_flags['peb_size'], ubi_args['peb_size']) + min_io = '%s %s' % (ubi_flags['min_io_size'], ubi_args['min_io_size']) + #leb_cnt = '%s %s' % (ubi_flags['max_leb_cnt'], ubi_args['max_leb_cnt']) + vid_hdr = '%s %s' % (ubi_flags['vid_hdr_offset'], ubi_args['vid_hdr_offset']) + sub_page = '%s %s' % (ubi_flags['sub_page_size'], ubi_args['sub_page_size']) + + buf += '/usr/sbin/mkfs.ubifs%s -r $%s %s\n' % (argstr, (vol_idx+1), ubifs_files[vol_idx]) + + vol_idx += 1 + + config.write(fini) + + ubinize_flags = ['peb_size', + 'min_io_size', + 'vid_hdr_offset', + 'sub_page_size', + 'version', + 'image_seq'] + + argstr = '' + for flag in ubinize_flags: + argstr += ' %s %s' % (ubi_flags[flag], ubi_args[flag]) + + buf += '/usr/sbin/ubinize%s -o %s %s\n' % (argstr, ubi_file, ini_file) + fscr.write(buf) + os.chmod(script_path, 0o755) + + +def utils_info( + filepath, + show_only=False, + log=False, + verbose=False, + block_size=None, + start_offset=None, + end_offset=None, + guess_offset=None, + warn_only_block_read_errors=False, + ignore_block_header_errors=False, + uboot_fix=False, + outpath=None + ): + settings.logging_on = log + + settings.logging_on_verbose = verbose + + settings.warn_only_block_read_errors = warn_only_block_read_errors + + settings.ignore_block_header_errors = ignore_block_header_errors + + settings.uboot_fix = uboot_fix + + if filepath: + path = filepath + if not os.path.exists(path): + raise UBIReaderParseError("File path doesn't exist.") + + if start_offset: + start_offset = start_offset + elif guess_offset: + start_offset = guess_start_offset(path, guess_offset) + else: + start_offset = guess_start_offset(path) + + if end_offset: + end_offset = end_offset + else: + end_offset = None + + filetype = guess_filetype(path, start_offset) + if filetype != UBI_EC_HDR_MAGIC: + raise UBIReaderParseError('File does not look like UBI data.') + + img_name = os.path.basename(path) + if outpath: + outpath = os.path.abspath(os.path.join(outpath, img_name)) + else: + outpath = os.path.join(settings.output_dir, img_name) + + if block_size: + block_size = block_size + else: + block_size = guess_peb_size(path) + + if not block_size: + raise UBIReaderParseError('Block size could not be determined.') + + # Create file object. + ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + + # Create UBI object + ubi_obj = ubi(ufile_obj) + + # Print info. + print_ubi_params(ubi_obj) + + if not show_only: + create_output_dir(outpath) + # Create build scripts. + make_files(ubi_obj, outpath) + + +description = 'Determine settings for recreating UBI image.' +usage = 'ubireader_utils_info [options] filepath' +parser = argparse.ArgumentParser( + usage=usage, + description=description, + parents=[parser_base], + add_help=False +) +parser.set_defaults(func=utils_info) + +parser.add_argument('-r', '--show-only', action='store_true', dest='show_only', + help='Print parameters to screen only. (default: false)') + +parser.add_argument('-o', '--output-dir', dest='outpath', + help='Specify output directory path.') From ac8c17e48e2511a23d2055a1ef59c9686d7fc060 Mon Sep 17 00:00:00 2001 From: AT0myks Date: Mon, 28 Nov 2022 20:10:06 +0100 Subject: [PATCH 04/15] Add main CLI --- ubireader/__main__.py | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 ubireader/__main__.py diff --git a/ubireader/__main__.py b/ubireader/__main__.py new file mode 100644 index 0000000..45b54a7 --- /dev/null +++ b/ubireader/__main__.py @@ -0,0 +1,57 @@ +import argparse +import sys + +from ubireader.exceptions import UBIReaderParseError +from ubireader.parsers import display_blocks, display_info, extract_files, extract_images, list_files, utils_info + + +def main(): + parser = argparse.ArgumentParser(description='Extract files from UBI and UBIFS images.') + subparsers = parser.add_subparsers() + subparsers.add_parser( + "display-blocks", + usage=display_blocks.usage, + description=display_blocks.description, + parents=[display_blocks.parser] + ) + subparsers.add_parser( + "display-info", + usage=display_info.usage, + description=display_info.description, + parents=[display_info.parser] + ) + subparsers.add_parser( + "extract-files", + usage=extract_files.usage, + description=extract_files.description, + parents=[extract_files.parser] + ) + subparsers.add_parser( + "extract-images", + usage=extract_images.usage, + description=extract_images.description, + parents=[extract_images.parser] + ) + subparsers.add_parser( + "list-files", + usage=list_files.usage, + description=list_files.description, + parents=[list_files.parser] + ) + subparsers.add_parser( + "utils-info", + usage=utils_info.usage, + description=utils_info.description, + parents=[utils_info.parser] + ) + + args = vars(parser.parse_args()) + func = args.pop("func") + try: + func(**args) + except UBIReaderParseError as e: + sys.exit("Error: " + str(e)) + + +if __name__ == "__main__": + main() From 75d63d2f087007b8e30a3d24d9bcbbaa895c59c1 Mon Sep 17 00:00:00 2001 From: AT0myks Date: Mon, 28 Nov 2022 20:10:35 +0100 Subject: [PATCH 05/15] Update setup.py --- setup.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup.py b/setup.py index eef9d99..048f11c 100755 --- a/setup.py +++ b/setup.py @@ -25,4 +25,9 @@ 'scripts/ubireader_utils_info', 'scripts/ubireader_display_blocks', ], + entry_points={ + 'console_scripts': [ + 'ubireader = __main__:main', + ] + } ) From 99c052fbea04f5ec820638f1c8f966b5d70b6d3d Mon Sep 17 00:00:00 2001 From: AT0myks Date: Mon, 28 Nov 2022 20:25:18 +0100 Subject: [PATCH 06/15] Fix usage display for new CLI --- ubireader/__main__.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/ubireader/__main__.py b/ubireader/__main__.py index 45b54a7..163fa07 100644 --- a/ubireader/__main__.py +++ b/ubireader/__main__.py @@ -5,42 +5,47 @@ from ubireader.parsers import display_blocks, display_info, extract_files, extract_images, list_files, utils_info +def correct_usage(text): + """Adapt old usage text to new format.""" + return text.replace('_', ' ', 1).replace('_', '-') + + def main(): parser = argparse.ArgumentParser(description='Extract files from UBI and UBIFS images.') subparsers = parser.add_subparsers() subparsers.add_parser( "display-blocks", - usage=display_blocks.usage, + usage=correct_usage(display_blocks.usage), description=display_blocks.description, parents=[display_blocks.parser] ) subparsers.add_parser( "display-info", - usage=display_info.usage, + usage=correct_usage(display_info.usage), description=display_info.description, parents=[display_info.parser] ) subparsers.add_parser( "extract-files", - usage=extract_files.usage, + usage=correct_usage(extract_files.usage), description=extract_files.description, parents=[extract_files.parser] ) subparsers.add_parser( "extract-images", - usage=extract_images.usage, + usage=correct_usage(extract_images.usage), description=extract_images.description, parents=[extract_images.parser] ) subparsers.add_parser( "list-files", - usage=list_files.usage, + usage=correct_usage(list_files.usage), description=list_files.description, parents=[list_files.parser] ) subparsers.add_parser( "utils-info", - usage=utils_info.usage, + usage=correct_usage(utils_info.usage), description=utils_info.description, parents=[utils_info.parser] ) From c258981918d2b0398bef83c5cc3cca27236a1e1c Mon Sep 17 00:00:00 2001 From: Jason Pruitt Date: Thu, 1 Dec 2022 10:48:16 -0800 Subject: [PATCH 07/15] Replace missed crc32 hex with define variable --- ubireader/ubi/block/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubireader/ubi/block/__init__.py b/ubireader/ubi/block/__init__.py index defd003..12c49ca 100755 --- a/ubireader/ubi/block/__init__.py +++ b/ubireader/ubi/block/__init__.py @@ -129,7 +129,7 @@ def extract_blocks(ubi): blk.file_offset = i blk.peb_num = ubi.first_peb_num + peb_count blk.size = ubi.file.block_size - blk.data_crc = (~crc32(buf[blk.ec_hdr.data_offset:blk.ec_hdr.data_offset+blk.vid_hdr.data_size]) & 0xFFFFFFFF) + blk.data_crc = (~crc32(buf[blk.ec_hdr.data_offset:blk.ec_hdr.data_offset+blk.vid_hdr.data_size]) & UBI_CRC32_INIT) blocks[blk.peb_num] = blk peb_count += 1 log(extract_blocks, blk) From 6fb201e4281dcedf4e5a1ab3169c33ca5f26c42c Mon Sep 17 00:00:00 2001 From: Jason Pruitt Date: Thu, 1 Dec 2022 13:44:15 -0800 Subject: [PATCH 08/15] Fixed PEB block nums being floating point. --- ubireader/ubi/block/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ubireader/ubi/block/__init__.py b/ubireader/ubi/block/__init__.py index 12c49ca..bb369d3 100755 --- a/ubireader/ubi/block/__init__.py +++ b/ubireader/ubi/block/__init__.py @@ -152,7 +152,7 @@ def extract_blocks(ubi): else: cur_offset += ubi.file.block_size - ubi.first_peb_num = cur_offset/ubi.file.block_size + ubi.first_peb_num = cur_offset//ubi.file.block_size ubi.file.start_offset = cur_offset return blocks From 85ce0b3661b0dd1f7371afa752be1857e83b2e27 Mon Sep 17 00:00:00 2001 From: Jason Pruitt Date: Sat, 3 Dec 2022 16:26:00 -0800 Subject: [PATCH 09/15] Fix rm old blocks, py2/3 extract the same blocks now. --- setup.py | 2 +- ubireader/ubi/__init__.py | 8 +--- ubireader/ubi/block/__init__.py | 66 +++++++++++++++++++++++++-------- ubireader/ubi/block/layout.py | 8 ++-- ubireader/ubi/image.py | 2 +- ubireader/ubi/volume.py | 5 ++- 6 files changed, 61 insertions(+), 30 deletions(-) diff --git a/setup.py b/setup.py index eef9d99..75c2d61 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = '0.8.3' +version = '0.8.4' setup( name='ubi_reader', diff --git a/ubireader/ubi/__init__.py b/ubireader/ubi/__init__.py index 39e48fc..e8d97e5 100755 --- a/ubireader/ubi/__init__.py +++ b/ubireader/ubi/__init__.py @@ -19,7 +19,7 @@ from ubireader.debug import error, log -from ubireader.ubi.block import sort, extract_blocks, rm_old_blocks +from ubireader.ubi.block import sort, extract_blocks from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, FILE_CHUNK_SZ from ubireader.ubi import display from ubireader.ubi.image import description as image @@ -148,10 +148,6 @@ def __init__(self, ubi_file): super(ubi, self).__init__(ubi_file) layout_list, data_list, int_vol_list, unknown_list = sort.by_type(self.blocks) - layout_list = rm_old_blocks(self.blocks, layout_list) - data_list = rm_old_blocks(self.blocks, data_list) - int_vol_list = rm_old_blocks(self.blocks, int_vol_list) - unknown_list = rm_old_blocks(self.blocks, unknown_list) if len(layout_list) < 2: error(self, 'Fatal', 'Less than 2 layout blocks found.') @@ -163,7 +159,7 @@ def __init__(self, ubi_file): layout_pairs = layout.group_pairs(self.blocks, self.layout_blocks_list) - layout_infos = layout.associate_blocks(self.blocks, layout_pairs, self.first_peb_num) + layout_infos = layout.associate_blocks(self.blocks, layout_pairs) self._images = [] for i in range(0, len(layout_infos)): diff --git a/ubireader/ubi/block/__init__.py b/ubireader/ubi/block/__init__.py index bb369d3..510c04a 100755 --- a/ubireader/ubi/block/__init__.py +++ b/ubireader/ubi/block/__init__.py @@ -162,7 +162,11 @@ def rm_old_blocks(blocks, block_list): del_blocks = [] for i in block_list: - if i in del_blocks or blocks[i].is_valid is not True: + if i in del_blocks: + continue + + if blocks[i].is_valid is not True: + del_blocks.append(i) continue for k in block_list: @@ -172,6 +176,10 @@ def rm_old_blocks(blocks, block_list): if k in del_blocks: continue + if blocks[k].is_valid is not True: + del_blocks.append(k) + continue + if blocks[i].leb_num != blocks[k].leb_num: continue @@ -179,26 +187,52 @@ def rm_old_blocks(blocks, block_list): continue second_newer = blocks[k].vid_hdr.sqnum > blocks[i].vid_hdr.sqnum - + del_block = None + use_block = None + if second_newer: if blocks[k].vid_hdr.copy_flag == 0: - log(rm_old_blocks, 'Old block removed (copy_flag): PEB %s, LEB %s' % (blocks[i].peb_num, blocks[i].leb_num)) - del_blocks.append(i) - break + del_block = i + use_block = k + else: if blocks[i].vid_hdr.copy_flag == 0: - log(rm_old_blocks, 'Old block removed (copy_flag): PEB %s, LEB %s' % (blocks[k].peb_num, blocks[k].leb_num)) - del_blocks.append(k) - break + del_block = k + use_block = i - if blocks[k].data_crc != blocks[k].vid_hdr.data_crc: - log(rm_old_blocks, 'Old block removed (data_crc): PEB %s, LEB %s' % (blocks[k].peb_num, blocks[k].leb_num)) - del_blocks.append(k) + if del_block is not None: + del_blocks.append(del_block) + log(rm_old_blocks, 'Old block removed (copy_flag): PEB %s, LEB %s, Using PEB%s' % (blocks[del_block].peb_num, blocks[del_block].leb_num, use_block)) break - elif blocks[i].data_crc != blocks[i].vid_hdr.data_crc: - log(rm_old_blocks, 'Old block removed (data_crc): PEB %s, LEB %s' % (blocks[i].peb_num, blocks[i].leb_num)) - del_blocks.append(i) - break - + if second_newer: + if blocks[k].data_crc != blocks[k].vid_hdr.data_crc: + del_block = k + use_block = i + else: + del_block = i + use_block = k + else: + if blocks[i].data_crc != blocks[i].vid_hdr.data_crc: + del_block = i + use_block = k + else: + del_block = k + use_block = i + + if del_block is not None: + del_blocks.append(del_block) + log(rm_old_blocks, 'Old block removed (data_crc): PEB %s, LEB %s, vid_hdr.data_crc %s / %s, Using PEB %s' % (blocks[del_block].peb_num, + blocks[del_block].leb_num, + blocks[del_block].vid_hdr.data_crc, + blocks[del_block].data_crc, + use_block)) + + else: + use_block = min(k, i) + del_blocks.append(use_block) + error('Warn', rm_old_blocks, 'Multiple PEB [%s] for LEB %s: Using first.' % (', '.join(i, k), blocks[i].leb_num, use_block)) + + break + return [j for j in block_list if j not in del_blocks] diff --git a/ubireader/ubi/block/layout.py b/ubireader/ubi/block/layout.py index de56ce9..8d7103d 100755 --- a/ubireader/ubi/block/layout.py +++ b/ubireader/ubi/block/layout.py @@ -30,8 +30,8 @@ def get_newest(blocks, layout_blocks): Returns: List -- Newest layout blocks in list """ - layout_temp = list(layout_blocks) + layout_temp = list(layout_blocks) for i in range(0, len(layout_temp)): for k in range(0, len(layout_blocks)): if blocks[layout_temp[i]].ec_hdr.image_seq != blocks[layout_blocks[k]].ec_hdr.image_seq: @@ -71,21 +71,21 @@ def group_pairs(blocks, layout_blocks_list): return list(image_dict.values()) -def associate_blocks(blocks, layout_pairs, start_peb_num): +def associate_blocks(blocks, layout_pairs): """Group block indexes with appropriate layout pairs Arguments: List:blocks -- List of block objects List:layout_pairs -- List of grouped layout blocks - Int:start_peb_num -- Number of the PEB to start from. Returns: List -- Layout block pairs grouped with associated block ranges. """ + seq_blocks = [] for layout_pair in layout_pairs: seq_blocks = sort.by_image_seq(blocks, blocks[layout_pair[0]].ec_hdr.image_seq) - + seq_blocks = [b for b in seq_blocks if b not in layout_pair] layout_pair.append(seq_blocks) return layout_pairs diff --git a/ubireader/ubi/image.py b/ubireader/ubi/image.py index 0a40c78..f634b28 100755 --- a/ubireader/ubi/image.py +++ b/ubireader/ubi/image.py @@ -38,7 +38,7 @@ def __repr__(self): def get_blocks(self, blocks): - return get_blocks_in_list(blocks, self._block_list) #range(self._start_peb, self._end_peb+1)) + return get_blocks_in_list(blocks, self._block_list) def _get_peb_range(self): diff --git a/ubireader/ubi/volume.py b/ubireader/ubi/volume.py index 633e95e..f02834b 100755 --- a/ubireader/ubi/volume.py +++ b/ubireader/ubi/volume.py @@ -19,7 +19,7 @@ from ubireader.debug import log from ubireader.ubi import display -from ubireader.ubi.block import sort, get_blocks_in_list +from ubireader.ubi.block import sort, get_blocks_in_list, rm_old_blocks class description(object): """UBI Volume object @@ -110,11 +110,12 @@ def get_volumes(blocks, layout_info): volumes = {} vol_blocks_lists = sort.by_vol_id(blocks, layout_info[2]) - for vol_rec in blocks[layout_info[0]].vtbl_recs: vol_name = vol_rec.name.strip(b'\x00').decode('utf-8') if vol_rec.rec_index not in vol_blocks_lists: vol_blocks_lists[vol_rec.rec_index] = [] + + vol_blocks_lists[vol_rec.rec_index] = rm_old_blocks(blocks, vol_blocks_lists[vol_rec.rec_index]) volumes[vol_name] = description(vol_rec.rec_index, vol_rec, vol_blocks_lists[vol_rec.rec_index]) return volumes From 3a2612b8ba0f04581ebe56bdef3e3f8385f303cc Mon Sep 17 00:00:00 2001 From: Jason Pruitt Date: Sat, 3 Dec 2022 16:58:07 -0800 Subject: [PATCH 10/15] Fix py2 commonpath issue, import clean up. --- ubireader/settings.py | 2 -- ubireader/ubi/__init__.py | 3 +-- ubireader/ubi/block/__init__.py | 1 - ubireader/ubi/block/layout.py | 2 +- ubireader/ubi/block/sort.py | 1 + ubireader/ubi_io.py | 1 - ubireader/ubifs/__init__.py | 2 -- ubireader/ubifs/display.py | 1 + ubireader/ubifs/list.py | 2 -- ubireader/ubifs/misc.py | 1 + ubireader/ubifs/output.py | 5 +++-- 11 files changed, 8 insertions(+), 13 deletions(-) diff --git a/ubireader/settings.py b/ubireader/settings.py index e4bc9ff..3ad9726 100755 --- a/ubireader/settings.py +++ b/ubireader/settings.py @@ -17,8 +17,6 @@ # along with this program. If not, see . ############################################################# -import os - output_dir = 'ubifs-root' error_action = True # if 'exit' on any error exit program. diff --git a/ubireader/ubi/__init__.py b/ubireader/ubi/__init__.py index e8d97e5..5f999cb 100755 --- a/ubireader/ubi/__init__.py +++ b/ubireader/ubi/__init__.py @@ -17,8 +17,7 @@ # along with this program. If not, see . ############################################################# - -from ubireader.debug import error, log +from ubireader.debug import error from ubireader.ubi.block import sort, extract_blocks from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, FILE_CHUNK_SZ from ubireader.ubi import display diff --git a/ubireader/ubi/block/__init__.py b/ubireader/ubi/block/__init__.py index 510c04a..ce80f96 100755 --- a/ubireader/ubi/block/__init__.py +++ b/ubireader/ubi/block/__init__.py @@ -17,7 +17,6 @@ # along with this program. If not, see . ############################################################# -import re from zlib import crc32 from ubireader import settings from ubireader.debug import error, log, verbose_display, verbose_log diff --git a/ubireader/ubi/block/layout.py b/ubireader/ubi/block/layout.py index 8d7103d..316596b 100755 --- a/ubireader/ubi/block/layout.py +++ b/ubireader/ubi/block/layout.py @@ -17,7 +17,7 @@ # along with this program. If not, see . ############################################################# -from ubireader.debug import error, log +from ubireader.debug import log from ubireader.ubi.block import sort def get_newest(blocks, layout_blocks): diff --git a/ubireader/ubi/block/sort.py b/ubireader/ubi/block/sort.py index eb26e62..e9e115b 100644 --- a/ubireader/ubi/block/sort.py +++ b/ubireader/ubi/block/sort.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . ############################################################# + from ubireader import settings def by_image_seq(blocks, image_seq): diff --git a/ubireader/ubi_io.py b/ubireader/ubi_io.py index d1c0c13..2a03c68 100755 --- a/ubireader/ubi_io.py +++ b/ubireader/ubi_io.py @@ -17,7 +17,6 @@ # along with this program. If not, see . ############################################################# -from ubireader import settings from ubireader.debug import error, log, verbose_log from ubireader.ubi.block import sort diff --git a/ubireader/ubifs/__init__.py b/ubireader/ubifs/__init__.py index ba634a1..3e2bbca 100755 --- a/ubireader/ubifs/__init__.py +++ b/ubireader/ubifs/__init__.py @@ -16,8 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . ############################################################# -import re -import struct from ubireader.debug import error, log, verbose_display from ubireader.ubifs.defines import * diff --git a/ubireader/ubifs/display.py b/ubireader/ubifs/display.py index b7fa57e..c7241f0 100644 --- a/ubireader/ubifs/display.py +++ b/ubireader/ubifs/display.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . ############################################################# + from ubireader.ubifs.defines import PRINT_UBIFS_FLGS, PRINT_UBIFS_MST def ubifs(ubifs, tab=''): diff --git a/ubireader/ubifs/list.py b/ubireader/ubifs/list.py index 70bae65..0bd2237 100755 --- a/ubireader/ubifs/list.py +++ b/ubireader/ubifs/list.py @@ -18,8 +18,6 @@ ############################################################# import os -import struct - import time from ubireader import settings from ubireader.ubifs.defines import * diff --git a/ubireader/ubifs/misc.py b/ubireader/ubifs/misc.py index d99b260..c6ce0ea 100755 --- a/ubireader/ubifs/misc.py +++ b/ubireader/ubifs/misc.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . ############################################################# + import lzo import struct import zlib diff --git a/ubireader/ubifs/output.py b/ubireader/ubifs/output.py index f7f5250..da7d438 100755 --- a/ubireader/ubifs/output.py +++ b/ubireader/ubifs/output.py @@ -29,7 +29,8 @@ def is_safe_path(basedir, path): basedir = os.path.realpath(basedir) path = os.path.realpath(os.path.join(basedir, path)) - return basedir == os.path.commonpath((basedir, path)) + return True if path.startswith(basedir) else False + def extract_files(ubifs, out_path, perms=False): """Extract UBIFS contents to_path/ @@ -196,7 +197,7 @@ def _process_reg_file(ubifs, inode, path): verbose_log(_process_reg_file, 'ino num: %s, compression: %s, path: %s' % (inode['ino'].key['ino_num'], compr_type, path)) except Exception as e: - error(_process_reg_file, 'Warn', 'inode num:%s :%s' % (inode['ino'].key['ino_num'], e)) + error(_process_reg_file, 'Warn', 'inode num:%s path:%s :%s' % (inode['ino'].key['ino_num'], path, e)) # Pad end of file with \x00 if needed. if inode['ino'].size > len(buf): From fbf48211c0ecdc83ad3c8fb671137dee6a69b191 Mon Sep 17 00:00:00 2001 From: Jason Pruitt Date: Sat, 3 Dec 2022 18:04:22 -0800 Subject: [PATCH 11/15] Replace layout.get_newest() with rm_old_blocks() --- ubireader/ubi/__init__.py | 14 ++++++++------ ubireader/ubi/block/layout.py | 27 --------------------------- 2 files changed, 8 insertions(+), 33 deletions(-) diff --git a/ubireader/ubi/__init__.py b/ubireader/ubi/__init__.py index 5f999cb..22a0da4 100755 --- a/ubireader/ubi/__init__.py +++ b/ubireader/ubi/__init__.py @@ -22,7 +22,7 @@ from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, FILE_CHUNK_SZ from ubireader.ubi import display from ubireader.ubi.image import description as image -from ubireader.ubi.block import layout +from ubireader.ubi.block import layout, rm_old_blocks class ubi_base(object): """UBI Base object @@ -148,15 +148,17 @@ def __init__(self, ubi_file): layout_list, data_list, int_vol_list, unknown_list = sort.by_type(self.blocks) - if len(layout_list) < 2: - error(self, 'Fatal', 'Less than 2 layout blocks found.') - - self._layout_blocks_list = layout.get_newest(self.blocks, layout_list) + self._layout_blocks_list = layout_list self._data_blocks_list = data_list self._int_vol_blocks_list = int_vol_list self._unknown_blocks_list = unknown_list + + newest_layout_list = rm_old_blocks(self.blocks, self.layout_blocks_list) + + if len(newest_layout_list) < 2: + error(self, 'Fatal', 'Less than 2 layout blocks found.') - layout_pairs = layout.group_pairs(self.blocks, self.layout_blocks_list) + layout_pairs = layout.group_pairs(self.blocks, newest_layout_list) layout_infos = layout.associate_blocks(self.blocks, layout_pairs) diff --git a/ubireader/ubi/block/layout.py b/ubireader/ubi/block/layout.py index 316596b..67388d9 100755 --- a/ubireader/ubi/block/layout.py +++ b/ubireader/ubi/block/layout.py @@ -20,33 +20,6 @@ from ubireader.debug import log from ubireader.ubi.block import sort -def get_newest(blocks, layout_blocks): - """Filter out old layout blocks from list - - Arguments: - List:blocks -- List of block objects - List:layout_blocks -- List of layout block indexes - - Returns: - List -- Newest layout blocks in list - """ - - layout_temp = list(layout_blocks) - for i in range(0, len(layout_temp)): - for k in range(0, len(layout_blocks)): - if blocks[layout_temp[i]].ec_hdr.image_seq != blocks[layout_blocks[k]].ec_hdr.image_seq: - continue - - if blocks[layout_temp[i]].leb_num != blocks[layout_blocks[k]].leb_num: - continue - - if blocks[layout_temp[i]].vid_hdr.sqnum > blocks[layout_blocks[k]].vid_hdr.sqnum: - del layout_blocks[k] - break - - return layout_blocks - - def group_pairs(blocks, layout_blocks_list): """Sort a list of layout blocks into pairs From cbae1179f8df5ca48aee12e08660b5b3bfa726c0 Mon Sep 17 00:00:00 2001 From: Jason Pruitt Date: Sat, 3 Dec 2022 19:25:51 -0800 Subject: [PATCH 12/15] General clean up, unused imports, text fixes. --- ubireader/ubi/__init__.py | 1 - ubireader/ubifs/list.py | 1 - ubireader/ubifs/output.py | 2 +- ubireader/utils.py | 19 ++++++++----------- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/ubireader/ubi/__init__.py b/ubireader/ubi/__init__.py index 22a0da4..cf2cb88 100755 --- a/ubireader/ubi/__init__.py +++ b/ubireader/ubi/__init__.py @@ -19,7 +19,6 @@ from ubireader.debug import error from ubireader.ubi.block import sort, extract_blocks -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, FILE_CHUNK_SZ from ubireader.ubi import display from ubireader.ubi.image import description as image from ubireader.ubi.block import layout, rm_old_blocks diff --git a/ubireader/ubifs/list.py b/ubireader/ubifs/list.py index 0bd2237..166ea83 100755 --- a/ubireader/ubifs/list.py +++ b/ubireader/ubifs/list.py @@ -19,7 +19,6 @@ import os import time -from ubireader import settings from ubireader.ubifs.defines import * from ubireader.ubifs import walk from ubireader.ubifs.misc import decompress diff --git a/ubireader/ubifs/output.py b/ubireader/ubifs/output.py index da7d438..b4aa653 100755 --- a/ubireader/ubifs/output.py +++ b/ubireader/ubifs/output.py @@ -66,7 +66,7 @@ def extract_dents(ubifs, inodes, dent_node, path='', perms=False): inode = inodes[dent_node.inum] if not is_safe_path(path, dent_node.name): - error(extract_dents, 'Warn', 'Path traversal attempt: %s, discarding' % (dent_node.name)) + error(extract_dents, 'Warn', 'Path traversal attempt: %s, discarding.' % (dent_node.name)) return dent_path = os.path.realpath(os.path.join(path, dent_node.name)) diff --git a/ubireader/utils.py b/ubireader/utils.py index a7dbbf3..1793285 100755 --- a/ubireader/utils.py +++ b/ubireader/utils.py @@ -60,7 +60,6 @@ def guess_start_offset(path, guess_offset=0): f.close() - def guess_filetype(path, start_offset=0): log(guess_filetype, 'Looking for file type at %s' % start_offset) @@ -82,7 +81,6 @@ def guess_filetype(path, start_offset=0): return ftype - def guess_leb_size(path): """Get LEB size from superblock @@ -127,7 +125,6 @@ def guess_leb_size(path): return block_size - def guess_peb_size(path): """Determine the most likely block size @@ -163,24 +160,24 @@ def guess_peb_size(path): file_offset += FILE_CHUNK_SZ f.close() - occurances = {} + occurrences = {} for i in range(0, len(offsets)): try: diff = offsets[i] - offsets[i-1] except: diff = offsets[i] - if diff not in occurances: - occurances[diff] = 0 + if diff not in occurrences: + occurrences[diff] = 0 - occurances[diff] += 1 + occurrences[diff] += 1 most_frequent = 0 block_size = None - for offset in occurances: - if occurances[offset] > most_frequent: - most_frequent = occurances[offset] + for offset in occurrences: + if occurrences[offset] > most_frequent: + most_frequent = occurrences[offset] block_size = offset - return block_size + return block_size \ No newline at end of file From 26c41bd1b63417e2eef07db2281b21f9a94d0fab Mon Sep 17 00:00:00 2001 From: Jason Pruitt Date: Sat, 3 Dec 2022 20:25:59 -0800 Subject: [PATCH 13/15] Update readme and setup.py --- README.md | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e4c44c..f4fe094 100755 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ This branch will probably remain seperate, as it is meant to be customized to ai ## Dependencies: -Since version v0.8.2, Python3 is required, with the following packages: +Python 2.7 or 3 with the following packages: $ sudo apt-get install liblzo2-dev $ sudo pip install python-lzo diff --git a/setup.py b/setup.py index 75c2d61..05c786c 100755 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -version = '0.8.4' +version = '0.8.5' setup( name='ubi_reader', From 584522938003afd31d38cd6ca76750f56cfa23d8 Mon Sep 17 00:00:00 2001 From: Jason Pruitt Date: Mon, 5 Dec 2022 01:31:03 -0800 Subject: [PATCH 14/15] Working on further consolidating CLI code. --- scripts/ubireader_display_blocks | 6 +- scripts/ubireader_display_info | 1 - scripts/ubireader_extract_files | 7 +- scripts/ubireader_extract_images | 7 +- scripts/ubireader_list_files | 1 - scripts/ubireader_utils_info | 1 - setup.py | 2 +- ubireader/__main__.py | 18 +-- ubireader/parsers/__init__.py | 216 +++++++++++++++++++++++++--- ubireader/parsers/display_blocks.py | 173 ++++++++-------------- ubireader/parsers/extract_files.py | 138 ++++-------------- ubireader/parsers/extract_images.py | 126 ++++------------ 12 files changed, 336 insertions(+), 360 deletions(-) diff --git a/scripts/ubireader_display_blocks b/scripts/ubireader_display_blocks index 49475f3..1efb923 100755 --- a/scripts/ubireader_display_blocks +++ b/scripts/ubireader_display_blocks @@ -31,9 +31,9 @@ from ubireader.parsers import display_blocks if __name__=='__main__': parser = argparse.ArgumentParser( - usage=display_blocks.usage, - description=display_blocks.description, - parents=[display_blocks.parser] + usage=display_blocks.ui.usage, + description=display_blocks.ui.description, + parents=[display_blocks.ui.parser] ) if len(sys.argv) == 1: diff --git a/scripts/ubireader_display_info b/scripts/ubireader_display_info index 30ecf71..b0c5ee4 100755 --- a/scripts/ubireader_display_info +++ b/scripts/ubireader_display_info @@ -20,7 +20,6 @@ import argparse import sys -import time from ubireader.exceptions import UBIReaderParseError from ubireader.parsers import display_info diff --git a/scripts/ubireader_extract_files b/scripts/ubireader_extract_files index 3873809..74775be 100755 --- a/scripts/ubireader_extract_files +++ b/scripts/ubireader_extract_files @@ -19,7 +19,6 @@ ############################################################# import sys -import time import argparse from ubireader.exceptions import UBIReaderParseError @@ -28,9 +27,9 @@ from ubireader.parsers import extract_files if __name__=='__main__': parser = argparse.ArgumentParser( - usage=extract_files.usage, - description=extract_files.description, - parents=[extract_files.parser] + usage=extract_files.ui.usage, + description=extract_files.ui.description, + parents=[extract_files.ui.parser] ) if len(sys.argv) == 1: diff --git a/scripts/ubireader_extract_images b/scripts/ubireader_extract_images index ccefa95..8129cd5 100755 --- a/scripts/ubireader_extract_images +++ b/scripts/ubireader_extract_images @@ -20,7 +20,6 @@ import argparse import sys -import time from ubireader.exceptions import UBIReaderParseError from ubireader.parsers import extract_images @@ -28,9 +27,9 @@ from ubireader.parsers import extract_images if __name__=='__main__': parser = argparse.ArgumentParser( - usage=extract_images.usage, - description=extract_images.description, - parents=[extract_images.parser] + usage=extract_images.ui.usage, + description=extract_images.ui.description, + parents=[extract_images.ui.parser] ) if len(sys.argv) == 1: diff --git a/scripts/ubireader_list_files b/scripts/ubireader_list_files index 0d52ad4..373eb84 100755 --- a/scripts/ubireader_list_files +++ b/scripts/ubireader_list_files @@ -20,7 +20,6 @@ import argparse import sys -import time from ubireader.exceptions import UBIReaderParseError from ubireader.parsers import list_files diff --git a/scripts/ubireader_utils_info b/scripts/ubireader_utils_info index e8e4496..4912ec4 100755 --- a/scripts/ubireader_utils_info +++ b/scripts/ubireader_utils_info @@ -20,7 +20,6 @@ import argparse import sys -import time from ubireader.exceptions import UBIReaderParseError from ubireader.parsers import utils_info diff --git a/setup.py b/setup.py index f7a8176..2d47a31 100755 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ ], entry_points={ 'console_scripts': [ - 'ubireader = __main__:main', + 'ubireader = ubireader.__main__:main', ] } ) diff --git a/ubireader/__main__.py b/ubireader/__main__.py index 163fa07..d363850 100644 --- a/ubireader/__main__.py +++ b/ubireader/__main__.py @@ -15,9 +15,9 @@ def main(): subparsers = parser.add_subparsers() subparsers.add_parser( "display-blocks", - usage=correct_usage(display_blocks.usage), - description=display_blocks.description, - parents=[display_blocks.parser] + usage=correct_usage(display_blocks.ui.usage), + description=display_blocks.ui.description, + parents=[display_blocks.ui.parser] ) subparsers.add_parser( "display-info", @@ -27,15 +27,15 @@ def main(): ) subparsers.add_parser( "extract-files", - usage=correct_usage(extract_files.usage), - description=extract_files.description, - parents=[extract_files.parser] + usage=correct_usage(extract_files.ui.usage), + description=extract_files.ui.description, + parents=[extract_files.ui.parser] ) subparsers.add_parser( "extract-images", - usage=correct_usage(extract_images.usage), - description=extract_images.description, - parents=[extract_images.parser] + usage=correct_usage(extract_images.ui.usage), + description=extract_images.ui.description, + parents=[extract_images.ui.parser] ) subparsers.add_parser( "list-files", diff --git a/ubireader/parsers/__init__.py b/ubireader/parsers/__init__.py index 9aa9059..0e973a3 100644 --- a/ubireader/parsers/__init__.py +++ b/ubireader/parsers/__init__.py @@ -1,33 +1,209 @@ +import os import argparse +from ubireader import settings +from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size, guess_leb_size +from ubireader.exceptions import UBIReaderParseError +from ubireader.debug import error, log +from ubireader.ubifs.defines import UBIFS_NODE_MAGIC +from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -parser_base = argparse.ArgumentParser(add_help=False) -parser_base.add_argument('filepath', help='UBI/UBIFS image file.') +class UIParser(object): + def __init__(self): + self._p = argparse.ArgumentParser(add_help=False) -parser_base.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') + def get_parser(self): + return self._p -parser_base.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') + parser = property(fget=get_parser) -parser_base.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') + def set_usage(self, usage): + self._p.usage = usage -parser_base.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') + def get_usage(self): + return self._p.usage -parser_base.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') + usage = property(get_usage, set_usage) -parser_base.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') -parser_base.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') + def set_description(self, description): + self._p.description = description -parser_base.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') + def get_description(self): + return self._p.description -parser_base.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') \ No newline at end of file + description = property(get_description, set_description) + + + def set_function(self, func): + self._p.set_defaults(func=func) + + function = property(fset=set_function) + + + # CLI options. + def arg_log(self): + self._p.add_argument('-l', '--log', action='store_true', dest='log', + help='Print extraction information to screen.') + + def arg_verbose_log(self): + self._p.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', + help='Prints nearly everything about anything to screen.') + + def arg_warn_only_block_read_errors(self): + self._p.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', + help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') + + def arg_ignore_block_header_errors(self): + self._p.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', + help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') + + def arg_u_boot_fix(self): + self._p.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', + help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') + + def arg_peb_size(self): + self._p.add_argument('-p', '--peb-size', type=int, dest='block_size', + help='Specify PEB size.') + + def arg_leb_size(self): + self._p.add_argument('-e', '--leb-size', type=int, dest='block_size', + help='Specify LEB size.') + + def arg_start_offset(self): + self._p.add_argument('-s', '--start-offset', type=int, dest='start_offset', + help='Specify offset of UBI/UBIFS data in file. (default: 0)') + + def arg_end_offset(self): + self._p.add_argument('-n', '--end-offset', type=int, dest='end_offset', + help='Specify end offset of UBI/UBIFS data in file.') + + def arg_guess_offset(self): + self._p.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', + help='Specify offset to start guessing where UBI data is in file. (default: 0)') + + def arg_image_type(self): + self._p.add_argument('-u', '--image-type', dest='image_type', + help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') + + def arg_keep_permissions(self): + self._p.add_argument('-k', '--keep-permissions', action='store_true', dest='permissions', + help='Maintain file permissions, requires running as root. (default: False)') + + def arg_output_dir(self): + self._p.add_argument('-o', '--output-dir', dest='output_dir', + help='Specify output directory path.') + + def arg_filepath(self): + self._p.add_argument('filepath', help='UBI/UBIFS image file.') + + def arg_image_type(self): + self._p.add_argument('-u', '--image-type', dest='image_type', + help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') + +# Processing variables. + def get_output_dir(self): + return self._output_dir + + output_dir = property(get_output_dir) + + def get_block_size(self): + return self._block_size + + block_size = property(get_block_size) + + def get_start_offset(self): + return self._start_offset + + start_offset = property(get_start_offset) + + def get_end_offset(self): + return self._end_offset + + end_offset = property(get_end_offset) + + def get_permissions(self): + return self._permissions + + permissions = property(get_permissions) + + def get_filepath(self): + return self._filepath + + filepath = property(get_filepath) + + def get_filetype(self): + return self._filetype + + filetype = property(get_filetype) + + def get_image_type(self): + return self._image_type + + image_type = property(get_image_type) + + def parse_args(self, *args, **kwargs): + settings.logging_on = kwargs['log'] + settings.logging_on_verbose = kwargs['verbose'] + settings.warn_only_block_read_errors = kwargs['warn_only_block_read_errors'] + settings.ignore_block_header_errors = kwargs['ignore_block_header_errors'] + settings.uboot_fix = kwargs['uboot_fix'] + + self._filepath = kwargs['filepath'] + if not os.path.exists(self.filepath): + raise UBIReaderParseError("File path doesn't exist.") + + if kwargs['start_offset']: + self._start_offset = kwargs['start_offset'] + + if kwargs['guess_offset']: + self._start_offset = guess_start_offset(self.filepath, kwargs['guess_offset']) + + else: + self._start_offset = guess_start_offset(self.filepath) + + self._end_offset = kwargs['end_offset'] + + + self._filetype = guess_filetype(self.filepath, self.start_offset) + if self.filetype not in [UBI_EC_HDR_MAGIC, UBIFS_NODE_MAGIC]: + raise UBIReaderParseError('Could not determine file type.') + + if kwargs['block_size']: + self._block_size = kwargs['block_size'] + else: + if self.filetype == UBI_EC_HDR_MAGIC: + self._block_size = guess_peb_size(self.filepath) + + elif self.filetype == UBIFS_NODE_MAGIC: + self._block_size = guess_leb_size(self.filepath) + + if not self.block_size: + raise UBIReaderParseError('PEB size could not be determined.') + + if 'image_type' in kwargs: + if kwargs['image_type']: + self._image_type = kwargs['image_type'].upper() + else: + self._image_type = 'UBIFS' + + if self._image_type not in ['UBI', 'UBIFS']: + raise UBIReaderParseError('Image type must be UBI or UBIFS.') + + if 'output_dir' in kwargs: + if kwargs['output_dir']: + self._output_dir = kwargs['output_dir'] + else: + self._output_dir = settings.output_dir + + if 'permissions' in kwargs: + self._permissions = kwargs['permissions'] + +def create_output_dir(outpath): + if not os.path.exists(outpath): + try: + os.makedirs(outpath) + log(create_output_dir, 'Created output path: %s' % outpath) + except Exception as e: + error(create_output_dir, 'Fatal', '%s' % e) \ No newline at end of file diff --git a/ubireader/parsers/display_blocks.py b/ubireader/parsers/display_blocks.py index 558ef84..6ad472d 100644 --- a/ubireader/parsers/display_blocks.py +++ b/ubireader/parsers/display_blocks.py @@ -1,91 +1,65 @@ -import argparse -import os - +from ubireader.parsers import UIParser from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import parser_base from ubireader.ubi import ubi_base from ubireader.ubi_io import ubi_file -from ubireader import settings -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - - -def display_blocks( - filepath, - block_search_params, - log=False, - verbose=False, - block_size=None, - start_offset=None, - end_offset=None, - guess_offset=None, - warn_only_block_read_errors=False, - ignore_block_header_errors=False, - uboot_fix=False - ): - settings.logging_on = log - - settings.logging_on_verbose = verbose - - settings.warn_only_block_read_errors = warn_only_block_read_errors - - settings.ignore_block_header_errors = ignore_block_header_errors - - settings.uboot_fix = uboot_fix - - if filepath: - path = filepath - if not os.path.exists(path): - raise UBIReaderParseError("File path doesn't exist.") - else: - raise UBIReaderParseError('File path must be provided.') - - if start_offset: - start_offset = start_offset - elif guess_offset: - start_offset = guess_start_offset(path, guess_offset) - else: - start_offset = guess_start_offset(path) - - if end_offset: - end_offset = end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - raise UBIReaderParseError('Could not determine file type.') - - if block_size: - block_size = block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - raise UBIReaderParseError('Block size could not be determined.') - - if block_search_params: - try: - search_params = eval(block_search_params) - - if not isinstance(search_params, dict): - raise UBIReaderParseError('Search Param Error: Params must be a Dict of block PEB object items:value pairs.') - - except NameError as e: - raise UBIReaderParseError('Search Param Error: Dict key block attrs must be single quoted.') - - except Exception as e: - raise UBIReaderParseError('Search Param Error: %s' % e) - - else: - raise UBIReaderParseError('No search parameters given, -b arg is required.') - - - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + +# Set up argparser and cli options. +ui = UIParser() +ui.description = 'Search for specified blocks and display information.' +ui.usage = """ +ubireader display-blocks "{'block.attr': value,...}" path/to/image + Search for blocks by given parameters and display information about them. + This is block only, no volume or image information is created, which can + be used to debug file and image extraction. +Example: + "{'peb_num':[0, 1] + range(100, 102), 'ec_hdr.ec': 1, 'is_valid': True}" + This matches block.peb_num 0, 1, 100, 101, and 102 + with a block.ec_hdr.ec (erase count) of 1, that are valid PEB blocks. + For a full list of parameters check ubireader.ubi.block.description. +""" + + +ui.parser.add_argument('block_search_params', + help=""" + Double quoted Dict of ubi.block.description attributes, which is run through eval(). + Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}" + """) + +ui.arg_log() +ui.arg_verbose_log() +ui.arg_peb_size() +ui.arg_leb_size() +ui.arg_start_offset() +ui.arg_end_offset() +ui.arg_guess_offset() +ui.arg_warn_only_block_read_errors() +ui.arg_ignore_block_header_errors() +ui.arg_u_boot_fix() +ui.arg_output_dir() +ui.arg_filepath() + + +def display_blocks(*args, **kwargs): + ui.parse_args(*args, **kwargs) + if 'block_search_params' in kwargs: + if kwargs['block_search_params']: + try: + search_params = eval(kwargs['block_search_params']) + + if not isinstance(search_params, dict): + raise UBIReaderParseError('Search Param Error: Params must be a Dict of block PEB object items:value pairs.') + + except NameError as e: + raise UBIReaderParseError('Search Param Error: Dict key block attrs must be single quoted.') + + except Exception as e: + raise UBIReaderParseError('Search Param Error: %s' % e) + + else: + raise UBIReaderParseError('No search parameters given, -b arg is required.') + + + ufile_obj = ubi_file(ui.filepath, ui.block_size, ui.start_offset, ui.end_offset) ubi_obj = ubi_base(ufile_obj) blocks = [] @@ -122,31 +96,4 @@ def display_blocks( print(block.display()) -description = 'Search for specified blocks and display information.' -usage = """ -ubireader display-blocks "{'block.attr': value,...}" path/to/image - Search for blocks by given parameters and display information about them. - This is block only, no volume or image information is created, which can - be used to debug file and image extraction. -Example: - "{'peb_num':[0, 1] + range(100, 102), 'ec_hdr.ec': 1, 'is_valid': True}" - This matches block.peb_num 0, 1, 100, 101, and 102 - with a block.ec_hdr.ec (erase count) of 1, that are valid PEB blocks. - For a full list of parameters check ubireader.ubi.block.description. -""" -parser = argparse.ArgumentParser( - usage=usage, - description=description, - parents=[parser_base], - add_help=False -) -parser.set_defaults(func=display_blocks) - -parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - -parser.add_argument('block_search_params', - help=""" - Double quoted Dict of ubi.block.description attributes, which is run through eval(). - Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}" - """) +ui.function=display_blocks \ No newline at end of file diff --git a/ubireader/parsers/extract_files.py b/ubireader/parsers/extract_files.py index 5b6bc1d..7cec849 100644 --- a/ubireader/parsers/extract_files.py +++ b/ubireader/parsers/extract_files.py @@ -1,98 +1,38 @@ -import argparse import os -from ubireader import settings -from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import parser_base +from ubireader.parsers import UIParser, create_output_dir from ubireader.ubi import ubi from ubireader.ubi.defines import UBI_EC_HDR_MAGIC from ubireader.ubifs import ubifs from ubireader.ubifs.output import extract_files as _extract_files from ubireader.ubifs.defines import UBIFS_NODE_MAGIC from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - -def create_output_dir(outpath): - if os.path.exists(outpath): - if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) - else: - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - - -def extract_files( - filepath, - permissions=False, - log=False, - verbose=False, - block_size=None, - start_offset=None, - end_offset=None, - guess_offset=None, - warn_only_block_read_errors=False, - ignore_block_header_errors=False, - uboot_fix=False, - outpath=None - ): - settings.logging_on = log - - settings.logging_on_verbose = verbose - - settings.warn_only_block_read_errors = warn_only_block_read_errors - - settings.ignore_block_header_errors = ignore_block_header_errors - - settings.uboot_fix = uboot_fix - - if filepath: - path = filepath - if not os.path.exists(path): - raise UBIReaderParseError("File path doesn't exist.") - - if start_offset: - start_offset = start_offset - elif guess_offset: - start_offset = guess_start_offset(path, guess_offset) - else: - start_offset = guess_start_offset(path) - - if end_offset: - end_offset = end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - raise UBIReaderParseError('Could not determine file type.') - - if outpath: - outpath = outpath - else: - outpath = settings.output_dir - - if block_size: - block_size = block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - raise UBIReaderParseError('Block size could not be determined.') - - perms = permissions +# Set up argparser and cli options. +ui = UIParser() +ui.description = 'Extract contents of a UBI or UBIFS image.' +ui.usage = 'ubireader_extract_files [options] filepath' +ui.arg_keep_permissions() +ui.arg_log() +ui.arg_verbose_log() +ui.arg_peb_size() +ui.arg_start_offset() +ui.arg_end_offset() +ui.arg_guess_offset() +ui.arg_warn_only_block_read_errors() +ui.arg_ignore_block_header_errors() +ui.arg_u_boot_fix() +ui.arg_output_dir() +ui.arg_filepath() + +# Process file. +def extract_files(*args, **kwargs): + ui.parse_args(*args, **kwargs) # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + ufile_obj = ubi_file(ui.filepath, ui.block_size, ui.start_offset, ui.end_offset) - if filetype == UBI_EC_HDR_MAGIC: + if ui.filetype == UBI_EC_HDR_MAGIC: # Create UBI object ubi_obj = ubi(ufile_obj) @@ -101,7 +41,7 @@ def extract_files( # Create path for specific image # In case multiple images in data - img_outpath = os.path.join(outpath, '%s' % image.image_seq) + img_outpath = os.path.join(ui.output_dir, '%s' % image.image_seq) # Loop through volumes in each image. for volume in image.volumes: @@ -127,39 +67,23 @@ def extract_files( # Extract files from UBI image. ubifs_obj = ubifs(lebv_file) print('Extracting files to: %s' % vol_outpath) - _extract_files(ubifs_obj, vol_outpath, perms) + _extract_files(ubifs_obj, vol_outpath, ui.permissions) - elif filetype == UBIFS_NODE_MAGIC: + elif ui.filetype == UBIFS_NODE_MAGIC: # Create UBIFS object ubifs_obj = ubifs(ufile_obj) # Create directory for files. - create_output_dir(outpath) + create_output_dir(ui.output_dir) # Extract files from UBIFS image. - print('Extracting files to: %s' % outpath) - _extract_files(ubifs_obj, outpath, perms) + print('Extracting files to: %s' % ui.output_dir) + _extract_files(ubifs_obj, ui.output_dir, ui.permissions) else: print('Something went wrong to get here.') -description = 'Extract contents of a UBI or UBIFS image.' -usage = 'ubireader_extract_files [options] filepath' -parser = argparse.ArgumentParser( - usage=usage, - description=description, - parents=[parser_base], - add_help=False -) -parser.set_defaults(func=extract_files) - -parser.add_argument('-k', '--keep-permissions', action='store_true', dest='permissions', - help='Maintain file permissions, requires running as root. (default: False)') - -parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - -parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') +# Add process function to parser. +ui.function = extract_files \ No newline at end of file diff --git a/ubireader/parsers/extract_images.py b/ubireader/parsers/extract_images.py index 81901de..b453fa0 100644 --- a/ubireader/parsers/extract_images.py +++ b/ubireader/parsers/extract_images.py @@ -1,98 +1,45 @@ -import argparse import os -from ubireader import settings -from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import parser_base +from ubireader.parsers import UIParser, create_output_dir from ubireader.ubi import ubi -from ubireader.ubi.defines import UBI_EC_HDR_MAGIC from ubireader.ubi_io import ubi_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size - - -def create_output_dir(outpath): - if not os.path.exists(outpath): - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - - -def extract_images( - filepath, - log=False, - verbose=False, - block_size=None, - image_type=None, - start_offset=None, - end_offset=None, - guess_offset=None, - warn_only_block_read_errors=False, - ignore_block_header_errors=False, - uboot_fix=False, - outpath=None - ): - settings.logging_on = log - - settings.logging_on_verbose = verbose - - settings.warn_only_block_read_errors = warn_only_block_read_errors - - settings.ignore_block_header_errors = ignore_block_header_errors - - settings.uboot_fix = uboot_fix - - if filepath: - path = filepath - if not os.path.exists(path): - raise UBIReaderParseError("File path doesn't exist.") - - if start_offset: - start_offset = start_offset - elif guess_offset: - start_offset = guess_start_offset(path, guess_offset) - else: - start_offset = guess_start_offset(path) - - if end_offset: - end_offset = end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if filetype != UBI_EC_HDR_MAGIC: - raise UBIReaderParseError('File does not look like UBI data.') +from ubireader import settings - img_name = os.path.basename(path) - if outpath: - outpath = os.path.abspath(os.path.join(outpath, img_name)) +# Set up argparser and cli options. +ui = UIParser() +ui.description = 'Extract UBI or UBIFS images from file containing UBI data in it.' +ui.usage = 'ubireader_extract_images [options] filepath' +ui.arg_log() +ui.arg_verbose_log() +ui.arg_peb_size() +ui.arg_image_type() +ui.arg_start_offset() +ui.arg_end_offset() +ui.arg_guess_offset() +ui.arg_warn_only_block_read_errors() +ui.arg_ignore_block_header_errors() +ui.arg_u_boot_fix() +ui.arg_output_dir() +ui.arg_filepath() + +def extract_images(*args, **kwargs): + ui.parse_args(*args, **kwargs) + + img_name = os.path.basename(ui.filepath) + if ui.output_dir: + outpath = os.path.abspath(os.path.join(ui.output_dir, img_name)) else: outpath = os.path.join(settings.output_dir, img_name) - if block_size: - block_size = block_size - else: - block_size = guess_peb_size(path) - - if not block_size: - raise UBIReaderParseError('Block size could not be determined.') - - if image_type: - image_type = image_type.upper() - else: - image_type = 'UBIFS' - # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + ufile_obj = ubi_file(ui.filepath, ui.block_size, ui.start_offset, ui.end_offset) # Create UBI object ubi_obj = ubi(ufile_obj) # Loop through found images in file. for image in ubi_obj.images: - if image_type == 'UBI': + if ui.image_type == 'UBI': # Create output path and open file. img_outpath = os.path.join(outpath, 'img-%s.ubi' % image.image_seq) create_output_dir(outpath) @@ -104,7 +51,7 @@ def extract_images( # Write block (PEB) to file f.write(ubi_obj.file.read_block(ubi_obj.blocks[block])) - elif image_type == 'UBIFS': + elif ui.image_type == 'UBIFS': # Loop through image volumes for volume in image.volumes: # Create output path and open file. @@ -117,18 +64,5 @@ def extract_images( f.write(block) -description = 'Extract UBI or UBIFS images from file containing UBI data in it.' -usage = 'ubireader_extract_images [options] filepath' -parser = argparse.ArgumentParser( - usage=usage, - description=description, - parents=[parser_base], - add_help=False -) -parser.set_defaults(func=extract_images) - -parser.add_argument('-u', '--image-type', dest='image_type', - help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') - -parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') +# Add process function to parser. +ui.function = extract_images \ No newline at end of file From 4e937b8aa6be2cbd24c2d6f4a01fc70b72b3b47e Mon Sep 17 00:00:00 2001 From: Jason Pruitt Date: Fri, 24 Feb 2023 14:51:57 -0800 Subject: [PATCH 15/15] More code cleanup of new UI. --- scripts/ubireader_display_blocks | 54 ++----- scripts/ubireader_display_info | 49 ++---- scripts/ubireader_extract_files | 49 ++---- scripts/ubireader_extract_images | 49 ++---- scripts/ubireader_list_files | 49 ++---- scripts/ubireader_utils_info | 49 ++---- ubireader/__main__.py | 65 +++----- ubireader/parsers/__init__.py | 188 ++++++----------------- ubireader/parsers/display_blocks.py | 50 +----- ubireader/parsers/display_info.py | 100 ++---------- ubireader/parsers/extract_files.py | 230 ++++++++++++++++++++++++---- ubireader/parsers/extract_images.py | 50 ++---- ubireader/parsers/list_files.py | 106 ++----------- ubireader/parsers/utils_info.py | 105 +------------ ubireader/ubifs/output.py | 206 ------------------------- 15 files changed, 405 insertions(+), 994 deletions(-) delete mode 100755 ubireader/ubifs/output.py diff --git a/scripts/ubireader_display_blocks b/scripts/ubireader_display_blocks index 1efb923..c2ff655 100755 --- a/scripts/ubireader_display_blocks +++ b/scripts/ubireader_display_blocks @@ -1,47 +1,25 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/scripts/ubireader_display_blocks -# (c) 2019 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -############################################################# -# Search by block parameters and display information about -# matching blocks. -############################################################# - -import argparse import sys +from ubireader.ui.scripts import DisplayBlocksUi from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import display_blocks - -if __name__=='__main__': - parser = argparse.ArgumentParser( - usage=display_blocks.ui.usage, - description=display_blocks.ui.description, - parents=[display_blocks.ui.parser] - ) +def main(): + ui = DisplayBlocksUi(True) + ui.usage='ubireader_display_blocks [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() + sys.exit(1) + + args = vars(ui.parser.parse_args()) + + func = args.pop('func') + filepath = args.pop('filepath') - args = vars(parser.parse_args()) - func = args.pop("func") try: - func(**args) + func(filepath, **args) except UBIReaderParseError as e: - parser.error(str(e)) + sys.exit("Error: " + str(e)) + +if __name__ == "__main__": + main() diff --git a/scripts/ubireader_display_info b/scripts/ubireader_display_info index b0c5ee4..f8416a1 100755 --- a/scripts/ubireader_display_info +++ b/scripts/ubireader_display_info @@ -1,44 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import argparse import sys +from ubireader.ui.scripts import DisplayInfoUi from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import display_info - -if __name__=='__main__': - parser = argparse.ArgumentParser( - usage=display_info.usage, - description=display_info.description, - parents=[display_info.parser] - ) +def main(): + ui = DisplayInfoUi(True) + ui.usage='ubireader_display_info [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) + + func = args.pop('func') + filepath = args.pop('filepath') - args = vars(parser.parse_args()) - func = args.pop("func") try: - func(**args) + func(filepath, **args) except UBIReaderParseError as e: - parser.error(str(e)) + sys.exit("Error: " + str(e)) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/ubireader_extract_files b/scripts/ubireader_extract_files index 74775be..fbc61f9 100755 --- a/scripts/ubireader_extract_files +++ b/scripts/ubireader_extract_files @@ -1,44 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - import sys -import argparse +from ubireader.ui.scripts import ExtractFilesUi from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import extract_files - -if __name__=='__main__': - parser = argparse.ArgumentParser( - usage=extract_files.ui.usage, - description=extract_files.ui.description, - parents=[extract_files.ui.parser] - ) +def main(): + ui = ExtractFilesUi(True) + ui.usage='ubireader_extract_files [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) + + func = args.pop('func') + filepath = args.pop('filepath') - args = vars(parser.parse_args()) - func = args.pop("func") try: - func(**args) + func(filepath, **args) except UBIReaderParseError as e: - parser.error(str(e)) + sys.exit("Error: " + str(e)) + +if __name__ == "__main__": + main() diff --git a/scripts/ubireader_extract_images b/scripts/ubireader_extract_images index 8129cd5..6e1bb8d 100755 --- a/scripts/ubireader_extract_images +++ b/scripts/ubireader_extract_images @@ -1,44 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import argparse import sys +from ubireader.ui.scripts import ExtractImagesUi from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import extract_images - -if __name__=='__main__': - parser = argparse.ArgumentParser( - usage=extract_images.ui.usage, - description=extract_images.ui.description, - parents=[extract_images.ui.parser] - ) +def main(): + ui = ExtractImagesUi(True) + ui.usage='ubireader_extract_images [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) + + func = args.pop('func') + filepath = args.pop('filepath') - args = vars(parser.parse_args()) - func = args.pop("func") try: - func(**args) + func(filepath, **args) except UBIReaderParseError as e: - parser.error(str(e)) + sys.exit("Error: " + str(e)) + +if __name__ == "__main__": + main() diff --git a/scripts/ubireader_list_files b/scripts/ubireader_list_files index 373eb84..71009a9 100755 --- a/scripts/ubireader_list_files +++ b/scripts/ubireader_list_files @@ -1,44 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (C) Collin Mulliner based on Jason Pruitt's ubireader_extract_images -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import argparse import sys +from ubireader.ui.scripts import ListFilesUi from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import list_files - -if __name__=='__main__': - parser = argparse.ArgumentParser( - usage=list_files.usage, - description=list_files.description, - parents=[list_files.parser] - ) +def main(): + ui = ListFilesUi(True) + ui.usage='ubireader_list_files [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) + + func = args.pop('func') + filepath = args.pop('filepath') - args = vars(parser.parse_args()) - func = args.pop("func") try: - func(**args) + func(filepath, **args) except UBIReaderParseError as e: - parser.error(str(e)) + sys.exit("Error: " + str(e)) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/ubireader_utils_info b/scripts/ubireader_utils_info index 4912ec4..3ba3622 100755 --- a/scripts/ubireader_utils_info +++ b/scripts/ubireader_utils_info @@ -1,44 +1,25 @@ -#!/usr/bin/env python - -############################################################# -# ubi_reader -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import argparse import sys +from ubireader.ui.scripts import UtilsInfoUi from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import utils_info - -if __name__=='__main__': - parser = argparse.ArgumentParser( - usage=utils_info.usage, - description=utils_info.description, - parents=[utils_info.parser] - ) +def main(): + ui = UtilsInfoUi(True) + ui.usage='ubireader_utils_info [options] filepath' if len(sys.argv) == 1: - parser.print_help() + ui.parser.print_help() sys.exit(1) + + args = vars(ui.parser.parse_args()) + + func = args.pop('func') + filepath = args.pop('filepath') - args = vars(parser.parse_args()) - func = args.pop("func") try: - func(**args) + func(filepath, **args) except UBIReaderParseError as e: - parser.error(str(e)) + sys.exit("Error: " + str(e)) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ubireader/__main__.py b/ubireader/__main__.py index d363850..8ebc2ce 100644 --- a/ubireader/__main__.py +++ b/ubireader/__main__.py @@ -1,59 +1,30 @@ -import argparse import sys +import inspect +import argparse +import ubireader.ui.scripts from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import display_blocks, display_info, extract_files, extract_images, list_files, utils_info - - -def correct_usage(text): - """Adapt old usage text to new format.""" - return text.replace('_', ' ', 1).replace('_', '-') - def main(): - parser = argparse.ArgumentParser(description='Extract files from UBI and UBIFS images.') + parser = argparse.ArgumentParser(description='UBI and UBIFS tools.') subparsers = parser.add_subparsers() - subparsers.add_parser( - "display-blocks", - usage=correct_usage(display_blocks.ui.usage), - description=display_blocks.ui.description, - parents=[display_blocks.ui.parser] - ) - subparsers.add_parser( - "display-info", - usage=correct_usage(display_info.usage), - description=display_info.description, - parents=[display_info.parser] - ) - subparsers.add_parser( - "extract-files", - usage=correct_usage(extract_files.ui.usage), - description=extract_files.ui.description, - parents=[extract_files.ui.parser] - ) - subparsers.add_parser( - "extract-images", - usage=correct_usage(extract_images.ui.usage), - description=extract_images.ui.description, - parents=[extract_images.ui.parser] - ) - subparsers.add_parser( - "list-files", - usage=correct_usage(list_files.usage), - description=list_files.description, - parents=[list_files.parser] - ) - subparsers.add_parser( - "utils-info", - usage=correct_usage(utils_info.usage), - description=utils_info.description, - parents=[utils_info.parser] - ) + + for name, obj in inspect.getmembers(sys.modules[ubireader.ui.scripts.__name__]): + if inspect.isclass(obj) and name.endswith('Ui'): + subp = obj() + subparsers.add_parser( + subp.cmd, + usage=subp.usage, + description=subp.description, + parents=[subp.parser] + ) args = vars(parser.parse_args()) - func = args.pop("func") + print(args) + func = args.pop('func') + filepath = args.pop('filepath') try: - func(**args) + func(filepath, **args) except UBIReaderParseError as e: sys.exit("Error: " + str(e)) diff --git a/ubireader/parsers/__init__.py b/ubireader/parsers/__init__.py index 0e973a3..7411b3e 100644 --- a/ubireader/parsers/__init__.py +++ b/ubireader/parsers/__init__.py @@ -1,5 +1,4 @@ import os -import argparse from ubireader import settings from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size, guess_leb_size @@ -8,149 +7,17 @@ from ubireader.ubifs.defines import UBIFS_NODE_MAGIC from ubireader.ubi.defines import UBI_EC_HDR_MAGIC +class ArgHandler(): + # Processing variables. -class UIParser(object): - def __init__(self): - self._p = argparse.ArgumentParser(add_help=False) - - def get_parser(self): - return self._p - - parser = property(fget=get_parser) - - def set_usage(self, usage): - self._p.usage = usage - - def get_usage(self): - return self._p.usage - - usage = property(get_usage, set_usage) - - - def set_description(self, description): - self._p.description = description - - def get_description(self): - return self._p.description - - description = property(get_description, set_description) - - - def set_function(self, func): - self._p.set_defaults(func=func) - - function = property(fset=set_function) - - - # CLI options. - def arg_log(self): - self._p.add_argument('-l', '--log', action='store_true', dest='log', - help='Print extraction information to screen.') - - def arg_verbose_log(self): - self._p.add_argument('-v', '--verbose-log', action='store_true', dest='verbose', - help='Prints nearly everything about anything to screen.') - - def arg_warn_only_block_read_errors(self): - self._p.add_argument('-w', '--warn-only-block-read-errors', action='store_true', dest='warn_only_block_read_errors', - help='Attempts to continue extracting files even with bad block reads. Some data will be missing or corrupted! (default: False)') - - def arg_ignore_block_header_errors(self): - self._p.add_argument('-i', '--ignore-block-header-errors', action='store_true', dest='ignore_block_header_errors', - help='Forces unused and error containing blocks to be included and also displayed with log/verbose. (default: False)') - - def arg_u_boot_fix(self): - self._p.add_argument('-f', '--u-boot-fix', action='store_true', dest='uboot_fix', - help='Assume blocks with image_seq 0 are because of older U-boot implementations and include them. (default: False)') - - def arg_peb_size(self): - self._p.add_argument('-p', '--peb-size', type=int, dest='block_size', - help='Specify PEB size.') - - def arg_leb_size(self): - self._p.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size.') - - def arg_start_offset(self): - self._p.add_argument('-s', '--start-offset', type=int, dest='start_offset', - help='Specify offset of UBI/UBIFS data in file. (default: 0)') - - def arg_end_offset(self): - self._p.add_argument('-n', '--end-offset', type=int, dest='end_offset', - help='Specify end offset of UBI/UBIFS data in file.') - - def arg_guess_offset(self): - self._p.add_argument('-g', '--guess-offset', type=int, dest='guess_offset', - help='Specify offset to start guessing where UBI data is in file. (default: 0)') - - def arg_image_type(self): - self._p.add_argument('-u', '--image-type', dest='image_type', - help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') - - def arg_keep_permissions(self): - self._p.add_argument('-k', '--keep-permissions', action='store_true', dest='permissions', - help='Maintain file permissions, requires running as root. (default: False)') - - def arg_output_dir(self): - self._p.add_argument('-o', '--output-dir', dest='output_dir', - help='Specify output directory path.') - - def arg_filepath(self): - self._p.add_argument('filepath', help='UBI/UBIFS image file.') - - def arg_image_type(self): - self._p.add_argument('-u', '--image-type', dest='image_type', - help='Specify image type to extract UBI or UBIFS. (default: UBIFS)') - -# Processing variables. - def get_output_dir(self): - return self._output_dir - - output_dir = property(get_output_dir) - - def get_block_size(self): - return self._block_size - - block_size = property(get_block_size) - - def get_start_offset(self): - return self._start_offset - - start_offset = property(get_start_offset) - - def get_end_offset(self): - return self._end_offset - - end_offset = property(get_end_offset) - - def get_permissions(self): - return self._permissions - - permissions = property(get_permissions) - - def get_filepath(self): - return self._filepath - - filepath = property(get_filepath) - - def get_filetype(self): - return self._filetype - - filetype = property(get_filetype) - - def get_image_type(self): - return self._image_type - - image_type = property(get_image_type) - - def parse_args(self, *args, **kwargs): + def __init__(self, filepath, *args, **kwargs): settings.logging_on = kwargs['log'] settings.logging_on_verbose = kwargs['verbose'] settings.warn_only_block_read_errors = kwargs['warn_only_block_read_errors'] settings.ignore_block_header_errors = kwargs['ignore_block_header_errors'] settings.uboot_fix = kwargs['uboot_fix'] - self._filepath = kwargs['filepath'] + self._filepath = filepath if not os.path.exists(self.filepath): raise UBIReaderParseError("File path doesn't exist.") @@ -180,7 +47,7 @@ def parse_args(self, *args, **kwargs): self._block_size = guess_leb_size(self.filepath) if not self.block_size: - raise UBIReaderParseError('PEB size could not be determined.') + raise UBIReaderParseError('PEB/LEB size could not be determined.') if 'image_type' in kwargs: if kwargs['image_type']: @@ -200,10 +67,45 @@ def parse_args(self, *args, **kwargs): if 'permissions' in kwargs: self._permissions = kwargs['permissions'] -def create_output_dir(outpath): - if not os.path.exists(outpath): + @property + def output_dir(self): + return self._output_dir + + @property + def block_size(self): + return self._block_size + + @property + def start_offset(self): + return self._start_offset + + @property + def end_offset(self): + return self._end_offset + + @property + def permissions(self): + return self._permissions + + @property + def filepath(self): + return self._filepath + + @property + def filetype(self): + return self._filetype + + @property + def image_type(self): + return self._image_type + +def makedir(outpath, overwrite=True): + if os.path.exists(outpath): + if os.listdir(outpath) and not overwrite: + error(makedir, 'Fatal', 'Output directory is not empty. %s' % outpath) + else: try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) + os.makedirs(outpath, exist_ok=overwrite) + log(makedir, 'Created output path: %s' % outpath) except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) \ No newline at end of file + error(makedir, 'Fatal', '%s' % e) \ No newline at end of file diff --git a/ubireader/parsers/display_blocks.py b/ubireader/parsers/display_blocks.py index 6ad472d..1a4636b 100644 --- a/ubireader/parsers/display_blocks.py +++ b/ubireader/parsers/display_blocks.py @@ -1,46 +1,11 @@ -from ubireader.parsers import UIParser +from ubireader.parsers import ArgHandler from ubireader.exceptions import UBIReaderParseError from ubireader.ubi import ubi_base from ubireader.ubi_io import ubi_file -# Set up argparser and cli options. -ui = UIParser() -ui.description = 'Search for specified blocks and display information.' -ui.usage = """ -ubireader display-blocks "{'block.attr': value,...}" path/to/image - Search for blocks by given parameters and display information about them. - This is block only, no volume or image information is created, which can - be used to debug file and image extraction. -Example: - "{'peb_num':[0, 1] + range(100, 102), 'ec_hdr.ec': 1, 'is_valid': True}" - This matches block.peb_num 0, 1, 100, 101, and 102 - with a block.ec_hdr.ec (erase count) of 1, that are valid PEB blocks. - For a full list of parameters check ubireader.ubi.block.description. -""" - - -ui.parser.add_argument('block_search_params', - help=""" - Double quoted Dict of ubi.block.description attributes, which is run through eval(). - Ex. "{\'peb_num\':[0, 1], \'ec_hdr.ec\': 1, \'is_valid\': True}" - """) - -ui.arg_log() -ui.arg_verbose_log() -ui.arg_peb_size() -ui.arg_leb_size() -ui.arg_start_offset() -ui.arg_end_offset() -ui.arg_guess_offset() -ui.arg_warn_only_block_read_errors() -ui.arg_ignore_block_header_errors() -ui.arg_u_boot_fix() -ui.arg_output_dir() -ui.arg_filepath() - - -def display_blocks(*args, **kwargs): - ui.parse_args(*args, **kwargs) +def parse(*args, **kwargs): + args = ArgHandler(*args, **kwargs) + if 'block_search_params' in kwargs: if kwargs['block_search_params']: try: @@ -59,7 +24,7 @@ def display_blocks(*args, **kwargs): raise UBIReaderParseError('No search parameters given, -b arg is required.') - ufile_obj = ubi_file(ui.filepath, ui.block_size, ui.start_offset, ui.end_offset) + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) ubi_obj = ubi_base(ufile_obj) blocks = [] @@ -93,7 +58,4 @@ def display_blocks(*args, **kwargs): print('\nBlock matches: %s' % len(blocks)) for block in blocks: - print(block.display()) - - -ui.function=display_blocks \ No newline at end of file + print(block.display()) \ No newline at end of file diff --git a/ubireader/parsers/display_info.py b/ubireader/parsers/display_info.py index 7a3fcd5..2b72d23 100644 --- a/ubireader/parsers/display_info.py +++ b/ubireader/parsers/display_info.py @@ -1,97 +1,34 @@ -import argparse -import os - -from ubireader import settings -from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import parser_base +from ubireader.parsers import ArgHandler from ubireader.ubi import ubi from ubireader.ubi.defines import UBI_EC_HDR_MAGIC from ubireader.ubifs import ubifs from ubireader.ubifs.defines import UBIFS_NODE_MAGIC -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size from ubireader.ubi_io import ubi_file, leb_virtual_file -def display_info( - filepath, - log=False, - verbose=False, - ubifs_info=False, - block_size=None, - start_offset=None, - end_offset=None, - guess_offset=None, - warn_only_block_read_errors=False, - ignore_block_header_errors=False, - uboot_fix=False - ): - settings.logging_on = log - - settings.logging_on_verbose = verbose - - settings.warn_only_block_read_errors = warn_only_block_read_errors - - settings.ignore_block_header_errors = ignore_block_header_errors - - settings.uboot_fix = uboot_fix - - if filepath: - path = filepath - if not os.path.exists(path): - raise UBIReaderParseError("File path doesn't exist.") - - if start_offset: - start_offset = start_offset - elif guess_offset: - start_offset = guess_start_offset(path, guess_offset) - else: - start_offset = guess_start_offset(path) - - if end_offset: - end_offset = end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if not filetype: - raise UBIReaderParseError('Could not determine file type.') - - ubifs_info = ubifs_info - - if block_size: - block_size = block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) - - if not block_size: - raise UBIReaderParseError('Block size could not be determined.') - - +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) - if filetype == UBI_EC_HDR_MAGIC: + if args.filetype == UBI_EC_HDR_MAGIC: # Create UBI object ubi_obj = ubi(ufile_obj) # Display UBI info if not UBIFS request. - if not ubifs_info: + if args.image_type == 'UBI': print(ubi_obj.display()) # Loop through found images in file. for image in ubi_obj.images: # Display image information if not UBIFS request. - if not ubifs_info: + if args.image_type == 'UBI': print('%s' % image.display('\t')) # Loop through volumes in each image. for volume in image.volumes: # Show UBI or UBIFS info. - if not ubifs_info: - + if args.image_type == 'UBI': # Display volume information. print(image.volumes[volume].display('\t\t')) @@ -118,7 +55,7 @@ def display_info( except: print('Master Node Error only one valid node.') - elif filetype == UBIFS_NODE_MAGIC: + elif args.filetype == UBIFS_NODE_MAGIC: # Create UBIFS object ubifs_obj = ubifs(ufile_obj) print(ubifs_obj.display()) @@ -130,21 +67,4 @@ def display_info( print('Master Node Error only one valid node.') else: - print('Something went wrong to get here.') - - -description = 'Show information about UBI or UBIFS image.' -usage = 'ubireader_display_info [options] filepath' -parser = argparse.ArgumentParser( - usage=usage, - description=description, - parents=[parser_base], - add_help=False -) -parser.set_defaults(func=display_info) - -parser.add_argument('-u', '--ubifs-info', action='store_true', dest='ubifs_info', - help='Get UBIFS information from inside a UBI image. (default: false)') - -parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') + print('Something went wrong to get here.') \ No newline at end of file diff --git a/ubireader/parsers/extract_files.py b/ubireader/parsers/extract_files.py index 7cec849..0bd9053 100644 --- a/ubireader/parsers/extract_files.py +++ b/ubireader/parsers/extract_files.py @@ -1,38 +1,24 @@ import os +import struct -from ubireader.parsers import UIParser, create_output_dir +from ubireader import settings +from ubireader.debug import error, log, verbose_log +from ubireader.parsers import ArgHandler, makedir from ubireader.ubi import ubi from ubireader.ubi.defines import UBI_EC_HDR_MAGIC -from ubireader.ubifs import ubifs -from ubireader.ubifs.output import extract_files as _extract_files -from ubireader.ubifs.defines import UBIFS_NODE_MAGIC +from ubireader.ubifs import ubifs, walk +from ubireader.ubifs.misc import decompress +from ubireader.ubifs.defines import * from ubireader.ubi_io import ubi_file, leb_virtual_file -# Set up argparser and cli options. -ui = UIParser() -ui.description = 'Extract contents of a UBI or UBIFS image.' -ui.usage = 'ubireader_extract_files [options] filepath' -ui.arg_keep_permissions() -ui.arg_log() -ui.arg_verbose_log() -ui.arg_peb_size() -ui.arg_start_offset() -ui.arg_end_offset() -ui.arg_guess_offset() -ui.arg_warn_only_block_read_errors() -ui.arg_ignore_block_header_errors() -ui.arg_u_boot_fix() -ui.arg_output_dir() -ui.arg_filepath() - # Process file. -def extract_files(*args, **kwargs): - ui.parse_args(*args, **kwargs) +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) # Create file object. - ufile_obj = ubi_file(ui.filepath, ui.block_size, ui.start_offset, ui.end_offset) + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) - if ui.filetype == UBI_EC_HDR_MAGIC: + if args.filetype == UBI_EC_HDR_MAGIC: # Create UBI object ubi_obj = ubi(ufile_obj) @@ -41,7 +27,7 @@ def extract_files(*args, **kwargs): # Create path for specific image # In case multiple images in data - img_outpath = os.path.join(ui.output_dir, '%s' % image.image_seq) + img_outpath = os.path.join(args.output_dir, '%s' % image.image_seq) # Loop through volumes in each image. for volume in image.volumes: @@ -53,7 +39,7 @@ def extract_files(*args, **kwargs): vol_outpath = os.path.join(img_outpath, volume) # Create volume output path directory. - create_output_dir(vol_outpath) + makedir(vol_outpath, False) # Skip volume if empty. if not len(vol_blocks): @@ -67,23 +53,199 @@ def extract_files(*args, **kwargs): # Extract files from UBI image. ubifs_obj = ubifs(lebv_file) print('Extracting files to: %s' % vol_outpath) - _extract_files(ubifs_obj, vol_outpath, ui.permissions) + extract(ubifs_obj, vol_outpath, args.permissions) - elif ui.filetype == UBIFS_NODE_MAGIC: + elif args.filetype == UBIFS_NODE_MAGIC: # Create UBIFS object ubifs_obj = ubifs(ufile_obj) # Create directory for files. - create_output_dir(ui.output_dir) + makedir(args.output_dir, False) # Extract files from UBIFS image. - print('Extracting files to: %s' % ui.output_dir) - _extract_files(ubifs_obj, ui.output_dir, ui.permissions) + print('Extracting files to: %s' % args.output_dir) + extract(ubifs_obj, args.output_dir, args.permissions) else: print('Something went wrong to get here.') -# Add process function to parser. -ui.function = extract_files \ No newline at end of file +def is_safe_path(basedir, path): + basedir = os.path.realpath(basedir) + path = os.path.realpath(os.path.join(basedir, path)) + return True if path.startswith(basedir) else False + + +def extract(ubifs, out_path, perms=False): + """Extract UBIFS contents to_path/ + + Arguments: + Obj:ubifs -- UBIFS object. + Str:out_path -- Path to extract contents to. + """ + try: + inodes = {} + bad_blocks = [] + + walk.index(ubifs, ubifs.master_node.root_lnum, ubifs.master_node.root_offs, inodes, bad_blocks) + + if len(inodes) < 2: + raise Exception('No inodes found') + + for dent in inodes[1]['dent']: + extract_dents(ubifs, inodes, dent, out_path, perms) + + if len(bad_blocks): + error(extract, 'Warn', 'Data may be missing or corrupted, bad blocks, LEB [%s]' % ','.join(map(str, bad_blocks))) + + except Exception as e: + error(extract, 'Error', '%s' % e) + + +def extract_dents(ubifs, inodes, dent_node, path='', perms=False): + if dent_node.inum not in inodes: + error(extract_dents, 'Error', 'inum: %s not found in inodes' % (dent_node.inum)) + return + + inode = inodes[dent_node.inum] + + if not is_safe_path(path, dent_node.name): + error(extract_dents, 'Warn', 'Path traversal attempt: %s, discarding.' % (dent_node.name)) + return + dent_path = os.path.realpath(os.path.join(path, dent_node.name)) + + if dent_node.type == UBIFS_ITYPE_DIR: + try: + if not os.path.exists(dent_path): + os.mkdir(dent_path) + log(extract_dents, 'Make Dir: %s' % (dent_path)) + + if perms: + _set_file_perms(dent_path, inode) + except Exception as e: + error(extract_dents, 'Warn', 'DIR Fail: %s' % e) + + if 'dent' in inode: + for dnode in inode['dent']: + extract_dents(ubifs, inodes, dnode, dent_path, perms) + + _set_file_timestamps(dent_path, inode) + + elif dent_node.type == UBIFS_ITYPE_REG: + try: + if inode['ino'].nlink > 1: + if 'hlink' not in inode: + inode['hlink'] = dent_path + buf = _process_reg_file(ubifs, inode, dent_path) + _write_reg_file(dent_path, buf) + else: + os.link(inode['hlink'], dent_path) + log(extract_dents, 'Make Link: %s > %s' % (dent_path, inode['hlink'])) + else: + buf = _process_reg_file(ubifs, inode, dent_path) + _write_reg_file(dent_path, buf) + + _set_file_timestamps(dent_path, inode) + + if perms: + _set_file_perms(dent_path, inode) + + except Exception as e: + error(extract_dents, 'Warn', 'FILE Fail: %s' % e) + + elif dent_node.type == UBIFS_ITYPE_LNK: + try: + # probably will need to decompress ino data if > UBIFS_MIN_COMPR_LEN + os.symlink('%s' % inode['ino'].data.decode('utf-8'), dent_path) + log(extract_dents, 'Make Symlink: %s > %s' % (dent_path, inode['ino'].data)) + + except Exception as e: + error(extract_dents, 'Warn', 'SYMLINK Fail: %s' % e) + + elif dent_node.type in [UBIFS_ITYPE_BLK, UBIFS_ITYPE_CHR]: + try: + dev = struct.unpack(' len(buf): + buf += b'\x00' * (inode['ino'].size - len(buf)) + + return bytes(buf) diff --git a/ubireader/parsers/extract_images.py b/ubireader/parsers/extract_images.py index b453fa0..92e6fe1 100644 --- a/ubireader/parsers/extract_images.py +++ b/ubireader/parsers/extract_images.py @@ -1,48 +1,32 @@ import os -from ubireader.parsers import UIParser, create_output_dir +from ubireader.parsers import ArgHandler, makedir from ubireader.ubi import ubi from ubireader.ubi_io import ubi_file from ubireader import settings -# Set up argparser and cli options. -ui = UIParser() -ui.description = 'Extract UBI or UBIFS images from file containing UBI data in it.' -ui.usage = 'ubireader_extract_images [options] filepath' -ui.arg_log() -ui.arg_verbose_log() -ui.arg_peb_size() -ui.arg_image_type() -ui.arg_start_offset() -ui.arg_end_offset() -ui.arg_guess_offset() -ui.arg_warn_only_block_read_errors() -ui.arg_ignore_block_header_errors() -ui.arg_u_boot_fix() -ui.arg_output_dir() -ui.arg_filepath() -def extract_images(*args, **kwargs): - ui.parse_args(*args, **kwargs) +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) - img_name = os.path.basename(ui.filepath) - if ui.output_dir: - outpath = os.path.abspath(os.path.join(ui.output_dir, img_name)) + img_name = os.path.basename(args.filepath) + if args.output_dir: + outpath = os.path.abspath(args.output_dir) else: - outpath = os.path.join(settings.output_dir, img_name) + outpath = settings.output_dir # Create file object. - ufile_obj = ubi_file(ui.filepath, ui.block_size, ui.start_offset, ui.end_offset) + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) # Create UBI object ubi_obj = ubi(ufile_obj) # Loop through found images in file. for image in ubi_obj.images: - if ui.image_type == 'UBI': + if args.image_type == 'UBI': # Create output path and open file. - img_outpath = os.path.join(outpath, 'img-%s.ubi' % image.image_seq) - create_output_dir(outpath) + img_outpath = os.path.join(outpath, f'{img_name}-{image.image_seq}.ubi') + makedir(outpath) f = open(img_outpath, 'wb') # Loop through UBI image blocks @@ -51,18 +35,14 @@ def extract_images(*args, **kwargs): # Write block (PEB) to file f.write(ubi_obj.file.read_block(ubi_obj.blocks[block])) - elif ui.image_type == 'UBIFS': + elif args.image_type == 'UBIFS': # Loop through image volumes for volume in image.volumes: # Create output path and open file. - vol_outpath = os.path.join(outpath, 'img-%s_vol-%s.ubifs' % (image.image_seq, volume)) - create_output_dir(outpath) + vol_outpath = os.path.join(outpath, f'{img_name}-{image.image_seq}-{volume}.ubifs') + makedir(outpath) f = open(vol_outpath, 'wb') # Loop through and write volume block data (LEB) to file. for block in image.volumes[volume].reader(ubi_obj): - f.write(block) - - -# Add process function to parser. -ui.function = extract_images \ No newline at end of file + f.write(block) \ No newline at end of file diff --git a/ubireader/parsers/list_files.py b/ubireader/parsers/list_files.py index 3af2d02..80b277d 100644 --- a/ubireader/parsers/list_files.py +++ b/ubireader/parsers/list_files.py @@ -1,80 +1,30 @@ -import argparse -import os - -from ubireader import settings -from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import parser_base +from ubireader.parsers import ArgHandler from ubireader.ubi import ubi from ubireader.ubi.defines import UBI_EC_HDR_MAGIC from ubireader.ubifs import ubifs from ubireader.ubifs.list import list_files, copy_file from ubireader.ubifs.defines import UBIFS_NODE_MAGIC from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_leb_size, guess_peb_size - - -def list_files( - filepath, - log=False, - verbose=False, - block_size=None, - start_offset=None, - end_offset=None, - guess_offset=None, - warn_only_block_read_errors=False, - ignore_block_header_errors=False, - uboot_fix=False, - listpath=None, - copyfile=None, - copyfiledest=None - ): - settings.logging_on = log - - settings.logging_on_verbose = verbose - - settings.warn_only_block_read_errors = warn_only_block_read_errors - - settings.ignore_block_header_errors = ignore_block_header_errors - - settings.uboot_fix = uboot_fix - - if filepath: - path = filepath - if not os.path.exists(path): - raise UBIReaderParseError("File path doesn't exist.") - - if start_offset: - start_offset = start_offset - elif guess_offset: - start_offset = guess_start_offset(path, guess_offset) - else: - start_offset = guess_start_offset(path) - if end_offset: - end_offset = end_offset - else: - end_offset = None +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) - filetype = guess_filetype(path, start_offset) - if not filetype: - raise UBIReaderParseError('Could not determine file type.') + copyfile = False + if 'copyfile' in kwargs: + copyfile = kwargs['copyfile'] - if block_size: - block_size = block_size - else: - if filetype == UBI_EC_HDR_MAGIC: - block_size = guess_peb_size(path) - elif filetype == UBIFS_NODE_MAGIC: - block_size = guess_leb_size(path) + copyfiledest = None + if 'copyfiledest' in kwargs: + copyfiledest = kwargs['copyfiledest'] - if not block_size: - raise UBIReaderParseError('Block size could not be determined.') + listpath = None + if 'listpath' in kwargs: + listpath = kwargs['listpath'] # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + ufile_obj = ubi_file(args.path, args.block_size, args.start_offset, args.end_offset) - if filetype == UBI_EC_HDR_MAGIC: + if args.filetype == UBI_EC_HDR_MAGIC: # Create UBI object ubi_obj = ubi(ufile_obj) @@ -104,37 +54,15 @@ def list_files( if copyfile and copyfiledest: copy_file(ubifs_obj, copyfile, copyfiledest) - elif filetype == UBIFS_NODE_MAGIC: + elif args.filetype == UBIFS_NODE_MAGIC: # Create UBIFS object ubifs_obj = ubifs(ufile_obj) if listpath: list_files(ubifs_obj, listpath) + if copyfile and copyfiledest: copy_file(ubifs_obj, copyfile, copyfiledest) else: - print('Something went wrong to get here.') - - -description = 'List and Extract files of a UBI or UBIFS image.' -usage = 'ubireader_list_files [options] filepath' -parser = argparse.ArgumentParser( - usage=usage, - description=description, - parents=[parser_base], - add_help=False -) -parser.set_defaults(func=list_files) - -parser.add_argument('-e', '--leb-size', type=int, dest='block_size', - help='Specify LEB size. (UBIFS Only)') - -parser.add_argument('-P', '--path', dest='listpath', - help='Path to list.') - -parser.add_argument('-C', '--copy', dest='copyfile', - help='File to Copy.') - -parser.add_argument('-D', '--copy-dest', dest='copyfiledest', - help='Copy Destination.') + print('Something went wrong to get here.') \ No newline at end of file diff --git a/ubireader/parsers/utils_info.py b/ubireader/parsers/utils_info.py index 2b9c1eb..8c42e04 100644 --- a/ubireader/parsers/utils_info.py +++ b/ubireader/parsers/utils_info.py @@ -1,4 +1,3 @@ -import argparse import os import sys if (sys.version_info > (3, 0)): @@ -6,29 +5,12 @@ else: import ConfigParser as configparser -from ubireader import settings -from ubireader.exceptions import UBIReaderParseError -from ubireader.parsers import parser_base +from ubireader.parsers import ArgHandler, makedir from ubireader.ubi import ubi from ubireader.ubi.defines import UBI_EC_HDR_MAGIC, PRINT_VOL_TYPE_LIST, UBI_VTBL_AUTORESIZE_FLG from ubireader.ubifs import ubifs from ubireader.ubifs.defines import PRINT_UBIFS_KEY_HASH, PRINT_UBIFS_COMPR from ubireader.ubi_io import ubi_file, leb_virtual_file -from ubireader.debug import error, log -from ubireader.utils import guess_filetype, guess_start_offset, guess_peb_size - - -def create_output_dir(outpath): - if os.path.exists(outpath): - if os.listdir(outpath): - error(create_output_dir, 'Fatal', 'Output directory is not empty. %s' % outpath) - else: - try: - os.makedirs(outpath) - log(create_output_dir, 'Created output path: %s' % outpath) - except Exception as e: - error(create_output_dir, 'Fatal', '%s' % e) - def get_ubi_params(ubi_obj): """Get ubi_obj utils params @@ -221,67 +203,11 @@ def make_files(ubi, outpath): os.chmod(script_path, 0o755) -def utils_info( - filepath, - show_only=False, - log=False, - verbose=False, - block_size=None, - start_offset=None, - end_offset=None, - guess_offset=None, - warn_only_block_read_errors=False, - ignore_block_header_errors=False, - uboot_fix=False, - outpath=None - ): - settings.logging_on = log - - settings.logging_on_verbose = verbose - - settings.warn_only_block_read_errors = warn_only_block_read_errors - - settings.ignore_block_header_errors = ignore_block_header_errors - - settings.uboot_fix = uboot_fix - - if filepath: - path = filepath - if not os.path.exists(path): - raise UBIReaderParseError("File path doesn't exist.") - - if start_offset: - start_offset = start_offset - elif guess_offset: - start_offset = guess_start_offset(path, guess_offset) - else: - start_offset = guess_start_offset(path) - - if end_offset: - end_offset = end_offset - else: - end_offset = None - - filetype = guess_filetype(path, start_offset) - if filetype != UBI_EC_HDR_MAGIC: - raise UBIReaderParseError('File does not look like UBI data.') - - img_name = os.path.basename(path) - if outpath: - outpath = os.path.abspath(os.path.join(outpath, img_name)) - else: - outpath = os.path.join(settings.output_dir, img_name) - - if block_size: - block_size = block_size - else: - block_size = guess_peb_size(path) - - if not block_size: - raise UBIReaderParseError('Block size could not be determined.') +def parse(filepath, *args, **kwargs): + args = ArgHandler(filepath, *args, **kwargs) # Create file object. - ufile_obj = ubi_file(path, block_size, start_offset, end_offset) + ufile_obj = ubi_file(args.filepath, args.block_size, args.start_offset, args.end_offset) # Create UBI object ubi_obj = ubi(ufile_obj) @@ -289,24 +215,7 @@ def utils_info( # Print info. print_ubi_params(ubi_obj) - if not show_only: - create_output_dir(outpath) + if not kwargs['show_only']: + makedir(args.output_dir) # Create build scripts. - make_files(ubi_obj, outpath) - - -description = 'Determine settings for recreating UBI image.' -usage = 'ubireader_utils_info [options] filepath' -parser = argparse.ArgumentParser( - usage=usage, - description=description, - parents=[parser_base], - add_help=False -) -parser.set_defaults(func=utils_info) - -parser.add_argument('-r', '--show-only', action='store_true', dest='show_only', - help='Print parameters to screen only. (default: false)') - -parser.add_argument('-o', '--output-dir', dest='outpath', - help='Specify output directory path.') + make_files(ubi_obj, args.output_dir) \ No newline at end of file diff --git a/ubireader/ubifs/output.py b/ubireader/ubifs/output.py deleted file mode 100755 index b4aa653..0000000 --- a/ubireader/ubifs/output.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python -############################################################# -# ubi_reader/ubifs -# (c) 2013 Jason Pruitt (jrspruitt@gmail.com) -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -############################################################# - -import os -import struct - -from ubireader import settings -from ubireader.ubifs.defines import * -from ubireader.ubifs import walk -from ubireader.ubifs.misc import decompress -from ubireader.debug import error, log, verbose_log - -def is_safe_path(basedir, path): - basedir = os.path.realpath(basedir) - path = os.path.realpath(os.path.join(basedir, path)) - return True if path.startswith(basedir) else False - - -def extract_files(ubifs, out_path, perms=False): - """Extract UBIFS contents to_path/ - - Arguments: - Obj:ubifs -- UBIFS object. - Str:out_path -- Path to extract contents to. - """ - try: - inodes = {} - bad_blocks = [] - - walk.index(ubifs, ubifs.master_node.root_lnum, ubifs.master_node.root_offs, inodes, bad_blocks) - - if len(inodes) < 2: - raise Exception('No inodes found') - - for dent in inodes[1]['dent']: - extract_dents(ubifs, inodes, dent, out_path, perms) - - if len(bad_blocks): - error(extract_files, 'Warn', 'Data may be missing or corrupted, bad blocks, LEB [%s]' % ','.join(map(str, bad_blocks))) - - except Exception as e: - error(extract_files, 'Error', '%s' % e) - - -def extract_dents(ubifs, inodes, dent_node, path='', perms=False): - if dent_node.inum not in inodes: - error(extract_dents, 'Error', 'inum: %s not found in inodes' % (dent_node.inum)) - return - - inode = inodes[dent_node.inum] - - if not is_safe_path(path, dent_node.name): - error(extract_dents, 'Warn', 'Path traversal attempt: %s, discarding.' % (dent_node.name)) - return - dent_path = os.path.realpath(os.path.join(path, dent_node.name)) - - if dent_node.type == UBIFS_ITYPE_DIR: - try: - if not os.path.exists(dent_path): - os.mkdir(dent_path) - log(extract_dents, 'Make Dir: %s' % (dent_path)) - - if perms: - _set_file_perms(dent_path, inode) - except Exception as e: - error(extract_dents, 'Warn', 'DIR Fail: %s' % e) - - if 'dent' in inode: - for dnode in inode['dent']: - extract_dents(ubifs, inodes, dnode, dent_path, perms) - - _set_file_timestamps(dent_path, inode) - - elif dent_node.type == UBIFS_ITYPE_REG: - try: - if inode['ino'].nlink > 1: - if 'hlink' not in inode: - inode['hlink'] = dent_path - buf = _process_reg_file(ubifs, inode, dent_path) - _write_reg_file(dent_path, buf) - else: - os.link(inode['hlink'], dent_path) - log(extract_dents, 'Make Link: %s > %s' % (dent_path, inode['hlink'])) - else: - buf = _process_reg_file(ubifs, inode, dent_path) - _write_reg_file(dent_path, buf) - - _set_file_timestamps(dent_path, inode) - - if perms: - _set_file_perms(dent_path, inode) - - except Exception as e: - error(extract_dents, 'Warn', 'FILE Fail: %s' % e) - - elif dent_node.type == UBIFS_ITYPE_LNK: - try: - # probably will need to decompress ino data if > UBIFS_MIN_COMPR_LEN - os.symlink('%s' % inode['ino'].data.decode('utf-8'), dent_path) - log(extract_dents, 'Make Symlink: %s > %s' % (dent_path, inode['ino'].data)) - - except Exception as e: - error(extract_dents, 'Warn', 'SYMLINK Fail: %s' % e) - - elif dent_node.type in [UBIFS_ITYPE_BLK, UBIFS_ITYPE_CHR]: - try: - dev = struct.unpack(' len(buf): - buf += b'\x00' * (inode['ino'].size - len(buf)) - - return bytes(buf)