diff --git a/README.md b/README.md index d742677..ad73da5 100644 --- a/README.md +++ b/README.md @@ -127,5 +127,31 @@ root@system:~/Desktop/page/page_brute/PAGE_BRUTE-2013-10-27-01:20:28-RESULTS/web 00000e0: 2020 3c74 6420 7374 796c 653d 2270 6164 > $outfile +done + +sort -t $'\t' -k2 -n allpage.lst > $outfile$sort + +cat allpage.lst.sort | awk -F '\t' '{print $2}' | sort | uniq -d > allpage.repeats +cat allpage.lst.sort | awk -F '\t' '{print $2}' | sort | uniq -c > allpage.uniq + +# extract repeated pageid togher by its rule name to cross reference analysis of the page blocks +python rules_IDrepeats_map.py allpage.repeats allpage.lst.sort > allpage.repeats.map +python rules_IDrepeats_map.py allpage.uniq allpage.lst.sort > allpage.uniq.map diff --git a/page_brute-BETA.py b/page_brute-BETA.py index 2c03f79..75fb8df 100755 --- a/page_brute-BETA.py +++ b/page_brute-BETA.py @@ -13,215 +13,251 @@ import binascii try: - import yara + import yara # main module except: print "[!] - ERROR: Could not import YARA..." print "...did you install yara and yara-python? Exiting." sys.exit() - +# check whether the block is null/zero out or not +# if so, skip. def is_block_null(block): - #Here we test to see if the block is null..if so, skip. - RAW_BLOCK=binascii.hexlify(block) - NULL_REF=binascii.hexlify(NULL_REFERENCE) - if RAW_BLOCK == NULL_REF: - return True - else: - return False + raw_block = binascii.hexlify(block) + null_ref = binascii.hexlify(NULL_REFERENCE) + + if raw_block == null_ref: + return True + else: + return False +# compile yara rules def build_ruleset(): - if RULETYPE == "FILE": - try: - rules=yara.compile(str(RULES)) - print "..... Ruleset Compilation Successful." - return rules - except: - print "[!] - Could not compile YARA rule: %s" % RULES - print "Exiting." - sys.exit() - - elif RULETYPE == "FOLDER": - RULEDATA="" - #::Get list of files ending in .yara - - RULE_COUNT = len(glob.glob1(RULES,"*.yar")) - if RULE_COUNT != 0: - for yara_file in glob.glob(os.path.join(RULES, "*.yar")): - try: - yara.compile(str(yara_file)) - print "..... Syntax appears to be OK: %s " % yara_file - try: - with open(yara_file, "r") as sig_file: - file_contents=sig_file.read() - RULEDATA=RULEDATA + "\n" + file_contents - except: - print "..... SKIPPING: Could not open file for reading: %s " % yara_file - except: - print "..... SKIPPING: Could not compile rule: %s " % yara_file - try: - rules=yara.compile(source=RULEDATA) - print "..... SUCCESS! Compiled noted yara rulesets.\n" - return rules - except: - print "[!] - Some catastropic error occurred in the compilation of signatureswithin the directory. Exiting." - sys.exit() - else: - print "No files ending in .yar within: %s " % RULES - print "Exiting." - sys.exit() - - elif RULETYPE == "DEFAULT": - rules=yara.compile(str(RULES)) - print "[+] - Ruleset Compilation Successful." - return rules - - else: - print "[!] - ERROR: Possible catastrophic error on build_ruleset. Exiting." - sys.exit() + if ruletype == "FILE": + try: + rules = yara.compile(str(RULES)) + print "..... Ruleset Compilation Successful." + return rules + except: + print "[!] - Could not compile YARA rule: %s" % RULES + print "Exiting." + sys.exit() + + elif ruletype == "FOLDER": + RULEDATA="" + # Get list of files ending in .yara + + RULE_COUNT = len(glob.glob1(RULES,"*.yar")) + if RULE_COUNT != 0: + for yara_file in glob.glob(os.path.join(RULES, "*.yar")): + try: + yara.compile(str(yara_file)) + print "..... Syntax appears to be OK: %s " % yara_file + try: + with open(yara_file, "r") as sig_file: + file_contents=sig_file.read() + RULEDATA=RULEDATA + "\n" + file_contents + except: + print "..... SKIPPING: Could not open file for reading: %s " % yara_file + except: + print "..... SKIPPING: Could not compile rule: %s " % yara_file + try: + rules = yara.compile(source=RULEDATA) + print "..... SUCCESS! Compiled noted yara rulesets.\n" + return rules + except: + print "[!] - Some catastropic error occurred in the " \ + "compilation of signatureswithin the directory. Exiting." + sys.exit() + else: + print "No files ending in .yar within: %s " % RULES + print "Exiting." + sys.exit() + + elif ruletype == "DEFAULT": + rules = yara.compile(str(RULES)) + print "[+] - Ruleset Compilation Successful." + return rules + + else: + print "[!] - ERROR: Possible catastrophic error on build_ruleset. Exiting." + sys.exit() +# print stdout messages def print_procedures(): - print "[+] - PAGE_BRUTE running with the following options:" - print "\t[-] - FILE: %s" % FILE - print "\t[-] - PAGE_SIZE: %s" % PAGE_SIZE - print "\t[-] - RULES TYPE: %s" % RULETYPE - print "\t[-] - RULE LOCATION: %s" % RULES - print "\t[-] - INVERSION SCAN: %s" % INVERT - print "\t[-] - WORKING DIR: %s" % WORKING_DIR - print "\t=================\n" + print "[+] - PAGE_BRUTE running with the following options:" + print "\t[-] - FILE: %s" % FILE + print "\t[-] - PAGE_SIZE: %s" % PAGE_SIZE + print "\t[-] - RULES TYPE: %s" % ruletype + print "\t[-] - RULE LOCATION: %s" % RULES + print "\t[-] - INVERSION SCAN: %s" % invert + print "\t[-] - WORKING DIR: %s" % WORKING_DIR + print "\t=================\n" + def main(): - global FILE - global PAGE_SIZE - global RULES - global SCANNAME - global INVERT - global RULETYPE - global NULL_REFERENCE - - argument_parser = argparse.ArgumentParser(description="Checks pages in pagefiles for YARA-based rule matches. Useful to identify forensic artifacts within Windows-based page files and characterize blocks based on regular expressions.") - - group_arg = argument_parser.add_argument_group() - group_arg.add_argument("-f", "--file", metavar="FILE", help="Pagefile or any chunk/block-based binary file") - group_arg.add_argument("-p", "--size", metavar="SIZE", help="Size of chunk/block in bytes (Default 4096)") - group_arg.add_argument("-o", "--scanname", metavar="SCANNAME", help="Descriptor of the scan session - used for output directory") - group_arg.add_argument("-i", "--invert", help="Given scan options, match all blocks that DO NOT match a ruleset",action='store_true') - - group_arg = argument_parser.add_mutually_exclusive_group() - group_arg.add_argument("-r", "--rules", metavar="RULEFILE", help="File/directory containing YARA signatures (must end with .yar)") - - args = argument_parser.parse_args() - - if len(sys.argv) < 2: - print argument_parser.print_help() - sys.exit() - - #::Check to see if file was provided::# - if args.file: - try: - with open(args.file): - FILE=args.file - print "[+] - PAGE_BRUTE processing file: %s" % FILE - except: - print "[!] - Could not open %s. Exiting." % FILE - sys.exit() - else: - print "[!] - No file provided. Use -f, --file to provide a file. Exiting." - sys.exit() - - #::Check to see if page size provided::# - if args.size: - PAGE_SIZE=int(args.size) - NULL_REFERENCE= '\x00' * PAGE_SIZE - else: - PAGE_SIZE=4096 - NULL_REFERENCE= '\x00' * PAGE_SIZE - - #::Check if --scan-name provided::# - if args.scanname: - SCANNAME=args.scanname - else: - SCANNAME="PAGE_BRUTE-" + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "-RESULTS" - - #::Check if --invert-match provided::# - if args.invert: - INVERT=True - else: - INVERT=False - - #::Check if --rule-file provdided - if not, use default ruleset::# - if args.rules: - RULES=args.rules - try: - #::Is File?::# - if os.path.isfile(RULES): - RULETYPE="FILE" - print "[+] - YARA rule of File type provided for compilation: %s" % RULES - elif os.path.isdir(RULES): - print "[+] - YARA rule of Folder type provided for compilation: %s" % RULES - RULETYPE="FOLDER" - except: - print "[!] - Possible catastrophic error with the provided rule file...exiting." - sys.exit() - else: - try: - with open("default_signatures.yar"): - RULES="default_signatures.yar" - RULETYPE="DEFAULT" - except: - print "[!] - Could not locate \"default_signature.yar\". Find it or provide custom signatures via --rules. Exiting." - sys.exit() - - #::Compile rules::# - authoritative_rules=build_ruleset() - #::Build directory structure - global WORKING_DIR - WORKING_DIR=SCANNAME - if not os.path.exists(WORKING_DIR): - os.makedirs(WORKING_DIR) - #::Let People Know what we're doing::# - print_procedures() - #::Find Evil::# - page_id=0 - with open(FILE, "rb") as page_file: - while True: - matched=False - raw_page=page_file.read(PAGE_SIZE) - if raw_page == "": - print "Done!" - print "Ending page_id is: %s" % page_id - break - if not is_block_null(raw_page): - #::Determine if block is null...: - for matches in authoritative_rules.match(data=raw_page): - if INVERT == True: - matched=True - else: - CHUNK_OUTPUT_DIR=os.path.join(WORKING_DIR,matches.rule) - print " [!] FLAGGED BLOCK " + str(page_id) + ": " + matches.rule - - if not os.path.exists(CHUNK_OUTPUT_DIR): - os.makedirs(CHUNK_OUTPUT_DIR) - - #::Save chunk to file::# - CHUNK_OUTPUT_FWD=os.path.join(CHUNK_OUTPUT_DIR,str(page_id) + ".block") - page_export=open(CHUNK_OUTPUT_FWD,'w+') - page_export.write(raw_page) - page_export.close() - - if INVERT == True: - if matched == False: - CHUNK_OUTPUT_DIR=os.path.join(WORKING_DIR,"INVERTED-MATCH") - print " [!] BLOCK DOES NOT MATCH ANY KNOWN SIGNATURE " + str(page_id) - if not os.path.exists(CHUNK_OUTPUT_DIR): - os.makedirs(CHUNK_OUTPUT_DIR) - - CHUNK_OUTPUT_FWD=os.path.join(CHUNK_OUTPUT_DIR,str(page_id) + ".block") - page_export=open(CHUNK_OUTPUT_FWD,'w+') - page_export.write(raw_page) - page_export.close() - #::Increment Counter for offset increment::# - page_id=page_id+1 + global FILE + global PAGE_SIZE + global RULES + global SCANNAME + global invert + global ruletype + global NULL_REFERENCE + + # parse input arguments + argument_parser = argparse.ArgumentParser(description="Checks pages in pagefiles for YARA-based rule matches." + " Useful to identify forensic artifacts within Windows-based page files and" + " characterize blocks based on regular expressions.") + group_arg = argument_parser.add_argument_group() + group_arg.add_argument("-f", "--file", metavar="FILE", help="Pagefile or any chunk/block-based binary file") + group_arg.add_argument("-p", "--size", metavar="SIZE", help="Size of chunk/block in bytes (Default 4096)") + group_arg.add_argument("-o", "--scanname", metavar="SCANNAME", help="Descriptor of the scan session - used for output directory") + group_arg.add_argument("-i", "--invert", help="Given scan options, match all blocks that DO NOT match a ruleset",action='store_true') + + group_arg = argument_parser.add_mutually_exclusive_group() + group_arg.add_argument("-r", "--rules", metavar="RULEFILE", help="File/directory containing YARA signatures (must end with .yar)") + args = argument_parser.parse_args() + + if len(sys.argv) < 2: + print argument_parser.print_help() + sys.exit() + + # validate input file + if args.file: + try: + with open(args.file, 'rb'): + FILE = args.file + print "[+] - PAGE_BRUTE processing file: %s" % FILE + except: + print "[!] - Could not open %s. Exiting." % FILE + sys.exit() + else: + print "[!] - No file provided. Use -f, --file to provide a file. Exiting." + sys.exit() + + + # Check to see if page size provided + if args.size: + PAGE_SIZE = int(args.size) + NULL_REFERENCE = '\x00' * PAGE_SIZE + else: + PAGE_SIZE = 4096 + NULL_REFERENCE = '\x00' * PAGE_SIZE + + # Check if --scan-name provided + if args.scanname: + SCANNAME = args.scanname + else: + SCANNAME = "PAGE_BRUTE-" + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "-RESULTS" + + # Check if --invert-match provided + if args.invert: + invert = True + else: + invert = False + + # Check if --rule-file provdided - if not, use default ruleset + if args.rules: + RULES = args.rules + try: + # Is File? + if os.path.isfile(RULES): + ruletype = "FILE" + print "[+] - YARA rule of File type provided for compilation: %s" % RULES + elif os.path.isdir(RULES): + print "[+] - YARA rule of Folder type provided for compilation: %s" % RULES + ruletype="FOLDER" + except: + print "[!] - Possible catastrophic error with the provided rule file...exiting." + sys.exit() + else: + try: + with open("default_signatures.yar"): + RULES = "default_signatures.yar" + ruletype = "DEFAULT" + except: + print "[!] - Could not locate \"default_signature.yar\". Find it or provide custom signatures via --rules. Exiting." + sys.exit() + + # Compile rules + authoritative_rules=build_ruleset() + # Build directory structure + global WORKING_DIR + WORKING_DIR = SCANNAME + if not os.path.exists(WORKING_DIR): + os.makedirs(WORKING_DIR) + # Let People Know what we're doing + print_procedures() + # Find Evil + page_id = 0 + page_list = {} + page_block = {} + with open(FILE, "rb") as page_file: + while True: + matched = False + raw_page = page_file.read(PAGE_SIZE) + if raw_page == "": + print "Done!" + print "Ending page_id is: %s" % page_id + break + if not is_block_null(raw_page): + # Determine if block is null + for matches in authoritative_rules.match(data=raw_page): + if invert == True: + matched = True + else: + chunk_output_dir = os.path.join(WORKING_DIR,matches.rule) + print " [!] FLAGGED BLOCK " + str(page_id) + ": " + matches.rule + + # save page id with matched rules to dictionary page_list + if matches.rule not in page_list: + page_list[matches.rule] = str(page_id) + else: + page_list[matches.rule] += ";"+str(page_id) + page_block[page_id] = raw_page + if not os.path.exists(chunk_output_dir): + os.makedirs(chunk_output_dir) + + # Save chunk to file + chunk_output_fwd = os.path.join(chunk_output_dir,str(page_id) + ".block") + page_export = open(chunk_output_fwd,'w+') + page_export.write(raw_page) + page_export.close() + + if invert == True: + if matched == False: + chunk_output_dir = os.path.join(WORKING_DIR,"INVERTED-MATCH") + print " [!] BLOCK DOES NOT MATCH ANY KNOWN SIGNATURE " + str(page_id) + if not os.path.exists(chunk_output_dir): + os.makedirs(chunk_output_dir) + + chunk_output_fwd = os.path.join(chunk_output_dir,str(page_id) + ".block") + page_export = open(chunk_output_fwd,'w+') + page_export.write(raw_page) + page_export.close() + + # Increment Counter for offset increment + page_id = page_id+1 + + # write to a file as rules and pageid for further analysis + o = open(SCANNAME+"_pagelist", 'w') + # write page id with yara rule names with its corresponding data block + o2 = open(SCANNAME+"_pageblocks", 'w') + page_list2 = [] + scan_name = [] + + for k,v in page_list.iteritems(): + for m in v.split(';'): + page_list2.append(m) + scan_name.append(k) + o.write(k+'\t'+m+"\n") + for k2,v2 in page_block.iteritems(): + hitrule = "" + if str(k2) in page_list2: + hitrule = scan_name[page_list2.index(str(k2))] + o2.write(">"+hitrule+"-"+str(k2)+'\n') + o2.write(str(v2)+'\n') + if __name__ == "__main__": main() diff --git a/rules_IDrepeats_map.py b/rules_IDrepeats_map.py new file mode 100644 index 0000000..18a1db6 --- /dev/null +++ b/rules_IDrepeats_map.py @@ -0,0 +1,33 @@ +#! /usr/bin/env/ python + +""" +# +# usage: python rules_IDrepeats_map.py allpage.repeats allpage.lst.sort > allpage.repeats.map +# output: +# Dev: __author__ = 'aung' +# Date: 20151019 +""" +import sys +repeatlist = sys.argv[1] +allpage = sys.argv[2] + +page_rule = [] +page_id = [] +with open(allpage,'rb') as f1: + for l1 in f1: + l1_split = l1.split('\t') + page_rule.append(l1_split[0]) + page_id.append(l1_split[1].strip('\n')) + +with open(repeatlist,'rb') as f2: + for l2 in f2: + repid = l2.strip('\n') + ind = [i for i, e in enumerate(page_id) if e == repid] + rules= "" + for i in ind: + if rules == "": + rules = page_rule[i] + else: + rules += ";"+page_rule[i] + sys.stdout.write(repid+'\t'+rules+'\n') +