From ee63fd81041db1dc6e1b8b4de48627792ab09a91 Mon Sep 17 00:00:00 2001 From: aungthurhahein Date: Mon, 19 Oct 2015 16:19:44 +0700 Subject: [PATCH 1/3] reporting and style updated --- default_signatures.yar | 564 ++++++++++++++++++++--------------------- page_brute-BETA.py | 427 +++++++++++++++++-------------- 2 files changed, 513 insertions(+), 478 deletions(-) diff --git a/default_signatures.yar b/default_signatures.yar index f809cc2..303928b 100644 --- a/default_signatures.yar +++ b/default_signatures.yar @@ -14,334 +14,334 @@ rule administrative_share_abuse rule remote_system_syntax { - meta: - author = "@matonis" - info = "Command syntax that is used to access remote systems by IP address" - strings: - $s1 = /\\\\\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/ - condition: - $s1 + meta: + author = "@matonis" + info = "Command syntax that is used to access remote systems by IP address" + strings: + $s1 = /\\\\\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/ + condition: + $s1 } rule http_request_header { - meta: - author="@matonis" - description="HTTP header fields" - strings: - //methods - $method0 = "OPTIONS" - $method1 = "GET" - $method2 = "HEAD" - $method3 = "POST" - $method4 = "PUT" - $method5 = "DELETE" - $method6 = "TRACE" - $method7 = "CONNECT" + meta: + author="@matonis" + description="HTTP header fields" + strings: + //methods + $method0 = "OPTIONS" + $method1 = "GET" + $method2 = "HEAD" + $method3 = "POST" + $method4 = "PUT" + $method5 = "DELETE" + $method6 = "TRACE" + $method7 = "CONNECT" - //http version - $version0 = "HTTP/1.1" - $version1 = "HTTP/1.0" + //http version + $version0 = "HTTP/1.1" + $version1 = "HTTP/1.0" - //headers - $header0 = "Host: " - $header1 = "User-Agent: " - $header2 = "Content-Encoding: " - $header3 = "Last-Modified: " - $header4 = "Expires: " - $header5 = "Connection: " - $header6 = "Accept-Language: " - $header7 = "Accept-Encoding: " - $header8 = "Accet-Charset: " - $header9 = "Cookie: " - $header10 = "Content-Length: " - $header11 = "Accept: " - condition: - (1 of ($method*)) and (1 of ($version*)) and (2 of ($header*)) + //headers + $header0 = "Host: " + $header1 = "User-Agent: " + $header2 = "Content-Encoding: " + $header3 = "Last-Modified: " + $header4 = "Expires: " + $header5 = "Connection: " + $header6 = "Accept-Language: " + $header7 = "Accept-Encoding: " + $header8 = "Accet-Charset: " + $header9 = "Cookie: " + $header10 = "Content-Length: " + $header11 = "Accept: " + condition: + (1 of ($method*)) and (1 of ($version*)) and (2 of ($header*)) } rule http_response_header { - meta: - author="@matonis" - description="HTTP Response headers" - strings: - //Response Codes - $response0 = "200 OK" - $response1 = "201 Created" - $response2 = "202 Accepted" - $response3 = "203 Non-Authoritative Information" - $response4 = "204 No Content" - $response5 = "205 Reset Content" - $response6 = "206 Partial Content" - $response7 = "300 Multiple Choices" - $response8 = "301 Moved Permanently" - $response9 = "302 Found" - $response10 = "303 See Other" - $response11 = "304 Not Modified" - $response12 = "305 Use Proxy" - $response13 = "307 Temporary Redirect" - $response14 = "400 Bad REQUEST" - $response15 = "401 Unauthorized" - $response16 = "403 Forbidden" - $response17 = "404 Not Found" - $response18 = "405 Method Not Allowed" - $response19 = "406 Not Acceptable" - $response20 = "407 Proxy Authentication Require" - $response21 = "408 Request Timeout" - $response22 = "409 Conflict" - $response23 = "410 Gone" - $response24 = "411 Length Required" - $response25 = "412 Precondition Failed" - $response26 = "413 Request Entity Too Large" - $response27 = "414 Request-URI Too Long" - $response28 = "415 Unsupported Media Type" - $response29 = "416 Requested Range Not Satisfiable" - $response30 = "417 Expectation Failed" - $response31 = "500 Internal Server Error" - $response32 = "501 Not Implemented" - $response33 = "502 Bad Gateway" - $response34 = "503 Service Unavailable" - $response35 = "504 Gateway Timeout" - $response36 = "505 HTTP Version Not Supported" + meta: + author="@matonis" + description="HTTP Response headers" + strings: + //Response Codes + $response0 = "200 OK" + $response1 = "201 Created" + $response2 = "202 Accepted" + $response3 = "203 Non-Authoritative Information" + $response4 = "204 No Content" + $response5 = "205 Reset Content" + $response6 = "206 Partial Content" + $response7 = "300 Multiple Choices" + $response8 = "301 Moved Permanently" + $response9 = "302 Found" + $response10 = "303 See Other" + $response11 = "304 Not Modified" + $response12 = "305 Use Proxy" + $response13 = "307 Temporary Redirect" + $response14 = "400 Bad REQUEST" + $response15 = "401 Unauthorized" + $response16 = "403 Forbidden" + $response17 = "404 Not Found" + $response18 = "405 Method Not Allowed" + $response19 = "406 Not Acceptable" + $response20 = "407 Proxy Authentication Require" + $response21 = "408 Request Timeout" + $response22 = "409 Conflict" + $response23 = "410 Gone" + $response24 = "411 Length Required" + $response25 = "412 Precondition Failed" + $response26 = "413 Request Entity Too Large" + $response27 = "414 Request-URI Too Long" + $response28 = "415 Unsupported Media Type" + $response29 = "416 Requested Range Not Satisfiable" + $response30 = "417 Expectation Failed" + $response31 = "500 Internal Server Error" + $response32 = "501 Not Implemented" + $response33 = "502 Bad Gateway" + $response34 = "503 Service Unavailable" + $response35 = "504 Gateway Timeout" + $response36 = "505 HTTP Version Not Supported" - //HTTP Versions - $version0 = "HTTP/1.1" - $version1 = "HTTP/1.0" + //HTTP Versions + $version0 = "HTTP/1.1" + $version1 = "HTTP/1.0" - //headers - $field0 = "Set-Cookie:" - $field1 = "Content-Type:" - $field2 = "X-Powered-By:" - $field3 = "Vary:" - $field4 = "Transfer-Encoding:" - $field5 = "Etag:" - $field6 = "Date:" - $field7 = "Server:" - $field8 = "Cache-Control:" - $field9 = "Connection:" - $field10 = "Last-Modified:" - condition: - (1 of ($response*)) and (1 of ($version*)) and (2 of ($field*)) + //headers + $field0 = "Set-Cookie:" + $field1 = "Content-Type:" + $field2 = "X-Powered-By:" + $field3 = "Vary:" + $field4 = "Transfer-Encoding:" + $field5 = "Etag:" + $field6 = "Date:" + $field7 = "Server:" + $field8 = "Cache-Control:" + $field9 = "Connection:" + $field10 = "Last-Modified:" + condition: + (1 of ($response*)) and (1 of ($version*)) and (2 of ($field*)) } rule webartifact_html { - meta: - author="@matonis" - description="HTML identifiers" - strings: - //sepcific tags - $html0 = "DOCTYPE" - $html1 = "head>" - $html2 = "body>" - $html3 = "title>" - $html4 = "body>" - $html5 = "html>" - $html6 = "" - $html7 = "" - $html9 = "br>" - $html10 = "script>" + meta: + author="@matonis" + description="HTML identifiers" + strings: + //sepcific tags + $html0 = "DOCTYPE" + $html1 = "head>" + $html2 = "body>" + $html3 = "title>" + $html4 = "body>" + $html5 = "html>" + $html6 = "" + $html7 = "" + $html9 = "br>" + $html10 = "script>" - condition: - 2 of them + condition: + 2 of them } rule webartifact_javascript { - meta: - author="@matonis" - description="Javascript signature" - strings: - $java0 = "document.write" nocase - $java1 = "createElement" nocase - $java2 = "getElementsByTagName" nocase - $java3 = "appendChild" nocase - $java4 = "eval" nocase - $java5 = "document.cookie" nocase - $java6 = "p,a,c,k,e,d" nocase - $java7 = ".substring" - condition: - 3 of them + meta: + author="@matonis" + description="Javascript signature" + strings: + $java0 = "document.write" nocase + $java1 = "createElement" nocase + $java2 = "getElementsByTagName" nocase + $java3 = "appendChild" nocase + $java4 = "eval" nocase + $java5 = "document.cookie" nocase + $java6 = "p,a,c,k,e,d" nocase + $java7 = ".substring" + condition: + 3 of them } rule cmdshell { - meta: - author="@matonis" - description="Command prompt syntax to identify potential priv escalation" - strings: - $cmd0 = "C:\\Documents and Settings\\Administrator" - $cmd2 = "C:\\Users\\Administrator" - condition: - any of them + meta: + author="@matonis" + description="Command prompt syntax to identify potential priv escalation" + strings: + $cmd0 = "C:\\Documents and Settings\\Administrator" + $cmd2 = "C:\\Users\\Administrator" + condition: + any of them } rule webartifact_gmail { - meta: - author="@matonis" - description="Gmail artifacts" - strings: - $s1 = "[\"ms\"," - $s2 = "[\"ce\"]" - $s3 = "[\"e\"" - condition: - 2 of them + meta: + author="@matonis" + description="Gmail artifacts" + strings: + $s1 = "[\"ms\"," + $s2 = "[\"ce\"]" + $s3 = "[\"e\"" + condition: + 2 of them } rule social_security_syntax { - meta: - author="@matonis" - description="SSN Syntax" - strings: - $s1 = /[0-9]{3}-[0-9]{2}-[0-9]{3}/ - condition: - $s1 + meta: + author="@matonis" + description="SSN Syntax" + strings: + $s1 = /[0-9]{3}-[0-9]{2}-[0-9]{3}/ + condition: + $s1 } rule smtp_fragments { - meta: - author="@matonis" - description="SMTP Artifacts" - strings: - $stmp0 = "HELO" - $stmp1 = "MAIL FROM" - $stmp2 = "RCPT TO" - $stmp4 = "From:" - $stmp5 = "To:" - $stmp6 = "Cc:" - $stmp7 = "Date:" - $stmp8 = "Subject:" - $stmp9 = "Delivered-To:" - $stmp10 = "Received: by" - $stmp11 = "Authentication-Results:" - $stmp12 = "Return-Path:" - $stmp13 = "Message-ID:" - $stmp14 = "Content-Transfer-Encoding:" - $stmp15 = "Content-Disposition:" - $stmp16 = "X-Forwarded-To:" - $stmp17 = "X-Forwarded-For:" - condition: - 7 of them + meta: + author="@matonis" + description="SMTP Artifacts" + strings: + $stmp0 = "HELO" + $stmp1 = "MAIL FROM" + $stmp2 = "RCPT TO" + $stmp4 = "From:" + $stmp5 = "To:" + $stmp6 = "Cc:" + $stmp7 = "Date:" + $stmp8 = "Subject:" + $stmp9 = "Delivered-To:" + $stmp10 = "Received: by" + $stmp11 = "Authentication-Results:" + $stmp12 = "Return-Path:" + $stmp13 = "Message-ID:" + $stmp14 = "Content-Transfer-Encoding:" + $stmp15 = "Content-Disposition:" + $stmp16 = "X-Forwarded-To:" + $stmp17 = "X-Forwarded-For:" + condition: + 7 of them } rule irc { - meta: - author="@matonis" - description="IRC Artifacts" - strings: - $irc0="has joined #" - $irc1 = "Channel created on" - $irc2 = "USER" - $irc3 = "PASS" - $irc5 = "NICK" - $irc6 = "CHANNEL" - $irc7 = /are [0-9]* users and [0-9]* invisible on/ - $irc8 = /[0-9]* operator(s) online/ + meta: + author="@matonis" + description="IRC Artifacts" + strings: + $irc0="has joined #" + $irc1 = "Channel created on" + $irc2 = "USER" + $irc3 = "PASS" + $irc5 = "NICK" + $irc6 = "CHANNEL" + $irc7 = /are [0-9]* users and [0-9]* invisible on/ + $irc8 = /[0-9]* operator(s) online/ - condition: - $irc0 or ($irc2 and $irc3 and $irc5 and $irc6) or $irc7 or $irc8 or $irc1 + condition: + $irc0 or ($irc2 and $irc3 and $irc5 and $irc6) or $irc7 or $irc8 or $irc1 } rule ftp { - meta: - author="@matonis" - description="FTP Command Artifacts" - strings: - $ftp1 = "150 File status okay; about to open data connection." - $ftp2 = "150 Opening BINARY mode data connection for" - $ftp3 = "150 Opening data connection." - $ftp4 = "200 Command PORT okay." - $ftp5 = "200 Command PROT okay." - $ftp6 = "200 Command SITE okay." - $ftp7 = "200 Command okay." - $ftp8 = "200 EPRT command okay." - $ftp9 = "200 Goodbye." - $ftp10 = "200 PORT command successful." - $ftp11 = "202 Already logged-in." - $ftp12 = "202 Command ACCT not implemented." - $ftp13 = "214 Help information." - $ftp14 = "221 Goodbye." - $ftp15 = "221 List of all the extensions supported." - $ftp16 = "226 ABOR command successful." - $ftp17 = "226 Closing data connection." - $ftp18 = "226 Transfer complete." - $ftp19 = "229 Entering passive mode" - $ftp20 = "230 Already logged-in." - $ftp21 = "230 User logged in, proceed." - $ftp22 = "234 AUTH command okay; starting SSL connection." - $ftp23 = "Transfer started." - $ftp24 = "250 Command okay." - $ftp25 = "250 Directory created." - $ftp26 = "250 Directory removed." - $ftp27 = "250 Requested file action okay, file renamed." - $ftp28 = "331 Guest login okay, send your complete e-mail address as password." - $ftp29 = "331 User name okay, need password." - $ftp30 = "350 Requested file action pending further information." - $ftp31 = "421 Maximum anonymous login limit has been reached." - $ftp32 = "421 Maximum login limit has been reached." - $ftp33 = "425 Can't open data connection." - $ftp34 = "425 Cannot open data connection." - $ftp35 = "425 Cannot open passive connection." - $ftp36 = "425 Cannot open the data connection." - $ftp37 = "426 Data connection error." - $ftp38 = "431 Security is disabled." - $ftp39 = "431 Service is unavailable." - $ftp40 = "450 Can't delete file." - $ftp41 = "450 No permission to delete." - $ftp42 = "500 Execution failed." - $ftp43 = "501 Syntax error in parameters or arguments." - $ftp44 = "501 Syntax error." - $ftp45 = "502 Command SITE not implemented for this argument." - $ftp46 = "502 Not yet implemented." - $ftp47 = "503 Cannot find the file which has to be renamed." - $ftp48 = "503 Login with USER first." - $ftp49 = "504 Command not implemented." - $ftp50 = "504 Not implemented for this command." - $ftp51 = "504 Server does not understand the specified protection level." - $ftp52 = "510 EPRT IP is not same as client IP." - $ftp53 = "510 EPRT is disabled." - $ftp54 = "510 PORT IP mismatch." - $ftp55 = "510 Port is disabled." - $ftp56 = "510 Syntax error in parameters." - $ftp57 = "510 Syntax error." - $ftp58 = "530 Access denied." - $ftp59 = "530 Anonymous connection is not allowed." - $ftp60 = "530 Authentication failed." - $ftp61 = "530 Invalid user name." - $ftp62 = "550 Already exists." - $ftp63 = "550 Cannot create directory." - $ftp64 = "550 Cannot remove directory." - $ftp65 = "550 File unavailable." - $ftp66 = "550 Invalid path." - $ftp67 = "550 No permission." - $ftp68 = "550 No such directory." - $ftp69 = "550 No such file or directory." - $ftp70 = "550 Not a plain file." - $ftp71 = "550 Not a valid directory." - $ftp72 = "550 Not a valid file." - $ftp73 = "550 Permission denied." - $ftp74 = "550 Unique file name error." - $ftp75 = "551 Error on input file." - $ftp76 = "551 Error on output file." - $ftp77 = "551 File listing failed." - $ftp78 = "552 Invalid port number." - $ftp79 = "552 Not a valid port number." - $ftp80 = "553 Cannot rename file." - $ftp81 = "553 Host unknown." - $ftp82 = "553 No permission." - $ftp83 = "553 Not a valid file name." - $ftp84 = "Interactive mode on." - $ftp85 = "bytes received in" - $ftp86 = "command successful" + meta: + author="@matonis" + description="FTP Command Artifacts" + strings: + $ftp1 = "150 File status okay; about to open data connection." + $ftp2 = "150 Opening BINARY mode data connection for" + $ftp3 = "150 Opening data connection." + $ftp4 = "200 Command PORT okay." + $ftp5 = "200 Command PROT okay." + $ftp6 = "200 Command SITE okay." + $ftp7 = "200 Command okay." + $ftp8 = "200 EPRT command okay." + $ftp9 = "200 Goodbye." + $ftp10 = "200 PORT command successful." + $ftp11 = "202 Already logged-in." + $ftp12 = "202 Command ACCT not implemented." + $ftp13 = "214 Help information." + $ftp14 = "221 Goodbye." + $ftp15 = "221 List of all the extensions supported." + $ftp16 = "226 ABOR command successful." + $ftp17 = "226 Closing data connection." + $ftp18 = "226 Transfer complete." + $ftp19 = "229 Entering passive mode" + $ftp20 = "230 Already logged-in." + $ftp21 = "230 User logged in, proceed." + $ftp22 = "234 AUTH command okay; starting SSL connection." + $ftp23 = "Transfer started." + $ftp24 = "250 Command okay." + $ftp25 = "250 Directory created." + $ftp26 = "250 Directory removed." + $ftp27 = "250 Requested file action okay, file renamed." + $ftp28 = "331 Guest login okay, send your complete e-mail address as password." + $ftp29 = "331 User name okay, need password." + $ftp30 = "350 Requested file action pending further information." + $ftp31 = "421 Maximum anonymous login limit has been reached." + $ftp32 = "421 Maximum login limit has been reached." + $ftp33 = "425 Can't open data connection." + $ftp34 = "425 Cannot open data connection." + $ftp35 = "425 Cannot open passive connection." + $ftp36 = "425 Cannot open the data connection." + $ftp37 = "426 Data connection error." + $ftp38 = "431 Security is disabled." + $ftp39 = "431 Service is unavailable." + $ftp40 = "450 Can't delete file." + $ftp41 = "450 No permission to delete." + $ftp42 = "500 Execution failed." + $ftp43 = "501 Syntax error in parameters or arguments." + $ftp44 = "501 Syntax error." + $ftp45 = "502 Command SITE not implemented for this argument." + $ftp46 = "502 Not yet implemented." + $ftp47 = "503 Cannot find the file which has to be renamed." + $ftp48 = "503 Login with USER first." + $ftp49 = "504 Command not implemented." + $ftp50 = "504 Not implemented for this command." + $ftp51 = "504 Server does not understand the specified protection level." + $ftp52 = "510 EPRT IP is not same as client IP." + $ftp53 = "510 EPRT is disabled." + $ftp54 = "510 PORT IP mismatch." + $ftp55 = "510 Port is disabled." + $ftp56 = "510 Syntax error in parameters." + $ftp57 = "510 Syntax error." + $ftp58 = "530 Access denied." + $ftp59 = "530 Anonymous connection is not allowed." + $ftp60 = "530 Authentication failed." + $ftp61 = "530 Invalid user name." + $ftp62 = "550 Already exists." + $ftp63 = "550 Cannot create directory." + $ftp64 = "550 Cannot remove directory." + $ftp65 = "550 File unavailable." + $ftp66 = "550 Invalid path." + $ftp67 = "550 No permission." + $ftp68 = "550 No such directory." + $ftp69 = "550 No such file or directory." + $ftp70 = "550 Not a plain file." + $ftp71 = "550 Not a valid directory." + $ftp72 = "550 Not a valid file." + $ftp73 = "550 Permission denied." + $ftp74 = "550 Unique file name error." + $ftp75 = "551 Error on input file." + $ftp76 = "551 Error on output file." + $ftp77 = "551 File listing failed." + $ftp78 = "552 Invalid port number." + $ftp79 = "552 Not a valid port number." + $ftp80 = "553 Cannot rename file." + $ftp81 = "553 Host unknown." + $ftp82 = "553 No permission." + $ftp83 = "553 Not a valid file name." + $ftp84 = "Interactive mode on." + $ftp85 = "bytes received in" + $ftp86 = "command successful" - condition: - 4 of them + condition: + 4 of them } diff --git a/page_brute-BETA.py b/page_brute-BETA.py index 2c03f79..5bd25af 100755 --- a/page_brute-BETA.py +++ b/page_brute-BETA.py @@ -13,215 +13,250 @@ import binascii try: - import yara + import yara # main module except: print "[!] - ERROR: Could not import YARA..." print "...did you install yara and yara-python? Exiting." sys.exit() - +# check whether the block is null/zero out or not +# if so, skip. def is_block_null(block): - #Here we test to see if the block is null..if so, skip. - RAW_BLOCK=binascii.hexlify(block) - NULL_REF=binascii.hexlify(NULL_REFERENCE) - if RAW_BLOCK == NULL_REF: - return True - else: - return False + RAW_BLOCK = binascii.hexlify(block) + NULL_REF = binascii.hexlify(NULL_REFERENCE) + + if RAW_BLOCK == NULL_REF: + return True + else: + return False +# compile yara rules def build_ruleset(): - if RULETYPE == "FILE": - try: - rules=yara.compile(str(RULES)) - print "..... Ruleset Compilation Successful." - return rules - except: - print "[!] - Could not compile YARA rule: %s" % RULES - print "Exiting." - sys.exit() - - elif RULETYPE == "FOLDER": - RULEDATA="" - #::Get list of files ending in .yara - - RULE_COUNT = len(glob.glob1(RULES,"*.yar")) - if RULE_COUNT != 0: - for yara_file in glob.glob(os.path.join(RULES, "*.yar")): - try: - yara.compile(str(yara_file)) - print "..... Syntax appears to be OK: %s " % yara_file - try: - with open(yara_file, "r") as sig_file: - file_contents=sig_file.read() - RULEDATA=RULEDATA + "\n" + file_contents - except: - print "..... SKIPPING: Could not open file for reading: %s " % yara_file - except: - print "..... SKIPPING: Could not compile rule: %s " % yara_file - try: - rules=yara.compile(source=RULEDATA) - print "..... SUCCESS! Compiled noted yara rulesets.\n" - return rules - except: - print "[!] - Some catastropic error occurred in the compilation of signatureswithin the directory. Exiting." - sys.exit() - else: - print "No files ending in .yar within: %s " % RULES - print "Exiting." - sys.exit() - - elif RULETYPE == "DEFAULT": - rules=yara.compile(str(RULES)) - print "[+] - Ruleset Compilation Successful." - return rules - - else: - print "[!] - ERROR: Possible catastrophic error on build_ruleset. Exiting." - sys.exit() + if RULETYPE == "FILE": + try: + rules = yara.compile(str(RULES)) + print "..... Ruleset Compilation Successful." + return rules + except: + print "[!] - Could not compile YARA rule: %s" % RULES + print "Exiting." + sys.exit() + + elif RULETYPE == "FOLDER": + RULEDATA="" + #::Get list of files ending in .yara + + RULE_COUNT = len(glob.glob1(RULES,"*.yar")) + if RULE_COUNT != 0: + for yara_file in glob.glob(os.path.join(RULES, "*.yar")): + try: + yara.compile(str(yara_file)) + print "..... Syntax appears to be OK: %s " % yara_file + try: + with open(yara_file, "r") as sig_file: + file_contents=sig_file.read() + RULEDATA=RULEDATA + "\n" + file_contents + except: + print "..... SKIPPING: Could not open file for reading: %s " % yara_file + except: + print "..... SKIPPING: Could not compile rule: %s " % yara_file + try: + rules=yara.compile(source=RULEDATA) + print "..... SUCCESS! Compiled noted yara rulesets.\n" + return rules + except: + print "[!] - Some catastropic error occurred in the compilation of signatureswithin the directory. Exiting." + sys.exit() + else: + print "No files ending in .yar within: %s " % RULES + print "Exiting." + sys.exit() + + elif RULETYPE == "DEFAULT": + rules=yara.compile(str(RULES)) + print "[+] - Ruleset Compilation Successful." + return rules + + else: + print "[!] - ERROR: Possible catastrophic error on build_ruleset. Exiting." + sys.exit() +# print stdout messages def print_procedures(): - print "[+] - PAGE_BRUTE running with the following options:" - print "\t[-] - FILE: %s" % FILE - print "\t[-] - PAGE_SIZE: %s" % PAGE_SIZE - print "\t[-] - RULES TYPE: %s" % RULETYPE - print "\t[-] - RULE LOCATION: %s" % RULES - print "\t[-] - INVERSION SCAN: %s" % INVERT - print "\t[-] - WORKING DIR: %s" % WORKING_DIR - print "\t=================\n" + print "[+] - PAGE_BRUTE running with the following options:" + print "\t[-] - FILE: %s" % FILE + print "\t[-] - PAGE_SIZE: %s" % PAGE_SIZE + print "\t[-] - RULES TYPE: %s" % RULETYPE + print "\t[-] - RULE LOCATION: %s" % RULES + print "\t[-] - INVERSION SCAN: %s" % INVERT + print "\t[-] - WORKING DIR: %s" % WORKING_DIR + print "\t=================\n" + def main(): - global FILE - global PAGE_SIZE - global RULES - global SCANNAME - global INVERT - global RULETYPE - global NULL_REFERENCE - - argument_parser = argparse.ArgumentParser(description="Checks pages in pagefiles for YARA-based rule matches. Useful to identify forensic artifacts within Windows-based page files and characterize blocks based on regular expressions.") - - group_arg = argument_parser.add_argument_group() - group_arg.add_argument("-f", "--file", metavar="FILE", help="Pagefile or any chunk/block-based binary file") - group_arg.add_argument("-p", "--size", metavar="SIZE", help="Size of chunk/block in bytes (Default 4096)") - group_arg.add_argument("-o", "--scanname", metavar="SCANNAME", help="Descriptor of the scan session - used for output directory") - group_arg.add_argument("-i", "--invert", help="Given scan options, match all blocks that DO NOT match a ruleset",action='store_true') - - group_arg = argument_parser.add_mutually_exclusive_group() - group_arg.add_argument("-r", "--rules", metavar="RULEFILE", help="File/directory containing YARA signatures (must end with .yar)") - - args = argument_parser.parse_args() - - if len(sys.argv) < 2: - print argument_parser.print_help() - sys.exit() - - #::Check to see if file was provided::# - if args.file: - try: - with open(args.file): - FILE=args.file - print "[+] - PAGE_BRUTE processing file: %s" % FILE - except: - print "[!] - Could not open %s. Exiting." % FILE - sys.exit() - else: - print "[!] - No file provided. Use -f, --file to provide a file. Exiting." - sys.exit() - - #::Check to see if page size provided::# - if args.size: - PAGE_SIZE=int(args.size) - NULL_REFERENCE= '\x00' * PAGE_SIZE - else: - PAGE_SIZE=4096 - NULL_REFERENCE= '\x00' * PAGE_SIZE - - #::Check if --scan-name provided::# - if args.scanname: - SCANNAME=args.scanname - else: - SCANNAME="PAGE_BRUTE-" + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "-RESULTS" - - #::Check if --invert-match provided::# - if args.invert: - INVERT=True - else: - INVERT=False - - #::Check if --rule-file provdided - if not, use default ruleset::# - if args.rules: - RULES=args.rules - try: - #::Is File?::# - if os.path.isfile(RULES): - RULETYPE="FILE" - print "[+] - YARA rule of File type provided for compilation: %s" % RULES - elif os.path.isdir(RULES): - print "[+] - YARA rule of Folder type provided for compilation: %s" % RULES - RULETYPE="FOLDER" - except: - print "[!] - Possible catastrophic error with the provided rule file...exiting." - sys.exit() - else: - try: - with open("default_signatures.yar"): - RULES="default_signatures.yar" - RULETYPE="DEFAULT" - except: - print "[!] - Could not locate \"default_signature.yar\". Find it or provide custom signatures via --rules. Exiting." - sys.exit() - - #::Compile rules::# - authoritative_rules=build_ruleset() - #::Build directory structure - global WORKING_DIR - WORKING_DIR=SCANNAME - if not os.path.exists(WORKING_DIR): - os.makedirs(WORKING_DIR) - #::Let People Know what we're doing::# - print_procedures() - #::Find Evil::# - page_id=0 - with open(FILE, "rb") as page_file: - while True: - matched=False - raw_page=page_file.read(PAGE_SIZE) - if raw_page == "": - print "Done!" - print "Ending page_id is: %s" % page_id - break - if not is_block_null(raw_page): - #::Determine if block is null...: - for matches in authoritative_rules.match(data=raw_page): - if INVERT == True: - matched=True - else: - CHUNK_OUTPUT_DIR=os.path.join(WORKING_DIR,matches.rule) - print " [!] FLAGGED BLOCK " + str(page_id) + ": " + matches.rule - - if not os.path.exists(CHUNK_OUTPUT_DIR): - os.makedirs(CHUNK_OUTPUT_DIR) - - #::Save chunk to file::# - CHUNK_OUTPUT_FWD=os.path.join(CHUNK_OUTPUT_DIR,str(page_id) + ".block") - page_export=open(CHUNK_OUTPUT_FWD,'w+') - page_export.write(raw_page) - page_export.close() - - if INVERT == True: - if matched == False: - CHUNK_OUTPUT_DIR=os.path.join(WORKING_DIR,"INVERTED-MATCH") - print " [!] BLOCK DOES NOT MATCH ANY KNOWN SIGNATURE " + str(page_id) - if not os.path.exists(CHUNK_OUTPUT_DIR): - os.makedirs(CHUNK_OUTPUT_DIR) - - CHUNK_OUTPUT_FWD=os.path.join(CHUNK_OUTPUT_DIR,str(page_id) + ".block") - page_export=open(CHUNK_OUTPUT_FWD,'w+') - page_export.write(raw_page) - page_export.close() - #::Increment Counter for offset increment::# - page_id=page_id+1 + global FILE + global PAGE_SIZE + global RULES + global SCANNAME + global INVERT + global RULETYPE + global NULL_REFERENCE + + # parse input arguments + argument_parser = argparse.ArgumentParser(description="Checks pages in pagefiles for YARA-based rule matches." + " Useful to identify forensic artifacts within Windows-based page files and" + " characterize blocks based on regular expressions.") + group_arg = argument_parser.add_argument_group() + group_arg.add_argument("-f", "--file", metavar="FILE", help="Pagefile or any chunk/block-based binary file") + group_arg.add_argument("-p", "--size", metavar="SIZE", help="Size of chunk/block in bytes (Default 4096)") + group_arg.add_argument("-o", "--scanname", metavar="SCANNAME", help="Descriptor of the scan session - used for output directory") + group_arg.add_argument("-i", "--invert", help="Given scan options, match all blocks that DO NOT match a ruleset",action='store_true') + + group_arg = argument_parser.add_mutually_exclusive_group() + group_arg.add_argument("-r", "--rules", metavar="RULEFILE", help="File/directory containing YARA signatures (must end with .yar)") + args = argument_parser.parse_args() + + if len(sys.argv) < 2: + print argument_parser.print_help() + sys.exit() + + # validate input file + if args.file: + try: + with open(args.file, 'rb'): + FILE = args.file + print "[+] - PAGE_BRUTE processing file: %s" % FILE + except: + print "[!] - Could not open %s. Exiting." % FILE + sys.exit() + else: + print "[!] - No file provided. Use -f, --file to provide a file. Exiting." + sys.exit() + + + # Check to see if page size provided + if args.size: + PAGE_SIZE = int(args.size) + NULL_REFERENCE = '\x00' * PAGE_SIZE + else: + PAGE_SIZE = 4096 + NULL_REFERENCE = '\x00' * PAGE_SIZE + + # Check if --scan-name provided + if args.scanname: + SCANNAME = args.scanname + else: + SCANNAME = "PAGE_BRUTE-" + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + "-RESULTS" + + # Check if --invert-match provided + if args.invert: + INVERT = True + else: + INVERT = False + + # Check if --rule-file provdided - if not, use default ruleset + if args.rules: + RULES = args.rules + try: + # Is File? + if os.path.isfile(RULES): + RULETYPE = "FILE" + print "[+] - YARA rule of File type provided for compilation: %s" % RULES + elif os.path.isdir(RULES): + print "[+] - YARA rule of Folder type provided for compilation: %s" % RULES + RULETYPE="FOLDER" + except: + print "[!] - Possible catastrophic error with the provided rule file...exiting." + sys.exit() + else: + try: + with open("default_signatures.yar"): + RULES = "default_signatures.yar" + RULETYPE = "DEFAULT" + except: + print "[!] - Could not locate \"default_signature.yar\". Find it or provide custom signatures via --rules. Exiting." + sys.exit() + + # Compile rules + authoritative_rules=build_ruleset() + # Build directory structure + global WORKING_DIR + WORKING_DIR = SCANNAME + if not os.path.exists(WORKING_DIR): + os.makedirs(WORKING_DIR) + # Let People Know what we're doing + print_procedures() + # Find Evil + page_id = 0 + page_list = {} + page_block = {} + with open(FILE, "rb") as page_file: + while True: + matched = False + raw_page = page_file.read(PAGE_SIZE) + if raw_page == "": + print "Done!" + print "Ending page_id is: %s" % page_id + break + if not is_block_null(raw_page): + # Determine if block is null + for matches in authoritative_rules.match(data=raw_page): + if INVERT == True: + matched = True + else: + CHUNK_OUTPUT_DIR = os.path.join(WORKING_DIR,matches.rule) + print " [!] FLAGGED BLOCK " + str(page_id) + ": " + matches.rule + + # save page id with matched rules to dictionary page_list + if matches.rule not in page_list: + page_list[matches.rule] = str(page_id) + else: + page_list[matches.rule] += ";"+str(page_id) + page_block[page_id] = raw_page + if not os.path.exists(CHUNK_OUTPUT_DIR): + os.makedirs(CHUNK_OUTPUT_DIR) + + # Save chunk to file + CHUNK_OUTPUT_FWD = os.path.join(CHUNK_OUTPUT_DIR,str(page_id) + ".block") + page_export = open(CHUNK_OUTPUT_FWD,'w+') + page_export.write(raw_page) + page_export.close() + + if INVERT == True: + if matched == False: + CHUNK_OUTPUT_DIR = os.path.join(WORKING_DIR,"INVERTED-MATCH") + print " [!] BLOCK DOES NOT MATCH ANY KNOWN SIGNATURE " + str(page_id) + if not os.path.exists(CHUNK_OUTPUT_DIR): + os.makedirs(CHUNK_OUTPUT_DIR) + + CHUNK_OUTPUT_FWD = os.path.join(CHUNK_OUTPUT_DIR,str(page_id) + ".block") + page_export = open(CHUNK_OUTPUT_FWD,'w+') + page_export.write(raw_page) + page_export.close() + + # Increment Counter for offset increment + page_id=page_id+1 + + # write to a file as rules and pageid for further analysis + o = open(SCANNAME+"_pagelist", 'w') + # write page id with yara rule names with its corresponding data block + o2 = open(SCANNAME+"_pageblocks", 'w') + page_list2 = [] + scan_name = [] + + for k,v in page_list.iteritems(): + for m in v.split(';'): + page_list2.append(m) + scan_name.append(k) + o.write(k+'\t'+m+"\n") + for k2,v2 in page_block.iteritems(): + hitrule = "" + if str(k2) in page_list2: + hitrule = scan_name[page_list2.index(str(k2))] + o2.write(">"+hitrule+"_"+str(k2)+'\n') + o2.write(str(v2)+'\n') + if __name__ == "__main__": main() From 0e3930fed08d4e7ce692c1d7b355ed9a024001b4 Mon Sep 17 00:00:00 2001 From: aungthurhahein Date: Thu, 3 Dec 2015 10:51:40 +0700 Subject: [PATCH 2/3] output and futher analysis --- further_analysis.sh | 21 ++++++++++++++ page_brute-BETA.py | 67 +++++++++++++++++++++++---------------------- 2 files changed, 55 insertions(+), 33 deletions(-) create mode 100644 further_analysis.sh diff --git a/further_analysis.sh b/further_analysis.sh new file mode 100644 index 0000000..c5c52c8 --- /dev/null +++ b/further_analysis.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# one bash liners to concat repeated page lists into a file +# sort the output file +# get repeated page list and also unique pagelist by page ids + +outfile="allpage.lst" +sort=".sort" +for i in *.pagelist +do + cat $i >> $outfile +done + +sort -t $'\t' -k2 -n allpage.lst > $outfile$sort + +cat allpage.lst.sort | awk -F '\t' '{print $2}' | sort | uniq -d > allpage.repeats +cat allpage.lst.sort | awk -F '\t' '{print $2}' | sort | uniq -c > allpage.uniq + +# map repeated pageid togher by its rule to analyze further analysis + + diff --git a/page_brute-BETA.py b/page_brute-BETA.py index 5bd25af..75fb8df 100755 --- a/page_brute-BETA.py +++ b/page_brute-BETA.py @@ -22,17 +22,17 @@ # check whether the block is null/zero out or not # if so, skip. def is_block_null(block): - RAW_BLOCK = binascii.hexlify(block) - NULL_REF = binascii.hexlify(NULL_REFERENCE) + raw_block = binascii.hexlify(block) + null_ref = binascii.hexlify(NULL_REFERENCE) - if RAW_BLOCK == NULL_REF: + if raw_block == null_ref: return True else: return False # compile yara rules def build_ruleset(): - if RULETYPE == "FILE": + if ruletype == "FILE": try: rules = yara.compile(str(RULES)) print "..... Ruleset Compilation Successful." @@ -42,9 +42,9 @@ def build_ruleset(): print "Exiting." sys.exit() - elif RULETYPE == "FOLDER": + elif ruletype == "FOLDER": RULEDATA="" - #::Get list of files ending in .yara + # Get list of files ending in .yara RULE_COUNT = len(glob.glob1(RULES,"*.yar")) if RULE_COUNT != 0: @@ -61,19 +61,20 @@ def build_ruleset(): except: print "..... SKIPPING: Could not compile rule: %s " % yara_file try: - rules=yara.compile(source=RULEDATA) + rules = yara.compile(source=RULEDATA) print "..... SUCCESS! Compiled noted yara rulesets.\n" return rules except: - print "[!] - Some catastropic error occurred in the compilation of signatureswithin the directory. Exiting." + print "[!] - Some catastropic error occurred in the " \ + "compilation of signatureswithin the directory. Exiting." sys.exit() else: print "No files ending in .yar within: %s " % RULES print "Exiting." sys.exit() - elif RULETYPE == "DEFAULT": - rules=yara.compile(str(RULES)) + elif ruletype == "DEFAULT": + rules = yara.compile(str(RULES)) print "[+] - Ruleset Compilation Successful." return rules @@ -86,9 +87,9 @@ def print_procedures(): print "[+] - PAGE_BRUTE running with the following options:" print "\t[-] - FILE: %s" % FILE print "\t[-] - PAGE_SIZE: %s" % PAGE_SIZE - print "\t[-] - RULES TYPE: %s" % RULETYPE + print "\t[-] - RULES TYPE: %s" % ruletype print "\t[-] - RULE LOCATION: %s" % RULES - print "\t[-] - INVERSION SCAN: %s" % INVERT + print "\t[-] - INVERSION SCAN: %s" % invert print "\t[-] - WORKING DIR: %s" % WORKING_DIR print "\t=================\n" @@ -99,8 +100,8 @@ def main(): global PAGE_SIZE global RULES global SCANNAME - global INVERT - global RULETYPE + global invert + global ruletype global NULL_REFERENCE # parse input arguments @@ -151,9 +152,9 @@ def main(): # Check if --invert-match provided if args.invert: - INVERT = True + invert = True else: - INVERT = False + invert = False # Check if --rule-file provdided - if not, use default ruleset if args.rules: @@ -161,11 +162,11 @@ def main(): try: # Is File? if os.path.isfile(RULES): - RULETYPE = "FILE" + ruletype = "FILE" print "[+] - YARA rule of File type provided for compilation: %s" % RULES elif os.path.isdir(RULES): print "[+] - YARA rule of Folder type provided for compilation: %s" % RULES - RULETYPE="FOLDER" + ruletype="FOLDER" except: print "[!] - Possible catastrophic error with the provided rule file...exiting." sys.exit() @@ -173,7 +174,7 @@ def main(): try: with open("default_signatures.yar"): RULES = "default_signatures.yar" - RULETYPE = "DEFAULT" + ruletype = "DEFAULT" except: print "[!] - Could not locate \"default_signature.yar\". Find it or provide custom signatures via --rules. Exiting." sys.exit() @@ -202,10 +203,10 @@ def main(): if not is_block_null(raw_page): # Determine if block is null for matches in authoritative_rules.match(data=raw_page): - if INVERT == True: + if invert == True: matched = True else: - CHUNK_OUTPUT_DIR = os.path.join(WORKING_DIR,matches.rule) + chunk_output_dir = os.path.join(WORKING_DIR,matches.rule) print " [!] FLAGGED BLOCK " + str(page_id) + ": " + matches.rule # save page id with matched rules to dictionary page_list @@ -214,29 +215,29 @@ def main(): else: page_list[matches.rule] += ";"+str(page_id) page_block[page_id] = raw_page - if not os.path.exists(CHUNK_OUTPUT_DIR): - os.makedirs(CHUNK_OUTPUT_DIR) + if not os.path.exists(chunk_output_dir): + os.makedirs(chunk_output_dir) # Save chunk to file - CHUNK_OUTPUT_FWD = os.path.join(CHUNK_OUTPUT_DIR,str(page_id) + ".block") - page_export = open(CHUNK_OUTPUT_FWD,'w+') + chunk_output_fwd = os.path.join(chunk_output_dir,str(page_id) + ".block") + page_export = open(chunk_output_fwd,'w+') page_export.write(raw_page) page_export.close() - if INVERT == True: + if invert == True: if matched == False: - CHUNK_OUTPUT_DIR = os.path.join(WORKING_DIR,"INVERTED-MATCH") + chunk_output_dir = os.path.join(WORKING_DIR,"INVERTED-MATCH") print " [!] BLOCK DOES NOT MATCH ANY KNOWN SIGNATURE " + str(page_id) - if not os.path.exists(CHUNK_OUTPUT_DIR): - os.makedirs(CHUNK_OUTPUT_DIR) + if not os.path.exists(chunk_output_dir): + os.makedirs(chunk_output_dir) - CHUNK_OUTPUT_FWD = os.path.join(CHUNK_OUTPUT_DIR,str(page_id) + ".block") - page_export = open(CHUNK_OUTPUT_FWD,'w+') + chunk_output_fwd = os.path.join(chunk_output_dir,str(page_id) + ".block") + page_export = open(chunk_output_fwd,'w+') page_export.write(raw_page) page_export.close() # Increment Counter for offset increment - page_id=page_id+1 + page_id = page_id+1 # write to a file as rules and pageid for further analysis o = open(SCANNAME+"_pagelist", 'w') @@ -254,7 +255,7 @@ def main(): hitrule = "" if str(k2) in page_list2: hitrule = scan_name[page_list2.index(str(k2))] - o2.write(">"+hitrule+"_"+str(k2)+'\n') + o2.write(">"+hitrule+"-"+str(k2)+'\n') o2.write(str(v2)+'\n') From 1ff4b512cbcf69a6ea6ef6c39589b58552eba817 Mon Sep 17 00:00:00 2001 From: aungthurhahein Date: Thu, 3 Dec 2015 11:43:32 +0700 Subject: [PATCH 3/3] readme and analysis bash file added --- README.md | 26 ++++++++++++++++++++++++++ further_analysis.sh | 6 +++--- rules_IDrepeats_map.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 rules_IDrepeats_map.py diff --git a/README.md b/README.md index d742677..ad73da5 100644 --- a/README.md +++ b/README.md @@ -127,5 +127,31 @@ root@system:~/Desktop/page/page_brute/PAGE_BRUTE-2013-10-27-01:20:28-RESULTS/web 00000e0: 2020 3c74 6420 7374 796c 653d 2270 6164 allpage.repeats.map +# output: +# Dev: __author__ = 'aung' +# Date: 20151019 +""" +import sys +repeatlist = sys.argv[1] +allpage = sys.argv[2] + +page_rule = [] +page_id = [] +with open(allpage,'rb') as f1: + for l1 in f1: + l1_split = l1.split('\t') + page_rule.append(l1_split[0]) + page_id.append(l1_split[1].strip('\n')) + +with open(repeatlist,'rb') as f2: + for l2 in f2: + repid = l2.strip('\n') + ind = [i for i, e in enumerate(page_id) if e == repid] + rules= "" + for i in ind: + if rules == "": + rules = page_rule[i] + else: + rules += ";"+page_rule[i] + sys.stdout.write(repid+'\t'+rules+'\n') +