From c2fa7912e8daf4a0e5dc642a5d16c7341a11724e Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 12:42:53 -0600 Subject: [PATCH 01/20] Initial Commit of MISP Automation --- automation/README.md | 86 +++++++++++++++++++++++++++++++++++++++ automation/flare/AIS.sh | 41 +++++++++++++++++++ automation/flare/CISCP.sh | 41 +++++++++++++++++++ automation/misp/MISP.sh | 41 +++++++++++++++++++ 4 files changed, 209 insertions(+) create mode 100644 automation/README.md create mode 100644 automation/flare/AIS.sh create mode 100644 automation/flare/CISCP.sh create mode 100644 automation/misp/MISP.sh diff --git a/automation/README.md b/automation/README.md new file mode 100644 index 0000000..f2e76d0 --- /dev/null +++ b/automation/README.md @@ -0,0 +1,86 @@ +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# +# Description: Simple set of scripts to automate AIS data pulls into MISP to # +# enable quick correlation, taxonomy tagging and review of data. # +# Automation can also be used as standalone scripts # +# Scripts are simply a working POC and written to only support TAXII 1.1 # +# Author: Cory Kennedy (@corykennedy) # +# _____ .___ _________ # +# / _ \ | |/ _____/ # +# / /_\ \| |\_____ \ # +# / | \ |/ \ # +# \____|__ /___/___v1.0_ / # +# \/ \/ # +# AIS Automation # +#==============================================================================# +# -=[Instructions Summary]=- # +# # +# * Install scripts & crontabs onto your MISP and Flare servers # +# * Need help with crontabs? Try: https://crontab.guru/ # +# * The below will execute the scripts at 2:30am daily and log all output # +# # +#==============================================================================# +# -=[Dependancies]=- # +# 1. Active AIS Participation # +# * https://www.us-cert.gov/ais # +# 2. Working Flare instance # +# 3. Working MISP instance # +# 4. Working CTI-Toolkit on MISP server # +# * https://github.com/certau/cti-toolkit.git # +#==============================================================================# + +#------------------------------------------------------------------------------# +# =[BEGIN FLARE]= # +#------------------------------------------------------------------------------# +[FLARE Script Installation] + +1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation +2. Move scripts from oss-automation/scripts/flare to your flare server. + * Example: mv oss-automation/scripts/flare/* /opt/Flare/scripts/ +3. Install crontabs + +[FLARE Crontab Installation] + +1. From a terminal type: crontab -e +2. Copy and paste the below into your crontab +30 2 * * * /opt/Flare/scripts/CISCP.sh &>/opt/Flare/scripts/logs/CISCP_`date +\%y-\%m-\%d`.out +32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out +3. Save crontab +Note: Flare server is complete. Proceed to MISP + + +#------------------------------------------------------------------------------# +# =[BEGIN MISP]= # +#------------------------------------------------------------------------------# + +[MISP Script Installation] + +1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation +2. Move scripts from oss-automation/scripts/misp to your MISP server + * Example: mv oss-automation/scripts/misp/* /home/misp/scripts/ +3. Install crontabs + +[MISP Crontab Installation] + +1. From a terminal type: crontab -e +2. Copy and paste the below into your crontab +30 2 * * * /home/misp/scripts/MISP.sh +3. Save crontab +Note: MISP server is complete. + +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# diff --git a/automation/flare/AIS.sh b/automation/flare/AIS.sh new file mode 100644 index 0000000..011be0d --- /dev/null +++ b/automation/flare/AIS.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# +# Simple shell script to pull the AIS feed from AIS Data +# There are much better ways to do this. This is only a quick working POC +# Author: Cory Kennedy (@corykennedy) +# _____ .___ _________ +# / _ \ | |/ _____/ +# / /_\ \| |\_____ \ +# / | \ |/ \ +# \____|__ /___/___v1.0_ / +# \/ \/ +# AIS Automation +# Usage: ./AIS.sh +# ----------------------------------------------------------------------------- +#Move into our working directory +cd /opt/Flare/ +#Poll AIS server for AIS feed using TAXII 1.1 and format date parameters +./poll.sh 1.1 AIS -b $(date -d '1 days ago' --utc "+%FT%T.%N" | sed -r 's/[[:digit:]]{6}$/Z/') -e $(date -d --utc "+%FT%T.%N" | sed -r 's/[[:digit:]]{6}$/Z/') +#Move into our working feed directory +cd /opt/Flare/subscribeFeeds/AIS/ +#Prepare all files for transport +tar -zcvf /opt/Flare/TRANSFER/AIS_`date +%y-%m-%d`.tgz . +#Cleanup files after completion +rm -rf /opt/Flare/subscribeFeeds/AIS/* +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# diff --git a/automation/flare/CISCP.sh b/automation/flare/CISCP.sh new file mode 100644 index 0000000..f68f87f --- /dev/null +++ b/automation/flare/CISCP.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# +# Simple shell script to pull the CISCP feed from AIS Data +# There are much better ways to do this. This is only a quick working POC. +# Author: Cory Kennedy (@corykennedy) +# _________ .___ ____________________________ +# \_ ___ \| |/ _____/\_ ___ \______ \ +# / \ \/| |\_____ \ / \ \/| ___/ +# \ \___| |/ \\ \___| | +# \______ /___/_______ / \______ /v1.1| +# \/ \/ \/ +# AIS Automation +# Usage: ./CISCP.sh +# ----------------------------------------------------------------------------- +#Move into our working directory +cd /opt/Flare/ +#Poll AIS server for AIS feed using TAXII 1.1 and format date parameters +./poll.sh 1.1 CISCP -b $(date -d '1 days ago' --utc "+%FT%T.%N"| sed -r 's/[[:digit:]]{6}$/Z/') -e $(date -d --utc "+%FT%T.%N" | sed -r 's/[[:digit:]]{6}$/Z/') +#Move into our working feed directory +cd /opt/Flare/subscribeFeeds/CISCP/ +#Prepare all files for transport +tar -zcvf /opt/Flare/TRANSFER/CISCP_`date +%y-%m-%d`.tgz . +#Cleanup files after completion +rm -rf /opt/Flare/subscribeFeeds/CISCP/* +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# diff --git a/automation/misp/MISP.sh b/automation/misp/MISP.sh new file mode 100644 index 0000000..05b9ff7 --- /dev/null +++ b/automation/misp/MISP.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# +# Simple script to automate AIS data into MISP +# Author: Cory Kennedy (@corykennedy) +# ----------------------------------------------------------------------------- +# _____ .___ _________ _____ .___ ___________________ +# / _ \ | |/ _____/ .__ / \ | |/ _____/\______ \ +# / /_\ \| |\_____ \ __| |___ / \ / \| |\_____ \ | ___/ +# / | \ |/ \ /__ __/ / Y \ |/ \ | | +# \____|__ /___/TAXII1.1 / |__| \____|__ /___/_______ / |____| +# \/ \/ \/ \/ +# Usage: ./MISP.sh +# ----------------------------------------------------------------------------- +#Transfer files from AIS host, then remove source files. +#Assumes you have ssh-copy-id yourself onto your flare host. +rsync -avz --remove-source-files -e ssh user@your.flare.host:/opt/Flare/TRANSFER /home/misp/feeds/ +#Move into our working directory +cd /home/misp/feeds/TRANSFER/ +#Find compressed AIS files and uncompress them +cat *.tgz | tar -zxvf - -i +#Import into MISP +python /home/misp/cti-toolkit/stixtransclient.py --file /home/misp/feeds/TRANSFER/ -r --misp --misp-url https://misp --misp-key [YOURMISPKEY] --misp-threat 2 --misp-distribution 0 --misp-info "AIS" +#Cleanup transfered files after completion +rm -rf /home/misp/feeds/TRANSFER/ +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# From 462b8e8e0c3336c6dbc9eb94baffca3e2a15aa56 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 12:42:53 -0600 Subject: [PATCH 02/20] Initial Commit of MISP Automation --- automation/README.md | 86 +++++++++++++++++++++++++++++++++++++++ automation/flare/AIS.sh | 41 +++++++++++++++++++ automation/flare/CISCP.sh | 41 +++++++++++++++++++ automation/misp/MISP.sh | 41 +++++++++++++++++++ 4 files changed, 209 insertions(+) create mode 100644 automation/README.md create mode 100644 automation/flare/AIS.sh create mode 100644 automation/flare/CISCP.sh create mode 100644 automation/misp/MISP.sh diff --git a/automation/README.md b/automation/README.md new file mode 100644 index 0000000..30c6e07 --- /dev/null +++ b/automation/README.md @@ -0,0 +1,86 @@ + ------------------------------DISCLAIMER------------------------------------- + ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT + THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO + – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM + SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL + NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE + USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM + THIS UNSUPPORTED SOFTWARE. + ------------------------------DISCLAIMER------------------------------------- + Description: Simple set of scripts to automate AIS data pulls into MISP to + enable quick correlation, taxonomy tagging and review of data. + Automation can also be used as standalone scripts + Scripts are simply a working POC and written to only support TAXII 1.1 + Author: Cory Kennedy (@corykennedy) + _____ .___ _________ + / _ \ | |/ _____/ + / /_\ \| |\_____ \ + / | \ |/ \ + \____|__ /___/___v1.0_ / + \/ \/ + AIS Automation +============================================================================== + -=[Instructions Summary]=- + + * Install scripts & crontabs onto your MISP and Flare servers + * Need help with crontabs? Try: https://crontab.guru/ + * The below will execute the scripts at 2:30am daily and log all output + +============================================================================== + -=[Dependancies]=- + 1. Active AIS Participation + * https://www.us-cert.gov/ais + 2. Working Flare instance + 3. Working MISP instance + 4. Working CTI-Toolkit on MISP server + * https://github.com/certau/cti-toolkit.git +============================================================================== + +------------------------------------------------------------------------------ + =[BEGIN FLARE]= +------------------------------------------------------------------------------ +[FLARE Script Installation] + +1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation +2. Move scripts from oss-automation/scripts/flare to your flare server. + * Example: mv oss-automation/scripts/flare/* /opt/Flare/scripts/ +3. Install crontabs + +[FLARE Crontab Installation] + +1. From a terminal type: crontab -e +2. Copy and paste the below into your crontab +30 2 * * * /opt/Flare/scripts/CISCP.sh &>/opt/Flare/scripts/logs/CISCP_`date +\%y-\%m-\%d`.out +32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out +3. Save crontab +Note: Flare server is complete. Proceed to MISP + + +------------------------------------------------------------------------------ + =[BEGIN MISP]= +------------------------------------------------------------------------------ + +[MISP Script Installation] + +1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation +2. Move scripts from oss-automation/scripts/misp to your MISP server + * Example: mv oss-automation/scripts/misp/* /home/misp/scripts/ +3. Install crontabs + +[MISP Crontab Installation] + +1. From a terminal type: crontab -e +2. Copy and paste the below into your crontab +30 2 * * * /home/misp/scripts/MISP.sh +3. Save crontab +Note: MISP server is complete. + + ------------------------------DISCLAIMER------------------------------------- + ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT + THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO + – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM + SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL + NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE + USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM + THIS UNSUPPORTED SOFTWARE. + ------------------------------DISCLAIMER------------------------------------- diff --git a/automation/flare/AIS.sh b/automation/flare/AIS.sh new file mode 100644 index 0000000..011be0d --- /dev/null +++ b/automation/flare/AIS.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# +# Simple shell script to pull the AIS feed from AIS Data +# There are much better ways to do this. This is only a quick working POC +# Author: Cory Kennedy (@corykennedy) +# _____ .___ _________ +# / _ \ | |/ _____/ +# / /_\ \| |\_____ \ +# / | \ |/ \ +# \____|__ /___/___v1.0_ / +# \/ \/ +# AIS Automation +# Usage: ./AIS.sh +# ----------------------------------------------------------------------------- +#Move into our working directory +cd /opt/Flare/ +#Poll AIS server for AIS feed using TAXII 1.1 and format date parameters +./poll.sh 1.1 AIS -b $(date -d '1 days ago' --utc "+%FT%T.%N" | sed -r 's/[[:digit:]]{6}$/Z/') -e $(date -d --utc "+%FT%T.%N" | sed -r 's/[[:digit:]]{6}$/Z/') +#Move into our working feed directory +cd /opt/Flare/subscribeFeeds/AIS/ +#Prepare all files for transport +tar -zcvf /opt/Flare/TRANSFER/AIS_`date +%y-%m-%d`.tgz . +#Cleanup files after completion +rm -rf /opt/Flare/subscribeFeeds/AIS/* +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# diff --git a/automation/flare/CISCP.sh b/automation/flare/CISCP.sh new file mode 100644 index 0000000..f68f87f --- /dev/null +++ b/automation/flare/CISCP.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# +# Simple shell script to pull the CISCP feed from AIS Data +# There are much better ways to do this. This is only a quick working POC. +# Author: Cory Kennedy (@corykennedy) +# _________ .___ ____________________________ +# \_ ___ \| |/ _____/\_ ___ \______ \ +# / \ \/| |\_____ \ / \ \/| ___/ +# \ \___| |/ \\ \___| | +# \______ /___/_______ / \______ /v1.1| +# \/ \/ \/ +# AIS Automation +# Usage: ./CISCP.sh +# ----------------------------------------------------------------------------- +#Move into our working directory +cd /opt/Flare/ +#Poll AIS server for AIS feed using TAXII 1.1 and format date parameters +./poll.sh 1.1 CISCP -b $(date -d '1 days ago' --utc "+%FT%T.%N"| sed -r 's/[[:digit:]]{6}$/Z/') -e $(date -d --utc "+%FT%T.%N" | sed -r 's/[[:digit:]]{6}$/Z/') +#Move into our working feed directory +cd /opt/Flare/subscribeFeeds/CISCP/ +#Prepare all files for transport +tar -zcvf /opt/Flare/TRANSFER/CISCP_`date +%y-%m-%d`.tgz . +#Cleanup files after completion +rm -rf /opt/Flare/subscribeFeeds/CISCP/* +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# diff --git a/automation/misp/MISP.sh b/automation/misp/MISP.sh new file mode 100644 index 0000000..05b9ff7 --- /dev/null +++ b/automation/misp/MISP.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# +# Simple script to automate AIS data into MISP +# Author: Cory Kennedy (@corykennedy) +# ----------------------------------------------------------------------------- +# _____ .___ _________ _____ .___ ___________________ +# / _ \ | |/ _____/ .__ / \ | |/ _____/\______ \ +# / /_\ \| |\_____ \ __| |___ / \ / \| |\_____ \ | ___/ +# / | \ |/ \ /__ __/ / Y \ |/ \ | | +# \____|__ /___/TAXII1.1 / |__| \____|__ /___/_______ / |____| +# \/ \/ \/ \/ +# Usage: ./MISP.sh +# ----------------------------------------------------------------------------- +#Transfer files from AIS host, then remove source files. +#Assumes you have ssh-copy-id yourself onto your flare host. +rsync -avz --remove-source-files -e ssh user@your.flare.host:/opt/Flare/TRANSFER /home/misp/feeds/ +#Move into our working directory +cd /home/misp/feeds/TRANSFER/ +#Find compressed AIS files and uncompress them +cat *.tgz | tar -zxvf - -i +#Import into MISP +python /home/misp/cti-toolkit/stixtransclient.py --file /home/misp/feeds/TRANSFER/ -r --misp --misp-url https://misp --misp-key [YOURMISPKEY] --misp-threat 2 --misp-distribution 0 --misp-info "AIS" +#Cleanup transfered files after completion +rm -rf /home/misp/feeds/TRANSFER/ +# ------------------------------DISCLAIMER-------------------------------------# +# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # +# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # +# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # +# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # +# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # +# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # +# THIS UNSUPPORTED SOFTWARE. # +# ------------------------------DISCLAIMER-------------------------------------# From 03b9b2af1d6e3e3cc1eb95023502fab69bcd26f5 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 12:59:31 -0600 Subject: [PATCH 03/20] Update README.md --- automation/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/automation/README.md b/automation/README.md index 10d6e03..33a0c36 100644 --- a/automation/README.md +++ b/automation/README.md @@ -6,27 +6,27 @@ NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM THIS UNSUPPORTED SOFTWARE.* - ------------------------------DISCLAIMER------------------------------------- + Description: Simple set of scripts to automate AIS data pulls into MISP to enable quick correlation, taxonomy tagging and review of data. Automation can also be used as standalone scripts Scripts are simply a working POC and written to only support TAXII 1.1 Author: Cory Kennedy (@corykennedy) - _____ .___ _________ + + ``` _____ .___ _________ / _ \ | |/ _____/ / /_\ \| |\_____ \ / | \ |/ \ \____|__ /___/___v1.0_ / \/ \/ - AIS Automation -============================================================================== - -=[Instructions Summary]=- + AIS Automation + ``` +- Installation Summary +.. Install scripts & crontabs onto your MISP and Flare servers +.. Need help with crontabs? Try: https://crontab.guru/ +.. The below will execute the scripts at 2:30am daily and log all output - * Install scripts & crontabs onto your MISP and Flare servers - * Need help with crontabs? Try: https://crontab.guru/ - * The below will execute the scripts at 2:30am daily and log all output -============================================================================== -=[Dependancies]=- 1. Active AIS Participation * https://www.us-cert.gov/ais From f7398f7a9286b113a829b5846e003cb3e4fedfbe Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:02:51 -0600 Subject: [PATCH 04/20] Update README.md --- automation/README.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/automation/README.md b/automation/README.md index 33a0c36..ba618f4 100644 --- a/automation/README.md +++ b/automation/README.md @@ -1,17 +1,18 @@ -###DISCLAIMER - *ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT - THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO - – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM - SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL - NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE - USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM - THIS UNSUPPORTED SOFTWARE.* - - Description: Simple set of scripts to automate AIS data pulls into MISP to - enable quick correlation, taxonomy tagging and review of data. - Automation can also be used as standalone scripts - Scripts are simply a working POC and written to only support TAXII 1.1 - Author: Cory Kennedy (@corykennedy) +####DISCLAIMER +>*ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT +>THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO +>– WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM +>SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL +>NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE +>USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM +>THIS UNSUPPORTED SOFTWARE.* + + Description: Simple set of scripts to automate AIS data pulls into MISP to enable: + + + Quick correlation, taxonomy tagging, visualization and review of data. + + Automation can also be used as standalone scripts + + *Scripts are simply a working POC and written to only support TAXII 1.1* + + Author: Cory Kennedy (@corykennedy) ``` _____ .___ _________ / _ \ | |/ _____/ From 4cb4e5b98cd8768b1a45316b8fb50fda164f2483 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:04:08 -0600 Subject: [PATCH 05/20] Update README.md --- automation/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/automation/README.md b/automation/README.md index ba618f4..204988f 100644 --- a/automation/README.md +++ b/automation/README.md @@ -14,14 +14,14 @@ + *Scripts are simply a working POC and written to only support TAXII 1.1* + Author: Cory Kennedy (@corykennedy) - ``` _____ .___ _________ - / _ \ | |/ _____/ - / /_\ \| |\_____ \ - / | \ |/ \ - \____|__ /___/___v1.0_ / - \/ \/ - AIS Automation - ``` + _____ .___ _________ + / _ \ | |/ _____/ + / /_\ \| |\_____ \ + / | \ |/ \ + \____|__ /___/___v1.0_ / + \/ \/ + AIS Automation + - Installation Summary .. Install scripts & crontabs onto your MISP and Flare servers .. Need help with crontabs? Try: https://crontab.guru/ From 4f993c71ae6c2ab2340fe7435a8d96b0625c882c Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:07:04 -0600 Subject: [PATCH 06/20] Update README.md --- automation/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/README.md b/automation/README.md index 204988f..67356d1 100644 --- a/automation/README.md +++ b/automation/README.md @@ -11,8 +11,7 @@ + Quick correlation, taxonomy tagging, visualization and review of data. + Automation can also be used as standalone scripts - + *Scripts are simply a working POC and written to only support TAXII 1.1* - + Author: Cory Kennedy (@corykennedy) + + *Scripts are simply a working POC and written to only support TAXII 1.1* _____ .___ _________ / _ \ | |/ _____/ @@ -21,6 +20,7 @@ \____|__ /___/___v1.0_ / \/ \/ AIS Automation + Author: Cory Kennedy (@corykennedy) - Installation Summary .. Install scripts & crontabs onto your MISP and Flare servers From 4f56246f5586778dfb2dbec4ecd574ee3805b41e Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:15:34 -0600 Subject: [PATCH 07/20] Update README.md --- automation/README.md | 155 ++++++++++++------------------------------- 1 file changed, 43 insertions(+), 112 deletions(-) diff --git a/automation/README.md b/automation/README.md index 67356d1..115ab71 100644 --- a/automation/README.md +++ b/automation/README.md @@ -5,144 +5,75 @@ >SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL >NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE >USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM ->THIS UNSUPPORTED SOFTWARE.* +>THIS UNSUPPORTED SOFTWARE. - Description: Simple set of scripts to automate AIS data pulls into MISP to enable: - + Quick correlation, taxonomy tagging, visualization and review of data. - + Automation can also be used as standalone scripts - + *Scripts are simply a working POC and written to only support TAXII 1.1* + Description: Simple set of scripts to automate AIS data pulls into MISP to enable: + + + Quick correlation, taxonomy tagging, visualization and review of data. + + Automation can also be used as standalone scripts + + *Scripts are simply a working POC and written to only support TAXII 1.1* + + _____ .___ _________ + / _ \ | |/ _____/ + / /_\ \| |\_____ \ + / | \ |/ \ + \____|__ /___/___v1.0_ / + \/ \/ + AIS Automation + Author: Cory Kennedy (@corykennedy) + +[Dependancies] +```bash +- Active AIS Participation | https://www.us-cert.gov/ais +- Working Flare instance +- Working MISP instance +- Working CTI-Toolkit on MISP server | https://github.com/certau/cti-toolkit.git +``` + +[Installation Summary] +```bash +- Install scripts & crontabs onto your MISP and Flare servers +- Need help with crontabs? Try: https://crontab.guru/ +- The below will execute the scripts at 2:30am daily and log all output +``` + +###BEGIN FLARE - _____ .___ _________ - / _ \ | |/ _____/ - / /_\ \| |\_____ \ - / | \ |/ \ - \____|__ /___/___v1.0_ / - \/ \/ - AIS Automation - Author: Cory Kennedy (@corykennedy) - -- Installation Summary -.. Install scripts & crontabs onto your MISP and Flare servers -.. Need help with crontabs? Try: https://crontab.guru/ -.. The below will execute the scripts at 2:30am daily and log all output - - - -=[Dependancies]=- - 1. Active AIS Participation - * https://www.us-cert.gov/ais - 2. Working Flare instance - 3. Working MISP instance - 4. Working CTI-Toolkit on MISP server - * https://github.com/certau/cti-toolkit.git -============================================================================== - ------------------------------------------------------------------------------- - =[BEGIN FLARE]= ------------------------------------------------------------------------------- -======= -# ------------------------------DISCLAIMER-------------------------------------# -# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # -# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # -# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # -# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # -# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # -# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # -# THIS UNSUPPORTED SOFTWARE. # -# ------------------------------DISCLAIMER-------------------------------------# -# Description: Simple set of scripts to automate AIS data pulls into MISP to # -# enable quick correlation, taxonomy tagging and review of data. # -# Automation can also be used as standalone scripts # -# Scripts are simply a working POC and written to only support TAXII 1.1 # -# Author: Cory Kennedy (@corykennedy) # -# _____ .___ _________ # -# / _ \ | |/ _____/ # -# / /_\ \| |\_____ \ # -# / | \ |/ \ # -# \____|__ /___/___v1.0_ / # -# \/ \/ # -# AIS Automation # -#==============================================================================# -# -=[Instructions Summary]=- # -# # -# * Install scripts & crontabs onto your MISP and Flare servers # -# * Need help with crontabs? Try: https://crontab.guru/ # -# * The below will execute the scripts at 2:30am daily and log all output # -# # -#==============================================================================# -# -=[Dependancies]=- # -# 1. Active AIS Participation # -# * https://www.us-cert.gov/ais # -# 2. Working Flare instance # -# 3. Working MISP instance # -# 4. Working CTI-Toolkit on MISP server # -# * https://github.com/certau/cti-toolkit.git # -#==============================================================================# - -#------------------------------------------------------------------------------# -# =[BEGIN FLARE]= # -#------------------------------------------------------------------------------# ->>>>>>> Automation [FLARE Script Installation] - +```bash 1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation 2. Move scripts from oss-automation/scripts/flare to your flare server. * Example: mv oss-automation/scripts/flare/* /opt/Flare/scripts/ 3. Install crontabs - +``` [FLARE Crontab Installation] - +```bash 1. From a terminal type: crontab -e 2. Copy and paste the below into your crontab 30 2 * * * /opt/Flare/scripts/CISCP.sh &>/opt/Flare/scripts/logs/CISCP_`date +\%y-\%m-\%d`.out 32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out 3. Save crontab -Note: Flare server is complete. Proceed to MISP +``` +*Flare server is complete. Proceed to MISP - -<<<<<<< HEAD ------------------------------------------------------------------------------- - =[BEGIN MISP]= ------------------------------------------------------------------------------- -======= -#------------------------------------------------------------------------------# -# =[BEGIN MISP]= # -#------------------------------------------------------------------------------# ->>>>>>> Automation +###BEGIN MISP [MISP Script Installation] +```bash 1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation 2. Move scripts from oss-automation/scripts/misp to your MISP server * Example: mv oss-automation/scripts/misp/* /home/misp/scripts/ 3. Install crontabs +``` [MISP Crontab Installation] - +```bash 1. From a terminal type: crontab -e 2. Copy and paste the below into your crontab 30 2 * * * /home/misp/scripts/MISP.sh 3. Save crontab -Note: MISP server is complete. +``` +*Note: MISP server is complete! -<<<<<<< HEAD - ------------------------------DISCLAIMER------------------------------------- - ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT - THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO - – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM - SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL - NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE - USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM - THIS UNSUPPORTED SOFTWARE. - ------------------------------DISCLAIMER------------------------------------- -======= -# ------------------------------DISCLAIMER-------------------------------------# -# ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT # -# THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO # -# – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM # -# SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL # -# NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE # -# USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM # -# THIS UNSUPPORTED SOFTWARE. # -# ------------------------------DISCLAIMER-------------------------------------# ->>>>>>> Automation From bfe1221718d04ac5bc1c6f4157937a2c37519d8a Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:16:56 -0600 Subject: [PATCH 08/20] Update README.md --- automation/README.md | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/automation/README.md b/automation/README.md index 115ab71..bf6a68e 100644 --- a/automation/README.md +++ b/automation/README.md @@ -1,11 +1,11 @@ ####DISCLAIMER ->*ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT ->THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO ->– WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM ->SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL ->NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE ->USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM ->THIS UNSUPPORTED SOFTWARE. +>>ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT +>>THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO +>>– WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM +>>SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL +>>NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE +>>USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM +>>THIS UNSUPPORTED SOFTWARE. Description: Simple set of scripts to automate AIS data pulls into MISP to enable: @@ -23,7 +23,7 @@ AIS Automation Author: Cory Kennedy (@corykennedy) -[Dependancies] +##[Dependancies] ```bash - Active AIS Participation | https://www.us-cert.gov/ais - Working Flare instance @@ -31,23 +31,21 @@ - Working CTI-Toolkit on MISP server | https://github.com/certau/cti-toolkit.git ``` -[Installation Summary] +##[Installation Summary] ```bash - Install scripts & crontabs onto your MISP and Flare servers - Need help with crontabs? Try: https://crontab.guru/ - The below will execute the scripts at 2:30am daily and log all output ``` - -###BEGIN FLARE - -[FLARE Script Installation] + +##[FLARE Script Installation] ```bash 1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation 2. Move scripts from oss-automation/scripts/flare to your flare server. * Example: mv oss-automation/scripts/flare/* /opt/Flare/scripts/ 3. Install crontabs ``` -[FLARE Crontab Installation] +###[FLARE Crontab Installation] ```bash 1. From a terminal type: crontab -e 2. Copy and paste the below into your crontab @@ -55,11 +53,9 @@ 32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out 3. Save crontab ``` -*Flare server is complete. Proceed to MISP - -###BEGIN MISP +Flare server is complete. Proceed to MISP -[MISP Script Installation] +##[MISP Script Installation] ```bash 1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation @@ -68,12 +64,12 @@ 3. Install crontabs ``` -[MISP Crontab Installation] +###[MISP Crontab Installation] ```bash 1. From a terminal type: crontab -e 2. Copy and paste the below into your crontab 30 2 * * * /home/misp/scripts/MISP.sh 3. Save crontab ``` -*Note: MISP server is complete! +Note: MISP server is complete! From 882a5f62b0afb4a82d73fc257ca092a057b67376 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:20:06 -0600 Subject: [PATCH 09/20] Update README.md --- automation/README.md | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/automation/README.md b/automation/README.md index bf6a68e..9bf6afd 100644 --- a/automation/README.md +++ b/automation/README.md @@ -23,52 +23,58 @@ AIS Automation Author: Cory Kennedy (@corykennedy) -##[Dependancies] +##Dependancies ```bash -- Active AIS Participation | https://www.us-cert.gov/ais -- Working Flare instance -- Working MISP instance -- Working CTI-Toolkit on MISP server | https://github.com/certau/cti-toolkit.git + + Active AIS Participation | https://www.us-cert.gov/ais + + Working Flare instance | https://github.com/bcmc/oss + + Working MISP instance | https://github.com/MISP/MISP + + Working CTI-Toolkit | https://github.com/certau/cti-toolkit.git (Installed on MISP server) ``` -##[Installation Summary] +##Installation Summary ```bash - Install scripts & crontabs onto your MISP and Flare servers - Need help with crontabs? Try: https://crontab.guru/ - The below will execute the scripts at 2:30am daily and log all output ``` -##[FLARE Script Installation] +##FLARE Script Installation ```bash 1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation 2. Move scripts from oss-automation/scripts/flare to your flare server. * Example: mv oss-automation/scripts/flare/* /opt/Flare/scripts/ 3. Install crontabs ``` -###[FLARE Crontab Installation] +###FLARE Crontab Installation ```bash 1. From a terminal type: crontab -e + 2. Copy and paste the below into your crontab -30 2 * * * /opt/Flare/scripts/CISCP.sh &>/opt/Flare/scripts/logs/CISCP_`date +\%y-\%m-\%d`.out -32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out + 30 2 * * * /opt/Flare/scripts/CISCP.sh &>/opt/Flare/scripts/logs/CISCP_`date +\%y-\%m-\%d`.out + 32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out + 3. Save crontab ``` Flare server is complete. Proceed to MISP -##[MISP Script Installation] +##MISP Script Installation ```bash 1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation + 2. Move scripts from oss-automation/scripts/misp to your MISP server * Example: mv oss-automation/scripts/misp/* /home/misp/scripts/ + 3. Install crontabs ``` -###[MISP Crontab Installation] +###MISP Crontab Installation ```bash 1. From a terminal type: crontab -e + 2. Copy and paste the below into your crontab -30 2 * * * /home/misp/scripts/MISP.sh + 30 2 * * * /home/misp/scripts/MISP.sh + 3. Save crontab ``` Note: MISP server is complete! From 72db5c48bcee638884a563e74dda59f5443dd671 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:28:16 -0600 Subject: [PATCH 10/20] Update README.md --- automation/README.md | 46 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/automation/README.md b/automation/README.md index 9bf6afd..faf46c6 100644 --- a/automation/README.md +++ b/automation/README.md @@ -1,27 +1,27 @@ -####DISCLAIMER ->>ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT ->>THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO ->>– WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM ->>SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL ->>NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE ->>USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM ->>THIS UNSUPPORTED SOFTWARE. - - - Description: Simple set of scripts to automate AIS data pulls into MISP to enable: - - + Quick correlation, taxonomy tagging, visualization and review of data. - + Automation can also be used as standalone scripts - + *Scripts are simply a working POC and written to only support TAXII 1.1* +##DISCLAIMER +```bash + ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT + THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO + – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM + SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL + NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE + USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM + THIS UNSUPPORTED SOFTWARE. + ``` - _____ .___ _________ - / _ \ | |/ _____/ - / /_\ \| |\_____ \ - / | \ |/ \ - \____|__ /___/___v1.0_ / - \/ \/ - AIS Automation - Author: Cory Kennedy (@corykennedy) +####Description: Simple set of scripts to automate AIS data pulls into MISP to enable +```bash +Automation to support the quick correlation, tagging, and visualization of AIS data. + _____ .___ _________ + / _ \ | |/ _____/ + / /_\ \| |\_____ \ +/ | \ |/ \ Author: Cory Kennedy (@corykennedy) +\____|__ /___/___v1.0_ / + \/ \/ + + + Automation can also be used as standalone scripts + + Scripts are simply a working POC and written to only support TAXII 1.1 +``` ##Dependancies ```bash From 06f61c05b409e8533c623b9af8ac907d64747c90 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:29:04 -0600 Subject: [PATCH 11/20] Update README.md --- automation/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automation/README.md b/automation/README.md index faf46c6..c23181f 100644 --- a/automation/README.md +++ b/automation/README.md @@ -9,7 +9,8 @@ THIS UNSUPPORTED SOFTWARE. ``` -####Description: Simple set of scripts to automate AIS data pulls into MISP to enable +####Description + ```bash Automation to support the quick correlation, tagging, and visualization of AIS data. _____ .___ _________ From 218ff21694335b8c3c4bd4442a9fdb8712b2b8f0 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:29:32 -0600 Subject: [PATCH 12/20] Update README.md --- automation/README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/automation/README.md b/automation/README.md index c23181f..19111e7 100644 --- a/automation/README.md +++ b/automation/README.md @@ -1,5 +1,5 @@ ##DISCLAIMER -```bash + ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM @@ -7,7 +7,6 @@ NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM THIS UNSUPPORTED SOFTWARE. - ``` ####Description From 43eb82b5e849814d0fd9e875886255c734f0056e Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:30:15 -0600 Subject: [PATCH 13/20] Update README.md --- automation/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/automation/README.md b/automation/README.md index 19111e7..51f1817 100644 --- a/automation/README.md +++ b/automation/README.md @@ -1,17 +1,17 @@ ##DISCLAIMER - ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT - THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO - – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM - SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL - NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE - USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM - THIS UNSUPPORTED SOFTWARE. + ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT + THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO + – WITHOUT LIMITATION – ANY COMPUTER SYSTEM OR LOSS OF DATA THAT RESULTS FROM + SUCH ACTIVITIES. SHOULD IT PROVE DEFECTIVE, USER ASSUMES THE COST OF ALL + NECESSARY SERVICING, REPAIR AND/OR CORRECTION. IT IS THEREFORE UP TO THE + USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM + THIS UNSUPPORTED SOFTWARE. ####Description ```bash -Automation to support the quick correlation, tagging, and visualization of AIS data. + Automation to support the quick correlation, tagging, and visualization of AIS data. _____ .___ _________ / _ \ | |/ _____/ / /_\ \| |\_____ \ From f97e01ba471685959e4305e282033aa7eca05173 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:30:41 -0600 Subject: [PATCH 14/20] Update README.md --- automation/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automation/README.md b/automation/README.md index 51f1817..2b145ab 100644 --- a/automation/README.md +++ b/automation/README.md @@ -1,4 +1,4 @@ -##DISCLAIMER +#####DISCLAIMER ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO @@ -8,7 +8,7 @@ USER TO TAKE ADEQUATE PRECAUTION AGAINST POSSIBLE DAMAGES RESULTING FROM THIS UNSUPPORTED SOFTWARE. -####Description +##Description ```bash Automation to support the quick correlation, tagging, and visualization of AIS data. From 4ea4dfb477260a14ea46c8821e6acefccd4c4f4c Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:31:33 -0600 Subject: [PATCH 15/20] Update README.md --- automation/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/README.md b/automation/README.md index 2b145ab..7292ef7 100644 --- a/automation/README.md +++ b/automation/README.md @@ -17,7 +17,7 @@ / /_\ \| |\_____ \ / | \ |/ \ Author: Cory Kennedy (@corykennedy) \____|__ /___/___v1.0_ / - \/ \/ + \/ AUTOMISPER \/ + Automation can also be used as standalone scripts + Scripts are simply a working POC and written to only support TAXII 1.1 From 5e389fafb73a8f4f35699db9e3a7717a94695ed6 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:41:25 -0600 Subject: [PATCH 16/20] Update README.md --- automation/README.md | 71 +++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/automation/README.md b/automation/README.md index 7292ef7..8b817cc 100644 --- a/automation/README.md +++ b/automation/README.md @@ -38,44 +38,55 @@ - The below will execute the scripts at 2:30am daily and log all output ``` -##FLARE Script Installation -```bash -1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation -2. Move scripts from oss-automation/scripts/flare to your flare server. - * Example: mv oss-automation/scripts/flare/* /opt/Flare/scripts/ -3. Install crontabs -``` -###FLARE Crontab Installation -```bash -1. From a terminal type: crontab -e +##FLARE Installation -2. Copy and paste the below into your crontab +Step 1: + + ```bash + git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation + ``` +Step 2: Move scripts from oss-automation/scripts/flare to your flare server + Example: + + ```bash + mv oss-automation/scripts/flare/* /opt/Flare/scripts/ + ``` + +Step 3: Install crontabs + ```bash + export VISUAL=nano; crontab -e + ``` +Example filename - GNU nano 2.3.1 File: /tmp/crontab.aisautomation +```bash 30 2 * * * /opt/Flare/scripts/CISCP.sh &>/opt/Flare/scripts/logs/CISCP_`date +\%y-\%m-\%d`.out 32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out - -3. Save crontab ``` -Flare server is complete. Proceed to MISP +Save with ctrl + x -##MISP Script Installation +###### Flare server is complete. Proceed to MISP -```bash -1. git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation - -2. Move scripts from oss-automation/scripts/misp to your MISP server - * Example: mv oss-automation/scripts/misp/* /home/misp/scripts/ - -3. Install crontabs -``` +##MISP Installation +Step 1: -###MISP Crontab Installation + ```bash + git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation + ``` +Step 2: Move scripts from oss-automation/scripts/misp to your MISP server + Example: + + ```bash + mv oss-automation/scripts/misp/* /home/misp/scripts/ + ``` + +Step 3: Install crontabs + ```bash + export VISUAL=nano; crontab -e + ``` +Example filename - GNU nano 2.3.1 File: /tmp/crontab.aisautomation ```bash -1. From a terminal type: crontab -e - -2. Copy and paste the below into your crontab 30 2 * * * /home/misp/scripts/MISP.sh - -3. Save crontab ``` -Note: MISP server is complete! +Save with ctrl + x + +###### MISP server is complete! YOU ARE DONE! From bc5bf71b9cdd33b71cf74751d5576fe3f87ec54e Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:43:59 -0600 Subject: [PATCH 17/20] Update README.md --- automation/README.md | 54 ++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/automation/README.md b/automation/README.md index 8b817cc..881b2a0 100644 --- a/automation/README.md +++ b/automation/README.md @@ -1,4 +1,4 @@ -#####DISCLAIMER +####DISCLAIMER ANY DOWNLOAD AND USE OF THIS UNSUPPORTED SOFTWARE PROGRAM PRODUCT IS DONE AT THE USERS OWN RISK AND THE USER WILL BE SOLELY RESPONSIBLE FOR ANY DAMAGE TO @@ -25,10 +25,10 @@ ##Dependancies ```bash - + Active AIS Participation | https://www.us-cert.gov/ais - + Working Flare instance | https://github.com/bcmc/oss - + Working MISP instance | https://github.com/MISP/MISP - + Working CTI-Toolkit | https://github.com/certau/cti-toolkit.git (Installed on MISP server) + + Active AIS Participation | https://www.us-cert.gov/ais + + Working Flare instance | https://github.com/bcmc/oss + + Working MISP instance | https://github.com/MISP/MISP + + Working CTI-Toolkit | https://github.com/certau/cti-toolkit.git (Installed on MISP server) ``` ##Installation Summary @@ -40,52 +40,48 @@ ##FLARE Installation -Step 1: +Step 1: GIT the automation branch - ```bash git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation - ``` + Step 2: Move scripts from oss-automation/scripts/flare to your flare server Example: - - ```bash +```bash mv oss-automation/scripts/flare/* /opt/Flare/scripts/ - ``` +``` Step 3: Install crontabs - ```bash + export VISUAL=nano; crontab -e - ``` + Example filename - GNU nano 2.3.1 File: /tmp/crontab.aisautomation -```bash - 30 2 * * * /opt/Flare/scripts/CISCP.sh &>/opt/Flare/scripts/logs/CISCP_`date +\%y-\%m-\%d`.out - 32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out -``` + + 30 2 * * * /opt/Flare/scripts/CISCP.sh &>/opt/Flare/scripts/logs/CISCP_`date +\%y-\%m-\%d`.out + 32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out + Save with ctrl + x ###### Flare server is complete. Proceed to MISP ##MISP Installation -Step 1: - ```bash +Step 1: GIT the automation branch + git clone -b Automation --single-branch https://github.com/NoDataFound/oss.git oss-automation - ``` + Step 2: Move scripts from oss-automation/scripts/misp to your MISP server Example: - ```bash - mv oss-automation/scripts/misp/* /home/misp/scripts/ - ``` - + mv oss-automation/scripts/misp/* /home/misp/scripts/ + Step 3: Install crontabs - ```bash + export VISUAL=nano; crontab -e - ``` + Example filename - GNU nano 2.3.1 File: /tmp/crontab.aisautomation -```bash - 30 2 * * * /home/misp/scripts/MISP.sh -``` + + 30 2 * * * /home/misp/scripts/MISP.sh + Save with ctrl + x ###### MISP server is complete! YOU ARE DONE! From b3f88a833d276a30f012c4f3ad78250093b90112 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:45:45 -0600 Subject: [PATCH 18/20] Update README.md --- automation/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automation/README.md b/automation/README.md index 881b2a0..f3dcd63 100644 --- a/automation/README.md +++ b/automation/README.md @@ -28,7 +28,7 @@ + Active AIS Participation | https://www.us-cert.gov/ais + Working Flare instance | https://github.com/bcmc/oss + Working MISP instance | https://github.com/MISP/MISP - + Working CTI-Toolkit | https://github.com/certau/cti-toolkit.git (Installed on MISP server) + + Working CTI-Toolkit | https://github.com/certau/cti-toolkit.git (MISP server) ``` ##Installation Summary @@ -54,7 +54,7 @@ Step 3: Install crontabs export VISUAL=nano; crontab -e -Example filename - GNU nano 2.3.1 File: /tmp/crontab.aisautomation +Example - GNU nano 2.3.1 File: /tmp/crontab.aisautomation 30 2 * * * /opt/Flare/scripts/CISCP.sh &>/opt/Flare/scripts/logs/CISCP_`date +\%y-\%m-\%d`.out 32 2 * * * /opt/Flare/scripts/AIS.sh &>/opt/Flare/scripts/logs/AIS_`date +\%y-\%m-\%d`.out @@ -78,7 +78,7 @@ Step 3: Install crontabs export VISUAL=nano; crontab -e -Example filename - GNU nano 2.3.1 File: /tmp/crontab.aisautomation +Example - GNU nano 2.3.1 File: /tmp/crontab.aisautomation 30 2 * * * /home/misp/scripts/MISP.sh From 3b4a485b657c2510660acf13e9eaa1f34de48240 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:45:58 -0600 Subject: [PATCH 19/20] Update README.md --- automation/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automation/README.md b/automation/README.md index f3dcd63..01643c9 100644 --- a/automation/README.md +++ b/automation/README.md @@ -78,7 +78,7 @@ Step 3: Install crontabs export VISUAL=nano; crontab -e -Example - GNU nano 2.3.1 File: /tmp/crontab.aisautomation +Example - GNU nano 2.3.1 File: /tmp/crontab.mispautomation 30 2 * * * /home/misp/scripts/MISP.sh From a6aeda00500869a7f1aa59c58223b858ad111d99 Mon Sep 17 00:00:00 2001 From: Cory Kennedy Date: Tue, 31 Jan 2017 13:46:40 -0600 Subject: [PATCH 20/20] Update README.md --- automation/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automation/README.md b/automation/README.md index 01643c9..7c8e950 100644 --- a/automation/README.md +++ b/automation/README.md @@ -33,9 +33,9 @@ ##Installation Summary ```bash -- Install scripts & crontabs onto your MISP and Flare servers -- Need help with crontabs? Try: https://crontab.guru/ -- The below will execute the scripts at 2:30am daily and log all output + + Install scripts & crontabs onto your MISP and Flare servers + + Need help with crontabs? Try: https://crontab.guru/ + + The below will execute the scripts at 2:30am daily and log all output ``` ##FLARE Installation