Skip to content

Enable wpad capabilities #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
computezrmle wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Enable wpad capabilities #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
127 changes: 84 additions & 43 deletions vbox/ATLASJobWrapper-dev.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

# Bootscript script for running ATLAS@Home tasks inside a virtual machine


#############################################
# set the VM's I/O scheduler to the most simple one
# this avoids I/O requests being resorted twice (guest/host)
Expand All @@ -31,59 +32,66 @@ else
fi
fi

if [ ! -z "$ATLAS_BRANCH_SUFFIX" ]; then
echo "This is the ${ATLAS_BRANCH_SUFFIX} version of the ATLAS job wrapper" | vboxmonitor
fi

#############################################
# Copy input files from shared dir to run dir
# can take a while to copy many files or large files
# hence, do it as bg process, remember it's pid
# and "wait" for that pid later

function cp_input_files {
rm -rf /home/atlas/RunAtlas
mkdir -p /home/atlas/RunAtlas
sudo find /home/atlas/shared/ -type f -exec cp {} /home/atlas/RunAtlas/ \;
sudo chown -R atlas:atlas /home/atlas/

# Move inputs into /data so that container works. Requires container_options: -B /data in AGIS
sudo mkdir /data
sudo chown atlas:atlas /data
find /home/atlas/RunAtlas -name "ATLAS.root_0" -exec mv {} /data/ \;
find /data -type f -execdir ln -s /data/{} /home/atlas/RunAtlas/{} \;
}

stdbuf -oL echo "[INFO] Copying input files into RunAtlas..." | vboxmonitor
cp_input_files &
bg_pid_cp_input_files=$!


echo "Copying input files" | vboxmonitor
rm -rf /home/atlas/RunAtlas
mkdir -p /home/atlas/RunAtlas
sudo find /home/atlas/shared/ -type f -exec cp {} /home/atlas/RunAtlas/ \;
sudo chown -R atlas:atlas /home/atlas/
#############################################
# create a wpad file

rm -f /home/atlas/create_local_wpad
cp /cvmfs/atlas.cern.ch/repo/sw/BOINC/agent/create_local_wpad-${ATLAS_BRANCH_SUFFIX} -o /home/atlas/create_local_wpad
. /home/atlas/create_local_wpad
if [[ $? != 0 ]]; then
stdbuf -oL echo "[ERROR] Failed to source 'create_local_wpad'" | vboxmonitor
sleep 90
sudo shutdown now
fi

bs_create_local_wpad

# Move inputs into /data so that container works. Requires container_options: -B /data in AGIS
sudo mkdir /data
sudo chown atlas:atlas /data
find /home/atlas/RunAtlas -name "ATLAS.root_0" -exec mv {} /data/ \;
find /data -type f -execdir ln -s /data/{} /home/atlas/RunAtlas/{} \;
echo "Copied input files into RunAtlas." | vboxmonitor

#############################################
# add required repositories to the CVMFS configuration
# update and reload modified CVMFS configuration

stdbuf -oL echo "[INFO] Updating CVMFS configuration..." | vboxmonitor
sudo sh -c "echo \"CVMFS_REPOSITORIES=atlas.cern.ch,atlas-condb.cern.ch\" >> /etc/cvmfs/default.local"
sudo cvmfs_config reload

#############################################
# set up http proxy for hosts behind firewall

init_data=/home/atlas/shared/init_data.xml
if [ -f $init_data ];
then
use_proxy=`grep use_http_proxy/ $init_data`
proxy_server=` grep http_server_name $init_data |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
proxy_port=` grep http_server_port $init_data |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"`
if [ ! -z "$use_proxy" -a ! -z "$proxy_server" -a ! -z "$proxy_port" ];
then
hproxy=http://$proxy_server:$proxy_port
export http_proxy=$hproxy
echo "Detected user-configured HTTP proxy at ${hproxy} - will set in /etc/cvmfs/default.local"|vboxmonitor
sudo sh -c "echo \"CVMFS_HTTP_PROXY=\\\"${hproxy};DIRECT\\\"\" >> /etc/cvmfs/default.local"
else
echo "This VM did not configure a local http proxy via BOINC."|vboxmonitor
echo "Small home clusters do not require a local http proxy but it is suggested if"|vboxmonitor
echo "more than 10 cores throughout the same LAN segment are regularly running ATLAS like tasks."|vboxmonitor
echo "Further information can be found at the LHC@home message board."|vboxmonitor
fi
sudo cvmfs_config reload
echo "Running cvmfs_config stat atlas.cern.ch"|vboxmonitor
cvmfs_config stat atlas.cern.ch|vboxmonitor
# print a hint to the logfile whether openhtc.io and/or a local proxy is used.
# required to identify possible misconfigurations
cvmfs_excerpt=($(cut -d ' ' -f 1,17,18 <<< "$(tail -n1 <<< "$(cvmfs_config stat atlas.cern.ch)")"))
stdbuf -oL echo "[INFO] Excerpt from \"cvmfs_config stat\": VERSION HOST PROXY" | vboxmonitor
stdbuf -oL echo "[INFO] $(echo "${cvmfs_excerpt[0]} ${cvmfs_excerpt[1]%"/cvmfs/atlas.cern.ch"} ${cvmfs_excerpt[2]}")" | vboxmonitor

if [[ "$http_proxy" != "" ]]; then
stdbuf -oL echo "[INFO] Environment HTTP proxy: $http_proxy" | vboxmonitor
else
echo "miss $init_data"|vboxmonitor
stdbuf -oL echo "[INFO] Environment HTTP proxy: not set" | vboxmonitor
fi


################################################
# Copy data into web area for graphics interface

Expand All @@ -98,22 +106,28 @@ sudo sh -c "echo $credit > /var/www/html/credit.txt"
sudo chmod a+r /var/www/html/user.txt /var/www/html/credit.txt
# Extract total number of events
sudo sh -c "tar xzf /home/atlas/RunAtlas/input.tar.gz -O | grep --binary-file=text -m1 -o -E maxEvents%3D[[:digit:]]+ | cut -dD -f2 > /var/www/html/totalevents.txt"
echo "copied the webapp to /var/www"|vboxmonitor

stdbuf -oL echo "[INFO] Copied the webapp to /var/www" | vboxmonitor


##############################################
# Cron to generate info for graphics interface

sudo sh -c 'echo "* * * * * root cd /var/www/html; python extract_info.py" > /etc/cron.d/atlas-top'


wait $bg_pid_cp_input_files
stdbuf -oL echo "[INFO] Copied input files into RunAtlas." | vboxmonitor

#############################################
# set up env for running Mcore job

core_number=`nproc`
if [ $core_number -gt 1 ];then
export ATHENA_PROC_NUMBER=$core_number
echo ATHENA_PROC_NUMBER=$ATHENA_PROC_NUMBER | vboxmonitor
stdbuf -oL echo "[INFO] ATHENA_PROC_NUMBER=$ATHENA_PROC_NUMBER" | vboxmonitor
else
echo core_number=$core_number | vboxmonitor
stdbuf -oL echo "[INFO] core_number=$core_number" | vboxmonitor
fi

#######################################
Expand All @@ -137,7 +151,34 @@ monitor_script="/home/atlas/tty3monitor/setup_top_on_tty3"
########################################
# Start the job

function patch_start_atlas {
stdbuf -oL echo "[DEVINFO] Apply experimental patch to start_atlas.sh" | vboxmonitor
stdbuf -oL echo "[DEVINFO] To make it permanent modify the original source." | vboxmonitor

rm -f /home/atlas/patchsnippet_start_atlas
cp /cvmfs/atlas.cern.ch/repo/sw/BOINC/agent/patchsnippet_start_atlas_full-${ATLAS_BRANCH_SUFFIX} /home/atlas/patchsnippet_start_atlas
# at least the lean patch MUST be used to make Frontier work with wpad.dat!
# uncomment the next line if the lean patch should be used instead of the full patch
#cp /cvmfs/atlas.cern.ch/repo/sw/BOINC/agent/patchsnippet_start_atlas_lean-${ATLAS_BRANCH_SUFFIX} /home/atlas/patchsnippet_start_atlas
if [ "$?" -ne "0" ]; then
stdbuf -oL echo "[ERROR] Failed to copy patchsnippet_start_atlas" | vboxmonitor
sleep 90
sudo shutdown now
fi

part1="$(sed '/Check for user-specific proxy/q' /home/atlas/RunAtlas/start_atlas.sh)"
part2="$(sed -n '/Set CERN-internal proxy for CERN hosts/,$ p' /home/atlas/RunAtlas/start_atlas.sh)"

echo "$part1" > /home/atlas/RunAtlas/start_atlas.sh
cat /home/atlas/patchsnippet_start_atlas >> /home/atlas/RunAtlas/start_atlas.sh
echo "$part2" >> /home/atlas/RunAtlas/start_atlas.sh
## cp the modification to the shared folder to allow monitoring from the host
sudo sh -c "cp /home/atlas/RunAtlas/start_atlas.sh /home/atlas/shared/start_atlas.mod.txt"
}


if [ -f /home/atlas/RunAtlas/start_atlas.sh ];then
patch_start_atlas
cd /home/atlas/RunAtlas/
pandaid=$(tar xzf input.tar.gz -O | grep --binary-file=text -m1 -o PandaID=..........)
taskid=$(tar xzf input.tar.gz -O | grep --binary-file=text -m1 -o taskID=........)
Expand Down
Loading