From 365e05e350e564d62c1c89131622fa08c33eabe3 Mon Sep 17 00:00:00 2001 From: computezrmle <57127745+computezrmle@users.noreply.github.com> Date: Fri, 2 Dec 2022 17:05:54 +0100 Subject: [PATCH] Enable wpad capabilities 'create_local_wpad' ensures a valid wpad.dat is present inside the VM 'patchsnippet...' modifies the 'start_atlas' script that is created by a different source. Would be better to modify it at the original source. --- vbox/ATLASJobWrapper-dev.sh | 127 ++++++++---- vbox/create_local_wpad-dev | 267 +++++++++++++++++++++++++ vbox/patchsnippet_start_atlas_full-dev | 38 ++++ vbox/patchsnippet_start_atlas_lean-dev | 12 ++ 4 files changed, 401 insertions(+), 43 deletions(-) create mode 100644 vbox/create_local_wpad-dev create mode 100644 vbox/patchsnippet_start_atlas_full-dev create mode 100644 vbox/patchsnippet_start_atlas_lean-dev diff --git a/vbox/ATLASJobWrapper-dev.sh b/vbox/ATLASJobWrapper-dev.sh index 1af1323..6a495ac 100644 --- a/vbox/ATLASJobWrapper-dev.sh +++ b/vbox/ATLASJobWrapper-dev.sh @@ -16,6 +16,7 @@ # Bootscript script for running ATLAS@Home tasks inside a virtual machine + ############################################# # set the VM's I/O scheduler to the most simple one # this avoids I/O requests being resorted twice (guest/host) @@ -31,59 +32,66 @@ else fi fi -if [ ! -z "$ATLAS_BRANCH_SUFFIX" ]; then - echo "This is the ${ATLAS_BRANCH_SUFFIX} version of the ATLAS job wrapper" | vboxmonitor -fi ############################################# # Copy input files from shared dir to run dir +# can take a while to copy many files or large files +# hence, do it as bg process, remember it's pid +# and "wait" for that pid later + +function cp_input_files { + rm -rf /home/atlas/RunAtlas + mkdir -p /home/atlas/RunAtlas + sudo find /home/atlas/shared/ -type f -exec cp {} /home/atlas/RunAtlas/ \; + sudo chown -R atlas:atlas /home/atlas/ + + # Move inputs into /data so that container works. Requires container_options: -B /data in AGIS + sudo mkdir /data + sudo chown atlas:atlas /data + find /home/atlas/RunAtlas -name "ATLAS.root_0" -exec mv {} /data/ \; + find /data -type f -execdir ln -s /data/{} /home/atlas/RunAtlas/{} \; +} + +stdbuf -oL echo "[INFO] Copying input files into RunAtlas..." | vboxmonitor +cp_input_files & +bg_pid_cp_input_files=$! + -echo "Copying input files" | vboxmonitor -rm -rf /home/atlas/RunAtlas -mkdir -p /home/atlas/RunAtlas -sudo find /home/atlas/shared/ -type f -exec cp {} /home/atlas/RunAtlas/ \; -sudo chown -R atlas:atlas /home/atlas/ +############################################# +# create a wpad file + +rm -f /home/atlas/create_local_wpad +cp /cvmfs/atlas.cern.ch/repo/sw/BOINC/agent/create_local_wpad-${ATLAS_BRANCH_SUFFIX} -o /home/atlas/create_local_wpad +. /home/atlas/create_local_wpad +if [[ $? != 0 ]]; then + stdbuf -oL echo "[ERROR] Failed to source 'create_local_wpad'" | vboxmonitor + sleep 90 + sudo shutdown now +fi + +bs_create_local_wpad -# Move inputs into /data so that container works. Requires container_options: -B /data in AGIS -sudo mkdir /data -sudo chown atlas:atlas /data -find /home/atlas/RunAtlas -name "ATLAS.root_0" -exec mv {} /data/ \; -find /data -type f -execdir ln -s /data/{} /home/atlas/RunAtlas/{} \; -echo "Copied input files into RunAtlas." | vboxmonitor ############################################# -# add required repositories to the CVMFS configuration +# update and reload modified CVMFS configuration +stdbuf -oL echo "[INFO] Updating CVMFS configuration..." | vboxmonitor sudo sh -c "echo \"CVMFS_REPOSITORIES=atlas.cern.ch,atlas-condb.cern.ch\" >> /etc/cvmfs/default.local" +sudo cvmfs_config reload -############################################# -# set up http proxy for hosts behind firewall - -init_data=/home/atlas/shared/init_data.xml -if [ -f $init_data ]; -then - use_proxy=`grep use_http_proxy/ $init_data` - proxy_server=` grep http_server_name $init_data |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"` - proxy_port=` grep http_server_port $init_data |awk -F '>' '{print $2}'|awk -F "<" '{print $1}'|sed -e "s# #_#g"` - if [ ! -z "$use_proxy" -a ! -z "$proxy_server" -a ! -z "$proxy_port" ]; - then - hproxy=http://$proxy_server:$proxy_port - export http_proxy=$hproxy - echo "Detected user-configured HTTP proxy at ${hproxy} - will set in /etc/cvmfs/default.local"|vboxmonitor - sudo sh -c "echo \"CVMFS_HTTP_PROXY=\\\"${hproxy};DIRECT\\\"\" >> /etc/cvmfs/default.local" - else - echo "This VM did not configure a local http proxy via BOINC."|vboxmonitor - echo "Small home clusters do not require a local http proxy but it is suggested if"|vboxmonitor - echo "more than 10 cores throughout the same LAN segment are regularly running ATLAS like tasks."|vboxmonitor - echo "Further information can be found at the LHC@home message board."|vboxmonitor - fi - sudo cvmfs_config reload - echo "Running cvmfs_config stat atlas.cern.ch"|vboxmonitor - cvmfs_config stat atlas.cern.ch|vboxmonitor +# print a hint to the logfile whether openhtc.io and/or a local proxy is used. +# required to identify possible misconfigurations +cvmfs_excerpt=($(cut -d ' ' -f 1,17,18 <<< "$(tail -n1 <<< "$(cvmfs_config stat atlas.cern.ch)")")) +stdbuf -oL echo "[INFO] Excerpt from \"cvmfs_config stat\": VERSION HOST PROXY" | vboxmonitor +stdbuf -oL echo "[INFO] $(echo "${cvmfs_excerpt[0]} ${cvmfs_excerpt[1]%"/cvmfs/atlas.cern.ch"} ${cvmfs_excerpt[2]}")" | vboxmonitor + +if [[ "$http_proxy" != "" ]]; then + stdbuf -oL echo "[INFO] Environment HTTP proxy: $http_proxy" | vboxmonitor else - echo "miss $init_data"|vboxmonitor + stdbuf -oL echo "[INFO] Environment HTTP proxy: not set" | vboxmonitor fi + ################################################ # Copy data into web area for graphics interface @@ -98,22 +106,28 @@ sudo sh -c "echo $credit > /var/www/html/credit.txt" sudo chmod a+r /var/www/html/user.txt /var/www/html/credit.txt # Extract total number of events sudo sh -c "tar xzf /home/atlas/RunAtlas/input.tar.gz -O | grep --binary-file=text -m1 -o -E maxEvents%3D[[:digit:]]+ | cut -dD -f2 > /var/www/html/totalevents.txt" -echo "copied the webapp to /var/www"|vboxmonitor + +stdbuf -oL echo "[INFO] Copied the webapp to /var/www" | vboxmonitor + ############################################## # Cron to generate info for graphics interface sudo sh -c 'echo "* * * * * root cd /var/www/html; python extract_info.py" > /etc/cron.d/atlas-top' + +wait $bg_pid_cp_input_files +stdbuf -oL echo "[INFO] Copied input files into RunAtlas." | vboxmonitor + ############################################# # set up env for running Mcore job core_number=`nproc` if [ $core_number -gt 1 ];then export ATHENA_PROC_NUMBER=$core_number - echo ATHENA_PROC_NUMBER=$ATHENA_PROC_NUMBER | vboxmonitor + stdbuf -oL echo "[INFO] ATHENA_PROC_NUMBER=$ATHENA_PROC_NUMBER" | vboxmonitor else - echo core_number=$core_number | vboxmonitor + stdbuf -oL echo "[INFO] core_number=$core_number" | vboxmonitor fi ####################################### @@ -137,7 +151,34 @@ monitor_script="/home/atlas/tty3monitor/setup_top_on_tty3" ######################################## # Start the job +function patch_start_atlas { + stdbuf -oL echo "[DEVINFO] Apply experimental patch to start_atlas.sh" | vboxmonitor + stdbuf -oL echo "[DEVINFO] To make it permanent modify the original source." | vboxmonitor + + rm -f /home/atlas/patchsnippet_start_atlas + cp /cvmfs/atlas.cern.ch/repo/sw/BOINC/agent/patchsnippet_start_atlas_full-${ATLAS_BRANCH_SUFFIX} /home/atlas/patchsnippet_start_atlas + # at least the lean patch MUST be used to make Frontier work with wpad.dat! + # uncomment the next line if the lean patch should be used instead of the full patch + #cp /cvmfs/atlas.cern.ch/repo/sw/BOINC/agent/patchsnippet_start_atlas_lean-${ATLAS_BRANCH_SUFFIX} /home/atlas/patchsnippet_start_atlas + if [ "$?" -ne "0" ]; then + stdbuf -oL echo "[ERROR] Failed to copy patchsnippet_start_atlas" | vboxmonitor + sleep 90 + sudo shutdown now + fi + + part1="$(sed '/Check for user-specific proxy/q' /home/atlas/RunAtlas/start_atlas.sh)" + part2="$(sed -n '/Set CERN-internal proxy for CERN hosts/,$ p' /home/atlas/RunAtlas/start_atlas.sh)" + + echo "$part1" > /home/atlas/RunAtlas/start_atlas.sh + cat /home/atlas/patchsnippet_start_atlas >> /home/atlas/RunAtlas/start_atlas.sh + echo "$part2" >> /home/atlas/RunAtlas/start_atlas.sh + ## cp the modification to the shared folder to allow monitoring from the host + sudo sh -c "cp /home/atlas/RunAtlas/start_atlas.sh /home/atlas/shared/start_atlas.mod.txt" +} + + if [ -f /home/atlas/RunAtlas/start_atlas.sh ];then + patch_start_atlas cd /home/atlas/RunAtlas/ pandaid=$(tar xzf input.tar.gz -O | grep --binary-file=text -m1 -o PandaID=..........) taskid=$(tar xzf input.tar.gz -O | grep --binary-file=text -m1 -o taskID=........) diff --git a/vbox/create_local_wpad-dev b/vbox/create_local_wpad-dev new file mode 100644 index 0000000..ca0e60c --- /dev/null +++ b/vbox/create_local_wpad-dev @@ -0,0 +1,267 @@ +#!/bin/bash + +# Using a wpad.dat to define the proxy settings allows proxy load balancing and automatic proxy fail-over. +# If none of the sources provides a complex wpad.dat a simple generic one is written to be used by CVMFS and Frontier. +# The wpad.dat is then copied to the VM's webserver directory. +# This avoids multiple wpad requests to external sources. + + +function bs_get_wpad_from_lhchomeproxy { + wlcg_wpad_services1=("lhchomeproxy.cern.ch" "lhchomeproxy.fnal.gov") + wlcg_wpad_services2=("lhchomeproxy.fnal.gov" "lhchomeproxy.cern.ch") + + case "$(sed -n '//,/<\/host_info>/ s/^.*.*[.]\([^.]\+[.][^.]\+\)<\/domain_name>/\1/p' /home/atlas/shared/init_data.xml)" in + cern.ch) + wlcg_wpad_services=("${wlcg_wpad_services1[@]}") + ;; + fnal.gov) + wlcg_wpad_services=("${wlcg_wpad_services2[@]}") + ;; + *) + if (( $(( BASHPID % 2 )) != 0 )); then + wlcg_wpad_services=("${wlcg_wpad_services1[@]}") + else + wlcg_wpad_services=("${wlcg_wpad_services2[@]}") + fi + ;; + esac + + for wpad_service in "${wlcg_wpad_services[@]}"; do + ipv4_address_list=($(uniq <<< "$(awk '{ print $1 }' <<< "$(getent ahostsv4 ${wpad_service} 2>/dev/null)")")) + + for ipv4_address in "${ipv4_address_list[@]}"; do + if curl -f -L -m 10 -s -H "Host: ${wpad_service}" "http://${ipv4_address}/wpad.dat" -o "$bs_lhchome_wpad_tmp" && \ + grep -Eim1 '^[[:blank:]]*function[[:blank:]]+FindProxyForURL' "$bs_lhchome_wpad_tmp" >/dev/null 2>&1; then + # test against the location string that is usually returned as part of the wpad.dat + if grep -Ei '//[[:blank:]]+no[[:blank:]]+(org|squid)[[:blank:]]+found[[:blank:]]+' <<< "$(head -n1 "$bs_lhchome_wpad_tmp")" >/dev/null 2>&1; then + # non-WLCG sites will get DIRECT for openhtc.io and lhchomeproxy or 'NONE' for others + # keep that for fallback + echo "1" >"$bs_lhchome_return" + return 1 + else + # WLCG sites will get proxies from here + # this can even be DIRECT + echo "0" >"$bs_lhchome_return" + return 0 + fi + fi + done + + done + + echo "2" >"$bs_lhchome_return" + return 2 +} + + +function bs_get_wpad_from_grid_wpad { + ipv4_address_list=($(uniq <<< "$(awk '{ print $1 }' <<< "$(getent ahostsv4 grid-wpad 2>/dev/null)")")) + + for ipv4_address in "${ipv4_address_list[@]}"; do + if curl -f -L -m 10 -s -H "Host: grid-wpad" "http://${ipv4_address}/wpad.dat" -o "$bs_grid_wpad_tmp" && \ + grep -Eim1 '^[[:blank:]]*function[[:blank:]]+FindProxyForURL' "$bs_grid_wpad_tmp" >/dev/null 2>&1; then + echo "0" >"$bs_grid_return" + return 0 + fi + done + + echo "1" >"$bs_grid_return" + return 1 +} + + +function bs_get_proxy_from_boinc { + init_data_proxy="$(sed -n -e '//,/<\/proxy_info>/ p; /<\/proxy_info>/ q' /home/atlas/shared/init_data.xml)" + + # check, if BOINC has it's proxy enabled + if grep -m1 '' <<< "${init_data_proxy}" >/dev/null 2>&1; then + proxy_host="$(sed -ne '// {s/^.*\([^<]\+\).*/\1/p;q}' <<< "${init_data_proxy}")" + proxy_host="${proxy_host#"http://"}" # remove the protocol prefix if it exists. + proxy_port="$(sed -ne '// {s/^.*\([^<]\+\).*/\1/p;q}' <<< "${init_data_proxy}")" + + if [[ -n "$proxy_host" ]] && [[ -n "$proxy_port" ]]; then + +# Do not indent the heredoc EOF! +cat << EOF_BS_LOCAL_WPAD_FILE_with_proxy >"$bs_boinc_wpad_tmp" +function FindProxyForURL(url, host) { + return "PROXY http://${proxy_host}:${proxy_port}"; +} +EOF_BS_LOCAL_WPAD_FILE_with_proxy + + return 0 + fi + fi + + return 2 +} + + +function bs_set_simple_proxy { + # At this point we have a valid wpad.dat in + # $bs_local_wpad_dat that can be used. + # Simple commands like wget or curl don't understand + # wpad and need a single proxy set via the environment. + + if [[ "$http_proxy" != "" ]]; then + export http_proxy="$http_proxy" + + # all done + return 0 + fi + + unset proxy_new + + if [[ -x /usr/bin/pactester ]]; then + # pactester never makes an internet request + # http://example.com is used to get the fallback proxy from a wpad.dat + # the output looks like this + # PROXY http://ca-proxy-atlas.cern.ch:3128; PROXY http://cvmfsbproxy.cern.ch:3126; ... + # or like this + # PROXY http://ca-proxy.cern.ch:3128 + # or like this + # DIRECT + + my_pacstring="$(/usr/bin/pactester -p $bs_local_wpad_dat -u http://example.com 2>/dev/null)" + IFS='; ' read -r -a proxy_raw_arr <<< "$my_pacstring" + + for proxy_raw in "${proxy_raw_arr[@]}"; do + proxy_new=$proxy_raw + if [[ "$proxy_new" != "PROXY" ]] && \ + [[ "$proxy_new" != "DIRECT" ]] && \ + [[ "$proxy_new" != "NONE" ]] && \ + grep -iv 'lhchomeproxy\.' <<< "$proxy_new" >/dev/null 2>&1; then + # lhchomeproxy.{cern.ch|fnal.gov} must be excluded + export http_proxy="$proxy_new" + + # all done + return 0 + fi + done + fi + + # Can't identify a proxy + export http_proxy= + + return 0 +} + + +function bgcopy_pactester { + repo_base="/cvmfs/atlas.cern.ch/repo/sw/software/21.0/sw/lcg/releases/pacparser/1.3.5-a65a3/x86_64-centos7-gcc62-opt" + sudo sh -c "cp ${repo_base}/lib/libpacparser.so.1 /usr/lib64/" + sudo sh -c "cp ${repo_base}/bin/pactester /usr/bin/" + sudo sh -c "chown root:root /usr/lib64/libpacparser.so.1 /usr/bin/pactester" + sudo sh -c "chmod 755 /usr/lib64/libpacparser.so.1 /usr/bin/pactester" + sudo sh -c "ln -s libpacparser.so.1 /usr/lib64/libpacparser.so" +} + + +function bs_create_local_wpad { + bs_local_wpad_dat="/var/www/html/wpad.dat" + sudo sh -c "rm -f \"$bs_local_wpad_dat\"" + + my_umask="$(umask)" + umask 077 + my_tmp_dir="$(mktemp -d)" + umask "${my_umask}" + + # array to control the bg jobs + bg_pids_wpad=() + + # copy pactester utility in the bg + bgcopy_pactester & + bg_pids_wpad+=($!) + + # Contact lhchomeproxy to allow monitoring + # no matter which proxy is used at the end. + bs_lhchome_wpad_tmp="${my_tmp_dir}/lhchome_wpad" + bs_lhchome_return="${my_tmp_dir}/lhchome_return" + bs_get_wpad_from_lhchomeproxy & + bg_pids_wpad+=($!) + + # Try to get a wpad file from grid-wpad. + # This generic source needs to be set up by a site admin. + bs_grid_wpad_tmp="${my_tmp_dir}/grid_wpad" + bs_grid_return="${my_tmp_dir}/grid_return" + bs_get_wpad_from_grid_wpad & + bg_pids_wpad+=($!) + + # Try to get a proxy from the BOINC client. + bs_boinc_wpad_tmp="${my_tmp_dir}/boinc_wpad" + bs_get_proxy_from_boinc + ret_boinc_proxy="$?" + + for pid in ${bg_pids_wpad[*]}; do + wait $pid + done + + + ret_lhchomeproxy=$(cat "$bs_lhchome_return") + ret_grid_wpad=$(cat "$bs_grid_return") + + (( ret_lhchomeproxy == 2 )) && stdbuf -oL echo "[DEBUG] Could not download a wpad.dat from lhchomeproxy.{cern.ch|fnal.gov}" | vboxmonitor + + # prefer proxies set by grid-wpad + if (( ret_grid_wpad == 0 )); then + sudo sh -c "cat \"$bs_grid_wpad_tmp\" >\"$bs_local_wpad_dat\"" + stdbuf -oL echo "[INFO] Got a wpad.dat from grid-wpad" | vboxmonitor + stdbuf -oL echo "[INFO] Will use proxies from there for CVMFS and Frontier" | vboxmonitor + rm -frd "$my_tmp_dir" + + bs_set_simple_proxy + return 0 + fi + + if (( ret_lhchomeproxy == 0 )); then + sudo sh -c "cat \"$bs_lhchome_wpad_tmp\" >\"$bs_local_wpad_dat\"" + stdbuf -oL echo "[INFO] Got a wpad.dat from lhchomeproxy.{cern.ch|fnal.gov}" | vboxmonitor + stdbuf -oL echo "[INFO] Will use proxies from there for CVMFS and Frontier" | vboxmonitor + rm -frd "$my_tmp_dir" + + bs_set_simple_proxy + return 0 + fi + + # If no proxy is present yet, check if BOINC has one set. + if (( ret_boinc_proxy == 0 )); then + sudo sh -c "cat \"$bs_boinc_wpad_tmp\" >\"$bs_local_wpad_dat\"" + stdbuf -oL echo "[INFO] Got a proxy from the local BOINC client" | vboxmonitor + stdbuf -oL echo "[INFO] Will use it for CVMFS and Frontier" | vboxmonitor + rm -frd "$my_tmp_dir" + + bs_set_simple_proxy + return 0 + fi + + # Use non-WLCG reply from lhchomeproxy or generic DIRECT if other methods fail + stdbuf -oL echo "[INFO] Could not find a local HTTP proxy" | vboxmonitor + stdbuf -oL echo "[INFO] CVMFS and Frontier will have to use DIRECT connections" | vboxmonitor + stdbuf -oL echo "[INFO] This makes the application less efficient" | vboxmonitor + stdbuf -oL echo "[INFO] It also puts higher load on the project servers" | vboxmonitor + stdbuf -oL echo "[INFO] Setting up a local HTTP proxy is highly recommended" | vboxmonitor + stdbuf -oL echo "[INFO] Advice can be found in the project forum" | vboxmonitor + + if (( ret_lhchomeproxy == 1 )); then + sudo sh -c "cat \"$bs_lhchome_wpad_tmp\" >\"$bs_local_wpad_dat\"" + rm -frd "$my_tmp_dir" + + bs_set_simple_proxy + return 0 + fi + + rm -frd "$my_tmp_dir" + +# Do not indent the heredoc EOF! +cat << EOF_BS_LOCAL_WPAD_FILE_with_DIRECT >"/home/atlas/generic_wpad" +function FindProxyForURL(url, host) { + return "DIRECT"; +} +EOF_BS_LOCAL_WPAD_FILE_with_DIRECT + +sudo sh -c "cat \"/home/atlas/generic_wpad\" >\"$bs_local_wpad_dat\"" + + # Can't identify or connect to a proxy + # run it to clear the environment proxy + bs_set_simple_proxy + return 0 +} diff --git a/vbox/patchsnippet_start_atlas_full-dev b/vbox/patchsnippet_start_atlas_full-dev new file mode 100644 index 0000000..e7e78f9 --- /dev/null +++ b/vbox/patchsnippet_start_atlas_full-dev @@ -0,0 +1,38 @@ + if [ -e ${RUNTIME_CONFIG_DIR}init_data.xml ] && grep "" ${RUNTIME_CONFIG_DIR}init_data.xml >/dev/null 2>&1; then + ## so far this branch is experimental and shown as proof of concept + export ATLAS_LOCAL_AREA=${RUNTIME_CONFIG_DIR} + FRONTIER_PROXYCONFIG_URLS="(proxyconfigurl=http://localhost/wpad.dat)" + ## if a proxyconfigurl is set 'backupproxyurl' must be used instead of 'proxyurl' to ensure the right priority + ## see: http://frontier.cern.ch/dist/FrontierClientUsage.html + FRONTIER_BACKUPPROXY_URLS="" + if [ -n $http_proxy ]; then + FRONTIER_BACKUPPROXY_URLS="(backupproxyurl=${http_proxy})" + fi + + case "$(sed -n '//,/<\/host_info>/ s/^.*.*[.]\([^.]\+[.][^.]\+\)<\/domain_name>/\1/p' ${RUNTIME_CONFIG_DIR}init_data.xml)" in + cern.ch) + FRONTIER_SERVER_URLS="(serverurl=http://atlasfrontier-ai.cern.ch:8000/atlr)(serverurl=http://ccfrontier.in2p3.fr:23128/ccin2p3-AtlasFrontier)" + FRONTIER_BACKUPPROXY_URLS="$FRONTIER_BACKUPPROXY_URLS(backupproxyurl=http://ca-proxy-atlas.cern.ch:3128)(backupproxyurl=http://db-atlas-squid.ndgf.org:3128)(backupproxyurl=http://atlas-db-squid.grid.uio.no:3128)(backupproxyurl=http://dcache.ijs.si:3128)(backupproxyurl=http://atlasbpfrontier.cern.ch:3127)(backupproxyurl=http://atlasbpfrontier.fnal.gov:3127)" + ;; + *) + # main alias + # resolves to multiple IPs + FRONTIER_SERVER_URLS="(serverurl=http://atlascern-frontier.openhtc.io:8080/atlr)" + server_index_arr=(1 2 3 4) # openhtc.io fail-over list + if command -v shuf >/dev/null 2>&1; then + server_index_arr=($(shuf -e ${server_index_arr[*]})) + fi + # add fail-over servers, if possible in random order + for i in "${server_index_arr[@]}"; do + FRONTIER_SERVER_URLS="${FRONTIER_SERVER_URLS}(serverurl=http://atlascern${i}-frontier.openhtc.io:8080/atlr)" + done + + if [ "$(($$ % 2))" -eq 1 ]; then + FRONTIER_BACKUPPROXY_URLS="${FRONTIER_BACKUPPROXY_URLS}(backupproxyurl=http://atlasbpfrontier.cern.ch:3127)(backupproxyurl=http://atlasbpfrontier.fnal.gov:3127)" + else + FRONTIER_BACKUPPROXY_URLS="${FRONTIER_BACKUPPROXY_URLS}(backupproxyurl=http://atlasbpfrontier.fnal.gov:3127)(backupproxyurl=http://atlasbpfrontier.cern.ch:3127)" + fi + ;; + esac + + echo "export FRONTIER_SERVER=\"${FRONTIER_SERVER_URLS}${FRONTIER_PROXYCONFIG_URLS}${FRONTIER_BACKUPPROXY_URLS}\"" > ${ATLAS_LOCAL_AREA}setup.sh.local diff --git a/vbox/patchsnippet_start_atlas_lean-dev b/vbox/patchsnippet_start_atlas_lean-dev new file mode 100644 index 0000000..6c36772 --- /dev/null +++ b/vbox/patchsnippet_start_atlas_lean-dev @@ -0,0 +1,12 @@ + if [ -e ${RUNTIME_CONFIG_DIR}init_data.xml ] && grep "" ${RUNTIME_CONFIG_DIR}init_data.xml >/dev/null 2>&1; then + export ATLAS_LOCAL_AREA=${RUNTIME_CONFIG_DIR} + ## if a proxyconfigurl is set 'backupproxyurl' must be used instead of 'proxyurl' to ensure the right priority + ## see: http://frontier.cern.ch/dist/FrontierClientUsage.html + FRONTIER_SERVER_URLS="(serverurl=http://atlascern-frontier.openhtc.io:8080/atlr)(serverurl=http://atlasfrontier-ai.cern.ch:8000/atlr)(serverurl=http://ccfrontier.in2p3.fr:23128/ccin2p3-AtlasFrontier)" + FRONTIER_PROXYCONFIG_URLS="(proxyconfigurl=http://localhost/wpad.dat)" + FRONTIER_BACKUPPROXY_URLS="" + if [ -n $http_proxy ]; then + FRONTIER_BACKUPPROXY_URLS="(backupproxyurl=${http_proxy})" + fi + FRONTIER_BACKUPPROXY_URLS="${FRONTIER_BACKUPPROXY_URLS}(backupproxyurl=http://atlasbpfrontier.cern.ch:3127)(backupproxyurl=http://atlasbpfrontier.fnal.gov:3127)" + echo "export FRONTIER_SERVER=\"${FRONTIER_SERVER_URLS}${FRONTIER_PROXYCONFIG_URLS}${FRONTIER_BACKUPPROXY_URLS}\"" > ${ATLAS_LOCAL_AREA}setup.sh.local