[WIP] Fixes to Run1 branch

README.md

@@ -16,11 +16,11 @@ cd DailyPythonScripts
 # get submodules:
 git submodule init && git submodule update
 
-# setup run:
-./setup.sh
+# setup environment (using virtualenv for python):
+./setup_with_conda.sh
 
 # setup environment (using virtualenv for python):
-source environment.sh
+source environment_conda.sh
 
 # make sure matplotlib is up to date (should return 1.3.1 or above):
 python -c 'import matplotlib; print matplotlib.__version__'
@@ -101,13 +101,23 @@ number of jobs: enter the number of merging jobs for the centre of mass energy i
 - ```cd``` up to the folder containing DailyPythonScripts and ```tar --exclude='external/vpython' --exclude='any other large/unnecessary folders in DailyPythonScripts' -cf dps.tar DailyPythonScripts``` (tar file should be approximately 100MB in size)
 - Merge the required BAT output files (e.g. SingleTop, QCD etc.) using ```condor_submit DailyPythonScripts/experimental/mergeBATOutputFilesOnDICE/submitMerge.description```
 
+### Prepare config files for background subtraction
+```python src/cross_section_measurement/create_measurement.py -c 8```
+```python src/cross_section_measurement/create_measurement.py -c 7```
+This puts config files in config/measurements/background_subtraction/ for use by the x_0Nb_all_vars scripts.
+
 ### Run final measurement scripts in bin/:
 ```
 x_01_all_vars
+x_01b_all_vars
 x_02_all_vars
+x_02b_all_vars
 x_03_all_vars
+x_03b_all_vars
 x_04_all_vars
+x_04b_all_vars
 x_05_all_vars
+x_05b_all_vars
 x_98_all_vars
 x_99_QCD_cross_checks
 x_make_binning_plots

bin/create_unfolding_pulls_on_DICE

@@ -76,8 +76,8 @@ def main():
             )
             pull_jobs.append(pull_job)
 
-    # Since we are analysing 300 x 300 = 90000 samples and one 10x10 sample
-    # takes around 7 seconds it will take 1.75h in a single job.
+    # Since, in testing, one 10x10 sample took around 7 seconds
+    # and here we are analysing 300 x 300 = 90000 samples, it will take 1.75h in a single job.
     # For very fast (~1 min) processing we want at least 100 jobs.
     n_jobs_to_run = 100  # this is per pull_job
     n_jobs_to_split = 100

bin/plots2tdr

@@ -17,15 +17,15 @@ plot_map_common = {}
 plot_map_paper = {
             # <src relative to DPS plots folder> : <dst relative to TDR plots folder>,
             # results 8 TeV
-            '7TeV/MET/central/normalised_xsection_combined_*.pdf': '7TeV/MET/',
-            '7TeV/HT/central/normalised_xsection_combined_*.pdf': '7TeV/HT/',
-            '7TeV/ST/central/normalised_xsection_combined_*.pdf': '7TeV/ST/',
-            '7TeV/WPT/central/normalised_xsection_combined_*.pdf': '7TeV/WPT/',
+            'fit/7TeV/MET/central/normalised_xsection_combined_*.pdf': '7TeV/MET/',
+            'fit/7TeV/HT/central/normalised_xsection_combined_*.pdf': '7TeV/HT/',
+            'fit/7TeV/ST/central/normalised_xsection_combined_*.pdf': '7TeV/ST/',
+            'fit/7TeV/WPT/central/normalised_xsection_combined_*.pdf': '7TeV/WPT/',
             # results 8 TeV
-            '8TeV/MET/central/normalised_xsection_combined_*.pdf': '8TeV/MET/',
-            '8TeV/HT/central/normalised_xsection_combined_*.pdf': '8TeV/HT/',
-            '8TeV/ST/central/normalised_xsection_combined_*.pdf': '8TeV/ST/',
-            '8TeV/WPT/central/normalised_xsection_combined_*.pdf': '8TeV/WPT/',
+            'fit/8TeV/MET/central/normalised_xsection_combined_*.pdf': '8TeV/MET/',
+            'fit/8TeV/HT/central/normalised_xsection_combined_*.pdf': '8TeV/HT/',
+            'fit/8TeV/ST/central/normalised_xsection_combined_*.pdf': '8TeV/ST/',
+            'fit/8TeV/WPT/central/normalised_xsection_combined_*.pdf': '8TeV/WPT/',
             # control plots 8 TeV
             'control_plots/after_fit/8TeV/*_HT_2orMoreBtags_with_ratio.pdf': 'control/',
             'control_plots/after_fit/8TeV/*_patType1CorrectedPFMet_2orMoreBtags_with_ratio.pdf': 'control/',

bin/x_01_all_vars

@@ -2,6 +2,9 @@
 echo "This will take a while ... grab a coffee/tea/water/cocktail"
 mkdir -p logs
 fit_var="absolute_eta,M3,angle_bl"
+# fit_var="absolute_eta"
+# fit_var="M3"
+#fit_var="angle_bl"
 nice_fit_var=`echo $fit_var | sed 's/,/_/g'`
 N_JOBS=6
 

bin/x_02_all_vars

@@ -2,6 +2,9 @@
 echo "This will take a while ... grab a coffee/tea/water"
 mkdir -p logs
 fit_var="absolute_eta,M3,angle_bl"
+# fit_var="absolute_eta"
+# fit_var="M3"
+# fit_var="angle_bl"
 nice_fit_var=`echo $fit_var | sed 's/,/_/g'`
 N_JOBS=6
 

bin/x_03_all_vars

@@ -2,6 +2,9 @@
 echo "This will take a while ... grab a coffee/tea/water"
 mkdir -p logs
 fit_var="absolute_eta,M3,angle_bl"
+# fit_var="absolute_eta"
+# fit_var="M3"
+# fit_var="angle_bl"
 nice_fit_var=`echo $fit_var | sed 's/,/_/g'`
 N_JOBS=6
 

bin/x_04_all_vars

@@ -3,24 +3,29 @@ echo "This will take a while ... grab a coffee/tea/water"
 mkdir -p logs
 mkdir -p plots
 fit_var="absolute_eta,M3,angle_bl"
+# fit_var="absolute_eta"
+# fit_var="M3"
+# fit_var="angle_bl"
 nice_fit_var=`echo $fit_var | sed 's/,/_/g'`
-N_JOBS=5
+N_JOBS=6
 
 echo "Using the fit variable(s): $fit_var"
 
 i=0
-for var in MET HT ST MT WPT; do
+for var in MET MT; do
 	echo "Plotting diff. x-section for distribution: $var"
-	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 7 -p data/$nice_fit_var -o plots/fit &> logs/04_${var}_plot_7TeV_${nice_fit_var}.log & # -a -s <--add -a option for additional plots, -s option to include the final results from background subtraction as well as from fitting
-	let i+=1
+	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 7 -p data/$nice_fit_var -o plots/results/fit/ &> logs/04_${var}_plot_7TeV_${nice_fit_var}.log & # -a -s <--add -a option for additional plots, -s option to include the final results from background subtraction as well as from fitting
+	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 7 -p data/$nice_fit_var -o plots/results/fit/ -s &> logs/04_${var}_plot_7TeV_${nice_fit_var}_with_bkgd_sub.log & # -a -s <--add -a option for additional plots, -s option to include the final results from background subtraction as well as from fitting
+	let i+=2
 	if (( $i % N_JOBS == 0 ))
 	then
 		echo "Waiting on the above to finish."
     	wait;
 	fi
 
-	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 8 -p data/$nice_fit_var -o plots/fit &> logs/04_${var}_plot_8TeV_${nice_fit_var}.log & # -a -s <--add -a option for additional plots, -s option to include the final results from background subtraction as well as from fitting
-	let i+=1
+	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 8 -p data/$nice_fit_var -o plots/results/fit/ &> logs/04_${var}_plot_8TeV_${nice_fit_var}.log & # -a -s <--add -a option for additional plots, -s option to include the final results from background subtraction as well as from fitting
+	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 8 -p data/$nice_fit_var -o plots/results/fit/ -s &> logs/04_${var}_plot_8TeV_${nice_fit_var}_with_bkgd_sub.log & # -a -s <--add -a option for additional plots, -s option to include the final results from background subtraction as well as from fitting
+	let i+=2
 	if (( $i % N_JOBS == 0 ))
 	then
 		echo "Waiting on the above to finish."

bin/x_04b_all_vars

@@ -8,15 +8,15 @@ i=0
 echo "Visible phase space"
 for var in MET HT ST WPT MT; do
 	echo "Plotting diff. x-section for distribution: $var"
-	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 7 -p data/normalisation/background_subtraction/ -o plots/background_subtraction &> logs/04b_${var}_plot_7TeV.log & # -a  <--add this option for additional plots
+	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 7 -p data/normalisation/background_subtraction/ -o plots/results/background_subtraction &> logs/04b_${var}_plot_7TeV.log & # -a  <--add this option for additional plots
 	let i+=1
 	if (( $i % N_JOBS == 0 ))
 	then
 		echo "Waiting on the above to finish."
     	wait;
 	fi
 
-	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 8 -p data/normalisation/background_subtraction/ -o plots/background_subtraction &> logs/04b_${var}_plot_8TeV.log & # -a  <--add this option for additional plots
+	nohup time python src/cross_section_measurement/04_make_plots_matplotlib.py -v $var -c 8 -p data/normalisation/background_subtraction/ -o plots/results/background_subtraction &> logs/04b_${var}_plot_8TeV.log & # -a  <--add this option for additional plots
 	let i+=1
 	if (( $i % N_JOBS == 0 ))
 	then

condor/run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 cmssw_version=CMSSW_7_4_7
-git_branch=master
+git_branch=run1
 echo "Setting up ${cmssw_version} ..."
 echo "... sourcing CMS default environment from CVMFS"
 source /cvmfs/cms.cern.ch/cmsset_default.sh
@@ -22,7 +22,7 @@ time git submodule init && git submodule update
 echo "... extracting ${_CONDOR_JOB_IWD}/dps.tar on top"
 tar -xf ${_CONDOR_JOB_IWD}/dps.tar --overwrite
 echo "... running setup routine"
->&2 "... running setup routine"
+>&2 echo "... running setup routine"
 time source setup_with_conda.sh
 echo "... enforcing conda python environment"
 # this is safe, as the dangerous part is only executed on soolin

config/CMS.py

@@ -22,22 +22,22 @@
               }
 
 y_axis_title_small = {
-              'fontsize':40,
+              'fontsize':42,
               'position' : (0, 1.),
               'verticalalignment': 'bottom',
               'horizontalalignment': 'right'
               }
 
 axis_label_major = {
               'which':'major',
-              'labelsize':41,
+              'labelsize':42,
               'length':20,
               'width': 1,
               'pad': 12
               }
 axis_label_minor = {
               'which':'minor',
-              'labelsize':41,
+              'labelsize':42,
               'length':10,
               'width': 1,
               'pad': 12

config/RooUnfold_template.py

@@ -15,7 +15,7 @@
 SVD_k_value = 5
 SVD_tau_value = -1
 SVD_n_toy = 1000
-Hreco = 2
+error_treatment = 3
 Bayes_n_repeat = 4
 
 unfolded_markerStyle = 20

config/__init__.py

@@ -42,8 +42,9 @@ class XSectionConfig():
                   'unfolding_matching_up', 'unfolding_matching_up_raw',
                   'unfolding_mass_down', 'unfolding_mass_up',
                   'unfolding_mcatnlo', 'unfolding_mcatnlo_raw',
-                  'unfolding_powheg_pythia', 'unfolding_powheg_pythia_raw',
-                  'unfolding_powheg_herwig', 'unfolding_powheg_herwig_raw',
+                  'unfolding_powheg_v2_pythia', 'unfolding_powheg_v2_pythia_raw',
+                  'unfolding_powheg_v1_herwig', 'unfolding_powheg_v1_herwig_raw',
+                  'unfolding_powheg_v2_herwig', 'unfolding_powheg_v2_herwig_raw',
                   'unfolding_scale_down', 'unfolding_scale_down_raw',
                   'unfolding_scale_up', 'unfolding_scale_up_raw',
                   'unfolding_ptreweight', 'unfolding_ptreweight_raw',
@@ -217,8 +218,10 @@ def __fill_defaults__( self ):
                                     }
 
         self.unfolding_madgraph_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV.root' % self.centre_of_mass_energy
-        self.unfolding_powheg_pythia_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_powheg.root' % self.centre_of_mass_energy
-        self.unfolding_powheg_herwig_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_powhegherwig.root' % self.centre_of_mass_energy
+        self.unfolding_powheg_v1_pythia_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_powhegV1pythia.root' % self.centre_of_mass_energy
+        self.unfolding_powheg_v2_pythia_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_powhegV2pythia.root' % self.centre_of_mass_energy
+        self.unfolding_powheg_v1_herwig_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_powhegV1herwig.root' % self.centre_of_mass_energy
+        self.unfolding_powheg_v2_herwig_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_powhegV2herwig.root' % self.centre_of_mass_energy
         self.unfolding_mcatnlo_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_mcatnlo.root' % self.centre_of_mass_energy
         self.unfolding_ptreweight_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_withTopPtReweighting.root' % self.centre_of_mass_energy
 
@@ -228,8 +231,10 @@ def __fill_defaults__( self ):
         self.unfolding_matching_up_raw = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_matchingup.root' % self.centre_of_mass_energy
 
         self.unfolding_madgraph = self.unfolding_madgraph_raw.replace( '.root', '_asymmetric.root' )
-        self.unfolding_powheg_pythia = self.unfolding_powheg_pythia_raw.replace( '.root', '_asymmetric.root' )
-        self.unfolding_powheg_herwig = self.unfolding_powheg_herwig_raw.replace( '.root', '_asymmetric.root' )
+        self.unfolding_powheg_v1_pythia = self.unfolding_powheg_v1_pythia_raw.replace( '.root', '_asymmetric.root' )
+        self.unfolding_powheg_v2_pythia = self.unfolding_powheg_v2_pythia_raw.replace( '.root', '_asymmetric.root' )
+        self.unfolding_powheg_v1_herwig = self.unfolding_powheg_v1_herwig_raw.replace( '.root', '_asymmetric.root' )
+        self.unfolding_powheg_v2_herwig = self.unfolding_powheg_v2_herwig_raw.replace( '.root', '_asymmetric.root' )
         self.unfolding_mcatnlo = self.unfolding_mcatnlo_raw.replace( '.root', '_asymmetric.root' )
         self.unfolding_ptreweight = path_to_unfolding_histograms + 'unfolding_TTJets_%dTeV_asymmetric_withTopPtReweighting.root' % self.centre_of_mass_energy
 
@@ -358,6 +363,12 @@ def __fill_defaults_7TeV__( self ):
                    'WPT' : 0
                    }
 
+        self.tau_values_electron = {
+        }
+
+        self.tau_values_muon = {
+        }
+
         self.categories_and_prefixes['PU_down'] = '_PU_64600mb'
         self.categories_and_prefixes['PU_up'] = '_PU_71400mb'
 
@@ -409,6 +420,12 @@ def __fill_defaults_8TeV__( self ):
                    'WPT' : 0
                    }
 
+        self.tau_values_electron = {
+        }
+
+        self.tau_values_muon = {
+        }
+
         self.categories_and_prefixes['PU_down'] = '_PU_65835mb'
         self.categories_and_prefixes['PU_up'] = '_PU_72765mb'
 
@@ -419,11 +436,15 @@ def __fill_defaults_8TeV__( self ):
                                     'JES_down': path_to_files + 'JES_down/SingleElectron' + middle + self.categories_and_prefixes['JES_down'] + '.root'
                                     }
 
-fit_var_inputs = ['absolute_eta', 'M3', 'M_bl', 'angle_bl',
-                      'absolute_eta_angle_bl',
-                      'absolute_eta_M3',
-                      'absolute_eta_M_bl',
-                      'absolute_eta_M_bl_angle_bl',
+fit_var_inputs = [
+                  'absolute_eta', 'M3', 
+# 'M_bl', 
+                  'angle_bl',
+                      # 'absolute_eta_angle_bl',
+                      # 'absolute_eta_M3',
+                      # 'absolute_eta_M_bl',
+                      # 'absolute_eta_M_bl_angle_bl',
                       'absolute_eta_M3_angle_bl',
-                      'absolute_eta_M_bl_M3',
-                      'absolute_eta_M_bl_M3_angle_bl' ]
+                      # 'absolute_eta_M_bl_M3',
+                      # 'absolute_eta_M_bl_M3_angle_bl'
+                       ]