From 77acb55ce92d03f3d5332ad36ba5726210693aba Mon Sep 17 00:00:00 2001 From: Andrei Cuceu Date: Mon, 15 Sep 2025 11:31:18 -0700 Subject: [PATCH 1/7] Remove old blinding from correlation function --- .gitattributes | 7 ++++++ py/picca/bin/picca_cf.py | 3 +-- py/picca/bin/picca_dmat.py | 3 +-- py/picca/bin/picca_fast_metal_dmat.py | 3 +-- py/picca/bin/picca_fast_metal_xdmat.py | 3 +-- py/picca/bin/picca_metal_dmat.py | 3 +-- py/picca/bin/picca_metal_xdmat.py | 3 +-- py/picca/bin/picca_wick.py | 3 +-- py/picca/bin/picca_xcf.py | 3 +-- py/picca/bin/picca_xcf_angl.py | 3 +-- py/picca/bin/picca_xdmat.py | 3 +-- py/picca/bin/picca_xwick.py | 3 +-- py/picca/constants.py | 35 +++----------------------- 13 files changed, 22 insertions(+), 53 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..1f05e6f9f --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +*.py text=auto +*.cfg text=auto +*.ini text=auto +*.yml text=auto +*.md text=auto +*.toml text=auto +pylintrc text=auto diff --git a/py/picca/bin/picca_cf.py b/py/picca/bin/picca_cf.py index 3966dd227..3a0f09733 100644 --- a/py/picca/bin/picca_cf.py +++ b/py/picca/bin/picca_cf.py @@ -290,8 +290,7 @@ def main(cmdargs=None): cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) t0 = time.time() diff --git a/py/picca/bin/picca_dmat.py b/py/picca/bin/picca_dmat.py index 6c3b296a0..99e6fdbbc 100644 --- a/py/picca/bin/picca_dmat.py +++ b/py/picca/bin/picca_dmat.py @@ -326,8 +326,7 @@ def main(cmdargs=None): cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) t0 = time.time() diff --git a/py/picca/bin/picca_fast_metal_dmat.py b/py/picca/bin/picca_fast_metal_dmat.py index f428656de..01a2fbd2f 100644 --- a/py/picca/bin/picca_fast_metal_dmat.py +++ b/py/picca/bin/picca_fast_metal_dmat.py @@ -472,8 +472,7 @@ def main(cmdargs=None): cf.cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) t0 = time.time() diff --git a/py/picca/bin/picca_fast_metal_xdmat.py b/py/picca/bin/picca_fast_metal_xdmat.py index d5a0f2e68..d8c812be0 100644 --- a/py/picca/bin/picca_fast_metal_xdmat.py +++ b/py/picca/bin/picca_fast_metal_xdmat.py @@ -444,8 +444,7 @@ def main(cmdargs=None): cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) xcf.cosmo = cosmo t0 = time.time() diff --git a/py/picca/bin/picca_metal_dmat.py b/py/picca/bin/picca_metal_dmat.py index 09693d0bf..874c08436 100644 --- a/py/picca/bin/picca_metal_dmat.py +++ b/py/picca/bin/picca_metal_dmat.py @@ -322,8 +322,7 @@ def main(cmdargs=None): cf.cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) t0 = time.time() diff --git a/py/picca/bin/picca_metal_xdmat.py b/py/picca/bin/picca_metal_xdmat.py index 0f6b2aa4f..b63885e06 100644 --- a/py/picca/bin/picca_metal_xdmat.py +++ b/py/picca/bin/picca_metal_xdmat.py @@ -300,8 +300,7 @@ def main(cmdargs=None): cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) xcf.cosmo = cosmo t0 = time.time() diff --git a/py/picca/bin/picca_wick.py b/py/picca/bin/picca_wick.py index 2fead65bf..1d2113d4d 100644 --- a/py/picca/bin/picca_wick.py +++ b/py/picca/bin/picca_wick.py @@ -311,8 +311,7 @@ def main(cmdargs): cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) # read data 1 data, num_data, z_min, z_max = io.read_deltas(args.in_dir, diff --git a/py/picca/bin/picca_xcf.py b/py/picca/bin/picca_xcf.py index 1239926be..5b0913f30 100644 --- a/py/picca/bin/picca_xcf.py +++ b/py/picca/bin/picca_xcf.py @@ -307,8 +307,7 @@ def main(cmdargs=None): cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) t0 = time.time() diff --git a/py/picca/bin/picca_xcf_angl.py b/py/picca/bin/picca_xcf_angl.py index e3fa77c08..ebb5ccf02 100644 --- a/py/picca/bin/picca_xcf_angl.py +++ b/py/picca/bin/picca_xcf_angl.py @@ -259,8 +259,7 @@ def main(cmdargs): cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) ### Read deltas data, num_data, z_min, z_max = io.read_deltas( diff --git a/py/picca/bin/picca_xdmat.py b/py/picca/bin/picca_xdmat.py index 3d424f4fc..a3b1e6e2f 100644 --- a/py/picca/bin/picca_xdmat.py +++ b/py/picca/bin/picca_xdmat.py @@ -319,8 +319,7 @@ def main(cmdargs=None): cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) t0 = time.time() diff --git a/py/picca/bin/picca_xwick.py b/py/picca/bin/picca_xwick.py index 0ed4bded1..877d5f97c 100644 --- a/py/picca/bin/picca_xwick.py +++ b/py/picca/bin/picca_xwick.py @@ -285,8 +285,7 @@ def main(cmdargs): cosmo = constants.Cosmo(Om=args.fid_Om, Or=args.fid_Or, Ok=args.fid_Ok, - wl=args.fid_wl, - blinding=blinding) + wl=args.fid_wl,) ### Read deltas data, num_data, z_min, z_max = io.read_deltas(args.in_dir, diff --git a/py/picca/constants.py b/py/picca/constants.py index cdbe5ef0d..3e85df5a6 100644 --- a/py/picca/constants.py +++ b/py/picca/constants.py @@ -7,7 +7,6 @@ import numpy as np from scipy import interpolate from scipy.constants import speed_of_light as speed_light -from pkg_resources import resource_filename from picca.utils import userprint # TODO: this constant is unused. They should be removed at some point @@ -17,10 +16,6 @@ SPEED_LIGHT = speed_light/1000. # [km/s] -# different strategies are explained in -# https://desi.lbl.gov/trac/wiki/keyprojects/y1kp6/Blinding -ACCEPTED_BLINDING_STRATEGIES = ["none", "minimal", "strategyB", "strategyC", - "strategyBC", "corr_yshift"] class Cosmo(object): """This class defines the fiducial cosmology @@ -160,7 +155,7 @@ def get_dist_v(self, z): """ raise NotImplementedError("Function should be specified at run-time") - def __init__(self,Om,Ok=0.,Or=0.,wl=-1.,blinding=False,verbose=True): + def __init__(self,Om,Ok=0.,Or=0.,wl=-1.,verbose=True): """Initializes the methods for this instance Args: @@ -182,31 +177,9 @@ def __init__(self,Om,Ok=0.,Or=0.,wl=-1.,blinding=False,verbose=True): # make what we are doing more clear. H0 = 100.0 - # Blind data - if blinding == "none": - if verbose: - userprint("ATTENTION: Analysis is not blinded!") - else: - userprint(f"ATTENTION: Analysis is blinded with strategy {blinding}") - - if blinding not in ["strategyB", "strategyBC"]: - if verbose: - userprint(f"Om={Om}, Or={Or}, wl={wl}") - else: - userprint("The specified cosmology is " - f"not used: Om={Om}, Or={Or}, wl={wl}") - # blind test small - filename = "DR16_blind_test_small/DR16_blind_test_small.fits" - # blind test large - #filename = "DR16_blind_test_small/DR16_blind_test_large.fits" - # load Om - filename = resource_filename('picca', 'fitter2')+'/models/{}'.format(filename) - hdu = fitsio.FITS(filename) - Om = hdu[1].read_header()['OM'] - Or = hdu[1].read_header()['OR'] - wl = hdu[1].read_header()['W'] - H0 = hdu[1].read_header()['H0'] - hdu.close() + if verbose: + userprint("ATTENTION: Analysis is not blinded!") + userprint(f"Om={Om}, Or={Or}, wl={wl}") # Ignore evolution of neutrinos from matter to radiation Ol = 1. - Ok - Om - Or From 21929ed860da59aaa09c5700639853778d2f1e4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignasi=20P=C3=A9rez-R=C3=A0fols?= Date: Tue, 23 Sep 2025 17:16:04 +0200 Subject: [PATCH 2/7] removed pkc_resources from true continuum --- .../expected_fluxes/true_continuum.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/py/picca/delta_extraction/expected_fluxes/true_continuum.py b/py/picca/delta_extraction/expected_fluxes/true_continuum.py index aa96fc369..c1cb6a403 100644 --- a/py/picca/delta_extraction/expected_fluxes/true_continuum.py +++ b/py/picca/delta_extraction/expected_fluxes/true_continuum.py @@ -1,4 +1,5 @@ """This module defines the class TrueContinuum""" +from importlib import resources import logging import multiprocessing @@ -6,7 +7,7 @@ import numpy as np from scipy.interpolate import interp1d import healpy -from pkg_resources import resource_filename +from importlib import resources from picca.delta_extraction.astronomical_objects.forest import Forest from picca.delta_extraction.astronomical_objects.pk1d_forest import Pk1dForest @@ -391,32 +392,34 @@ def read_raw_statistics(self): if self.raw_statistics_filename != "": filename = self.raw_statistics_filename else: - filename = resource_filename( - 'picca', 'delta_extraction') + '/expected_fluxes/raw_stats/' + file_path_aux = "" if Forest.wave_solution == "log": - filename += 'colore_v9_lya_log.fits.gz' + file_path_aux += 'colore_v9_lya_log.fits.gz' elif Forest.wave_solution == "lin" and np.isclose( 10**Forest.log_lambda_grid[1] - 10**Forest.log_lambda_grid[0], 0.8, rtol=0.1): - filename += 'colore_v9_lya_lin_0.8.fits.gz' + file_path_aux += 'colore_v9_lya_lin_0.8.fits.gz' elif Forest.wave_solution == "lin" and np.isclose( 10**Forest.log_lambda_grid[1] - 10**Forest.log_lambda_grid[0], 2.4, rtol=0.1): - filename += 'colore_v9_lya_lin_2.4.fits.gz' + file_path_aux += 'colore_v9_lya_lin_2.4.fits.gz' elif Forest.wave_solution == "lin" and np.isclose( 10**Forest.log_lambda_grid[1] - 10**Forest.log_lambda_grid[0], 3.2, rtol=0.1): - filename += 'colore_v9_lya_lin_3.2.fits.gz' + file_path_aux += 'colore_v9_lya_lin_3.2.fits.gz' else: raise ExpectedFluxError( "Couldn't find compatible raw satistics file. Provide a " "custom one using 'raw statistics file' field.") + with resources.path("picca.delta_extraction.expected_fluxes.raw_stats", + file_path_aux) as file_path: + filename = str(file_path) self.logger.info( f'Reading raw statistics var_lss and mean_flux from file: {filename}' ) From 06f604710e851327b6ffe519c5abab865dfa5465 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignasi=20P=C3=A9rez-R=C3=A0fols?= Date: Tue, 23 Sep 2025 17:18:42 +0200 Subject: [PATCH 3/7] removed pkg_resources from test_helpers --- py/picca/tests/test_helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py/picca/tests/test_helpers.py b/py/picca/tests/test_helpers.py index 59c38ad12..8a213cda5 100644 --- a/py/picca/tests/test_helpers.py +++ b/py/picca/tests/test_helpers.py @@ -1,4 +1,5 @@ import configparser as ConfigParser +from importlib import resources from picca.utils import userprint import fitsio import os @@ -7,7 +8,7 @@ import sys import unittest import shutil, tempfile -from pkg_resources import resource_filename +from importlib import resources ### Make ConfigParser case sensitive class CaseConfigParser(ConfigParser.ConfigParser): @@ -253,8 +254,8 @@ def setUpClass(cls): """ cls._branchFiles = tempfile.mkdtemp() + "/" cls.produce_folder(cls) - cls.picca_base = resource_filename('picca', - './').replace('py/picca/./', '') + with resources.path('picca', '.') as picca_path: + cls.picca_base = str(picca_path).replace('py/picca/./', '') cls.send_requirements(cls.load_requirements(cls.picca_base)) np.random.seed(42) cls._masterFiles = cls.picca_base + '/py/picca/tests/data/' From a87395978daaf11c6c78a9af460ee5144a54e5ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignasi=20P=C3=A9rez-R=C3=A0fols?= Date: Tue, 23 Sep 2025 17:21:45 +0200 Subject: [PATCH 4/7] removed repeated import --- py/picca/delta_extraction/expected_fluxes/true_continuum.py | 1 - 1 file changed, 1 deletion(-) diff --git a/py/picca/delta_extraction/expected_fluxes/true_continuum.py b/py/picca/delta_extraction/expected_fluxes/true_continuum.py index c1cb6a403..ccd346830 100644 --- a/py/picca/delta_extraction/expected_fluxes/true_continuum.py +++ b/py/picca/delta_extraction/expected_fluxes/true_continuum.py @@ -7,7 +7,6 @@ import numpy as np from scipy.interpolate import interp1d import healpy -from importlib import resources from picca.delta_extraction.astronomical_objects.forest import Forest from picca.delta_extraction.astronomical_objects.pk1d_forest import Pk1dForest From 282cae287faca100a774715472eb4d3ab02c589c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignasi=20P=C3=A9rez-R=C3=A0fols?= Date: Tue, 23 Sep 2025 17:24:16 +0200 Subject: [PATCH 5/7] removed repeated import --- py/picca/tests/test_helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/py/picca/tests/test_helpers.py b/py/picca/tests/test_helpers.py index 8a213cda5..247ca8298 100644 --- a/py/picca/tests/test_helpers.py +++ b/py/picca/tests/test_helpers.py @@ -8,7 +8,6 @@ import sys import unittest import shutil, tempfile -from importlib import resources ### Make ConfigParser case sensitive class CaseConfigParser(ConfigParser.ConfigParser): From 5bfad2105852d306afa2300de7ccc732bbc52c06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignasi=20P=C3=A9rez-R=C3=A0fols?= Date: Tue, 23 Sep 2025 17:43:07 +0200 Subject: [PATCH 6/7] fixed issues introduced when merging master --- py/picca/constants.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/py/picca/constants.py b/py/picca/constants.py index 3e85df5a6..519e73934 100644 --- a/py/picca/constants.py +++ b/py/picca/constants.py @@ -155,7 +155,7 @@ def get_dist_v(self, z): """ raise NotImplementedError("Function should be specified at run-time") - def __init__(self,Om,Ok=0.,Or=0.,wl=-1.,verbose=True): + def __init__(self,Om,Ok=0.,Or=0.,wl=-1.,blinding=False,verbose=True): """Initializes the methods for this instance Args: @@ -177,7 +177,7 @@ def __init__(self,Om,Ok=0.,Or=0.,wl=-1.,verbose=True): # make what we are doing more clear. H0 = 100.0 - if verbose: + if verbose and blinding == "none": userprint("ATTENTION: Analysis is not blinded!") userprint(f"Om={Om}, Or={Or}, wl={wl}") From 5ecc180844ecb367d9f836aeff1a03fe6ec3f137 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ignasi=20P=C3=A9rez-R=C3=A0fols?= Date: Tue, 23 Sep 2025 17:45:29 +0200 Subject: [PATCH 7/7] removed importlib resources --- py/picca/tests/test_helpers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/py/picca/tests/test_helpers.py b/py/picca/tests/test_helpers.py index 247ca8298..4409ac8a6 100644 --- a/py/picca/tests/test_helpers.py +++ b/py/picca/tests/test_helpers.py @@ -1,5 +1,4 @@ import configparser as ConfigParser -from importlib import resources from picca.utils import userprint import fitsio import os @@ -9,6 +8,9 @@ import unittest import shutil, tempfile +THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +PICCA_BASE = THIS_DIR.replace("py/picca/tests", "") + ### Make ConfigParser case sensitive class CaseConfigParser(ConfigParser.ConfigParser): @@ -253,8 +255,7 @@ def setUpClass(cls): """ cls._branchFiles = tempfile.mkdtemp() + "/" cls.produce_folder(cls) - with resources.path('picca', '.') as picca_path: - cls.picca_base = str(picca_path).replace('py/picca/./', '') + cls.picca_base = PICCA_BASE cls.send_requirements(cls.load_requirements(cls.picca_base)) np.random.seed(42) cls._masterFiles = cls.picca_base + '/py/picca/tests/data/'