From a3a4765f32582edf00508b0701e2797f34e26a19 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 30 Oct 2025 14:32:46 -0500 Subject: [PATCH 01/25] Adding check for dark before science to avoid bad characterization --- potpyri/primitives/sort_files.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/potpyri/primitives/sort_files.py b/potpyri/primitives/sort_files.py index b59b425..df99d16 100755 --- a/potpyri/primitives/sort_files.py +++ b/potpyri/primitives/sort_files.py @@ -282,14 +282,14 @@ def sort_files(files, file_list, tel, paths, incl_bad=False, log=None): file_type = 'BIAS' moved_path = paths['raw'] bias_num += 1 - elif is_science(hdr, tel): - file_type = 'SCIENCE' - moved_path = paths['raw'] - sci_num += 1 elif is_dark(hdr, tel): file_type = 'DARK' moved_path = paths['raw'] dark_num += 1 + elif is_science(hdr, tel): + file_type = 'SCIENCE' + moved_path = paths['raw'] + sci_num += 1 else: file_type = 'BAD' moved_path = paths['bad'] From 174ed742e9ff2cd8cdcb39bcb962209e5c8d42d7 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 30 Oct 2025 14:35:34 -0500 Subject: [PATCH 02/25] Updating MOSFIRE to require darks --- potpyri/instruments/MOSFIRE.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/potpyri/instruments/MOSFIRE.py b/potpyri/instruments/MOSFIRE.py index 6d0f8b1..88f7170 100755 --- a/potpyri/instruments/MOSFIRE.py +++ b/potpyri/instruments/MOSFIRE.py @@ -33,7 +33,7 @@ def __init__(self): self.min_exptime = 1.0 # Run dark/bias/flat calibration? - self.dark = False + self.dark = True self.bias = False self.flat = True From eea7c15ac63b10aa5091078af0e6a91e452a370a Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Fri, 7 Nov 2025 15:05:43 -0600 Subject: [PATCH 03/25] Fixing parameter for generating final catalog with aperture photometry --- potpyri/primitives/photometry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/potpyri/primitives/photometry.py b/potpyri/primitives/photometry.py index 23f91bd..2176d22 100755 --- a/potpyri/primitives/photometry.py +++ b/potpyri/primitives/photometry.py @@ -520,10 +520,10 @@ def photloop(stack, phot_sn_min=3.0, phot_sn_max=40.0, fwhm_init=5.0, log=None): epsf = None ; fwhm = None while signal_to_noise > phot_sn_min: - if log: log.info(f'Trying photometry with final S/N={signal_to_noise}') + if log: log.info(f'Trying PSF generation with S/N={signal_to_noise}') star_param = {'snthresh_psf': signal_to_noise*2.0, 'fwhm_init': fwhm_init, - 'snthresh_final': signal_to_noise} + 'snthresh_final': 5.0} try: do_phot(stack, star_param=star_param) except Exception as e: From 4fa35f9577ff7cff56142e85e230359efc423040 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Fri, 7 Nov 2025 15:25:58 -0600 Subject: [PATCH 04/25] Adjusting maximum sn threshold for photloop --- potpyri/primitives/photometry.py | 2 +- potpyri/utils/options.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/potpyri/primitives/photometry.py b/potpyri/primitives/photometry.py index 2176d22..8695391 100755 --- a/potpyri/primitives/photometry.py +++ b/potpyri/primitives/photometry.py @@ -523,7 +523,7 @@ def photloop(stack, phot_sn_min=3.0, phot_sn_max=40.0, fwhm_init=5.0, log=None): if log: log.info(f'Trying PSF generation with S/N={signal_to_noise}') star_param = {'snthresh_psf': signal_to_noise*2.0, 'fwhm_init': fwhm_init, - 'snthresh_final': 5.0} + 'snthresh_final': signal_to_noise} try: do_phot(stack, star_param=star_param) except Exception as e: diff --git a/potpyri/utils/options.py b/potpyri/utils/options.py index 9bf6624..f0ca658 100755 --- a/potpyri/utils/options.py +++ b/potpyri/utils/options.py @@ -53,7 +53,7 @@ def init_options(): help='''Minimum signal-to-noise to try in photometry loop.''') params.add_argument('--phot-sn-max', type=float, - default=40.0, + default=20.0, help='''Maximum signal-to-noise to try in photometry loop.''') params.add_argument('--fwhm-init', type=float, From 25e93c8388b468625df064a53bb03484c857bac4 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Mon, 10 Nov 2025 14:32:40 -0600 Subject: [PATCH 05/25] Fixing issues with catalog caching and catalog size from Vizier --- potpyri/primitives/absphot.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/potpyri/primitives/absphot.py b/potpyri/primitives/absphot.py index 6b32084..3ee9217 100755 --- a/potpyri/primitives/absphot.py +++ b/potpyri/primitives/absphot.py @@ -113,7 +113,9 @@ def get_catalog(self, coords, catalog, filt, log=None): if log: log.info(f'Getting {catalog} catalog with ID {cat_ID} in filt {filt}') log.info(f'Querying around {coord_ra}, {coord_dec} deg') - cat = vizier.query_region(med_coord, width=1.2*max_sep*u.degree, + Vizier.clear_cache() + width = np.max([2.0 * max_sep, 0.5]) + cat = vizier.query_region(med_coord, width=width*u.degree, catalog=cat_ID) if len(cat)>0: @@ -182,10 +184,10 @@ def find_zeropoint(self, cmpfile, tel, match_radius=2.5*u.arcsec, cat, catalog, cat_ID = self.get_catalog(coords, catalog, filt, log=log) - min_mag = self.get_minmag(filt) - cat = cat[cat['mag']>min_mag] - if cat: + min_mag = self.get_minmag(filt) + cat = cat[cat['mag']>min_mag] + coords_cat = SkyCoord(cat['ra'], cat['dec'], unit='deg') idx, d2, d3 = coords_cat.match_to_catalog_sky(coords) From 3862b77e4cbcfbd44b0e8dc1bb75f236a3c0d963 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 13:34:03 -0600 Subject: [PATCH 06/25] Testing dynamic FWHM on astropyscrappy/lacosmic --- potpyri/primitives/image_procs.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 18481f7..5687904 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -29,6 +29,7 @@ # Internal dependencies from potpyri.primitives import solve_wcs from potpyri.utils import utilities +from potpyri.photometry import run_sextractor def remove_pv_distortion(header): @@ -616,6 +617,14 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.5, hdu = fits.open(science_data) data = np.ascontiguousarray(hdu[0].data.astype(np.float32)) + # Estimate FWHM size for LACosmic + table = run_sextractor(science_data) + if table is not None: + # Clip fwhm_stars by fwhm + fwhm, meanfwhm, stdfwhm = sigma_clipped_stats(table['FWHM_IMAGE']) + else: + fwhm = 3.5 + # Astroscrappy requires added sky background, so add this value back # Set the sky background to some nominal value if it is too low for CR rej skybkg = hdu[0].header['SKYBKG'] @@ -659,7 +668,8 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.5, if cosmic_ray: newdata, mask_cr = cosmicray_lacosmic(data, readnoise=rdnoise, satlevel=saturation, verbose=True, - sigclip=sigclip, sigfrac=sigfrac, objlim=objlim, niter=niter) + sigclip=sigclip, sigfrac=sigfrac, objlim=objlim, niter=niter, + psffwhm=fwhm) mask_cr = mask_cr.astype(np.uint8) #set cosmic ray mask type else: From 4b385cb37450685858d04c8e3828d2a90e36b3aa Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 13:36:38 -0600 Subject: [PATCH 07/25] Typo in photometry import for image_procs --- potpyri/primitives/image_procs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 5687904..190f1fa 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -27,9 +27,9 @@ from ccdproc import cosmicray_lacosmic # Internal dependencies +from potpyri.primitives import photometry from potpyri.primitives import solve_wcs from potpyri.utils import utilities -from potpyri.photometry import run_sextractor def remove_pv_distortion(header): @@ -618,7 +618,7 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.5, data = np.ascontiguousarray(hdu[0].data.astype(np.float32)) # Estimate FWHM size for LACosmic - table = run_sextractor(science_data) + table = photometry.run_sextractor(science_data) if table is not None: # Clip fwhm_stars by fwhm fwhm, meanfwhm, stdfwhm = sigma_clipped_stats(table['FWHM_IMAGE']) From 43adc800d1a743d25416a13b189b51838f015c97 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 21:06:18 -0600 Subject: [PATCH 08/25] Testing PSF parameters for cosmic ray --- potpyri/primitives/image_procs.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 190f1fa..c96c451 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -139,8 +139,11 @@ def align_images(reduced_files, paths, tel, binn, use_wcs=None, fieldcenter=None if log: log.info('Rejecting the following images for high astrometric dispersion:') - for i,m in enumerate(mask): - if not m: log.info(solved_images[i]) + if np.all(mask): + log.info('No images rejected') + else: + for i,m in enumerate(mask): + if not m: log.info(solved_images[i]) solved_images = np.array(solved_images)[mask] else: @@ -618,10 +621,11 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.5, data = np.ascontiguousarray(hdu[0].data.astype(np.float32)) # Estimate FWHM size for LACosmic - table = photometry.run_sextractor(science_data) + table = photometry.run_sextractor(science_data, log=log) if table is not None: # Clip fwhm_stars by fwhm fwhm, meanfwhm, stdfwhm = sigma_clipped_stats(table['FWHM_IMAGE']) + if log: log.info(f'Using FWHM for cosmic rays: {fwhm}') else: fwhm = 3.5 @@ -665,11 +669,14 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.5, mask_cr = np.zeros(data.shape) mask_cr = mask_cr.astype(np.uint8) + psfsize = int(np.round(2.5*fwhm)) + if psfsize%2==0: psfsize+=1 + if cosmic_ray: newdata, mask_cr = cosmicray_lacosmic(data, readnoise=rdnoise, satlevel=saturation, verbose=True, sigclip=sigclip, sigfrac=sigfrac, objlim=objlim, niter=niter, - psffwhm=fwhm) + psffwhm=fwhm, psfsize=psfsize) mask_cr = mask_cr.astype(np.uint8) #set cosmic ray mask type else: From 9600b6fb918a5addc7169911290b20d2029740a6 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 21:29:42 -0600 Subject: [PATCH 09/25] Testing PSF parameters for cosmic ray --- potpyri/primitives/image_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index c96c451..20c60e5 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -709,7 +709,7 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.5, nsat = np.sum(mask & 4 == 4) ngrow = np.sum(mask & 8 == 8) - mask_hdu = fits.PrimaryHDU(mask) #create mask Primary HDU + mask_hdu = fits.ImageHDU(mask) #create mask Primary HDU mask_hdu.header['VER'] = (__version__, 'Version of image procedures used.') #name of log file mask_hdu.header['USE'] = 'Complex mask using additive flags.'#header comment From 90c3cc60ffd5dc46e7353cba18e12e21afee1ac0 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 21:33:38 -0600 Subject: [PATCH 10/25] Testing PSF parameters for cosmic ray --- potpyri/primitives/image_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 20c60e5..c14c817 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -606,7 +606,7 @@ def mask_satellites(images, filenames, log=None): os.remove(tmpfile) -def create_mask(science_data, saturation, rdnoise, sigclip=3.5, +def create_mask(science_data, saturation, rdnoise, sigclip=3.0, sigfrac=0.2, objlim=4.5, niter=6, outpath='', grow=0, satellites=True, cosmic_ray=True, log=None): From 9fbb5549ed5d702f0fa1f6fae0a387a95cc34623 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 21:38:18 -0600 Subject: [PATCH 11/25] Testing PSF parameters for cosmic ray --- potpyri/primitives/image_procs.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index c14c817..df1db50 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -607,8 +607,8 @@ def mask_satellites(images, filenames, log=None): os.remove(tmpfile) def create_mask(science_data, saturation, rdnoise, sigclip=3.0, - sigfrac=0.2, objlim=4.5, niter=6, outpath='', grow=0, satellites=True, - cosmic_ray=True, log=None): + sigfrac=0.2, objlim=4.5, niter=6, outpath='', grow=0, cosmic_ray=True, + fsmode='convolve', log=None): t_start = time.time() @@ -676,7 +676,7 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.0, newdata, mask_cr = cosmicray_lacosmic(data, readnoise=rdnoise, satlevel=saturation, verbose=True, sigclip=sigclip, sigfrac=sigfrac, objlim=objlim, niter=niter, - psffwhm=fwhm, psfsize=psfsize) + psffwhm=fwhm, psfsize=psfsize, fsmode=fsmode) mask_cr = mask_cr.astype(np.uint8) #set cosmic ray mask type else: From 7d4fbe21faa2ead51b46a0c6bb6fdb27316c4786 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 21:48:35 -0600 Subject: [PATCH 12/25] Testing PSF parameters for cosmic ray --- potpyri/primitives/image_procs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index df1db50..245029d 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -608,7 +608,7 @@ def mask_satellites(images, filenames, log=None): def create_mask(science_data, saturation, rdnoise, sigclip=3.0, sigfrac=0.2, objlim=4.5, niter=6, outpath='', grow=0, cosmic_ray=True, - fsmode='convolve', log=None): + fsmode='convolve', cleanmask='medmask', log=None): t_start = time.time() @@ -676,7 +676,8 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.0, newdata, mask_cr = cosmicray_lacosmic(data, readnoise=rdnoise, satlevel=saturation, verbose=True, sigclip=sigclip, sigfrac=sigfrac, objlim=objlim, niter=niter, - psffwhm=fwhm, psfsize=psfsize, fsmode=fsmode) + psffwhm=fwhm, psfsize=psfsize, fsmode=fsmode, + cleanmask=cleanmask) mask_cr = mask_cr.astype(np.uint8) #set cosmic ray mask type else: From bd03789eff05dfff99723859e5e008b75576b676 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 21:50:00 -0600 Subject: [PATCH 13/25] Testing PSF parameters for cosmic ray --- potpyri/primitives/image_procs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 245029d..9013373 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -608,7 +608,7 @@ def mask_satellites(images, filenames, log=None): def create_mask(science_data, saturation, rdnoise, sigclip=3.0, sigfrac=0.2, objlim=4.5, niter=6, outpath='', grow=0, cosmic_ray=True, - fsmode='convolve', cleanmask='medmask', log=None): + fsmode='convolve', cleantype='medmask', log=None): t_start = time.time() @@ -677,7 +677,7 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.0, readnoise=rdnoise, satlevel=saturation, verbose=True, sigclip=sigclip, sigfrac=sigfrac, objlim=objlim, niter=niter, psffwhm=fwhm, psfsize=psfsize, fsmode=fsmode, - cleanmask=cleanmask) + cleantype=cleantype, sepmed=False) mask_cr = mask_cr.astype(np.uint8) #set cosmic ray mask type else: From 724b905d6ce358540e63561dde934545c61329c4 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 21:53:10 -0600 Subject: [PATCH 14/25] Testing PSF parameters for cosmic ray --- potpyri/primitives/image_procs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 9013373..4e205a6 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -607,8 +607,8 @@ def mask_satellites(images, filenames, log=None): os.remove(tmpfile) def create_mask(science_data, saturation, rdnoise, sigclip=3.0, - sigfrac=0.2, objlim=4.5, niter=6, outpath='', grow=0, cosmic_ray=True, - fsmode='convolve', cleantype='medmask', log=None): + sigfrac=0.15, objlim=4.5, niter=6, outpath='', grow=0, cosmic_ray=True, + fsmode='convolve', cleantype='meanmask', log=None): t_start = time.time() From ff3fc655ec57c65de30c0b168d291d9c66d91816 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 21:55:21 -0600 Subject: [PATCH 15/25] Testing PSF parameters for cosmic ray --- potpyri/primitives/image_procs.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 4e205a6..d28c29c 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -608,7 +608,7 @@ def mask_satellites(images, filenames, log=None): def create_mask(science_data, saturation, rdnoise, sigclip=3.0, sigfrac=0.15, objlim=4.5, niter=6, outpath='', grow=0, cosmic_ray=True, - fsmode='convolve', cleantype='meanmask', log=None): + fsmode='convolve', cleantype='medmask', log=None): t_start = time.time() @@ -625,7 +625,10 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.0, if table is not None: # Clip fwhm_stars by fwhm fwhm, meanfwhm, stdfwhm = sigma_clipped_stats(table['FWHM_IMAGE']) - if log: log.info(f'Using FWHM for cosmic rays: {fwhm}') + if log: + log.info(f'Using FWHM for cosmic rays: {fwhm}') + else: + print(f'Using FWHM for cosmic rays: {fwhm}') else: fwhm = 3.5 @@ -677,7 +680,7 @@ def create_mask(science_data, saturation, rdnoise, sigclip=3.0, readnoise=rdnoise, satlevel=saturation, verbose=True, sigclip=sigclip, sigfrac=sigfrac, objlim=objlim, niter=niter, psffwhm=fwhm, psfsize=psfsize, fsmode=fsmode, - cleantype=cleantype, sepmed=False) + cleantype=cleantype) mask_cr = mask_cr.astype(np.uint8) #set cosmic ray mask type else: From 472eb271a685f23f3c1704bd956a4f294cb002ae Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Thu, 20 Nov 2025 21:58:29 -0600 Subject: [PATCH 16/25] Testing PSF parameters for cosmic ray --- potpyri/primitives/image_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index d28c29c..1129d9a 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -607,7 +607,7 @@ def mask_satellites(images, filenames, log=None): os.remove(tmpfile) def create_mask(science_data, saturation, rdnoise, sigclip=3.0, - sigfrac=0.15, objlim=4.5, niter=6, outpath='', grow=0, cosmic_ray=True, + sigfrac=0.10, objlim=4.0, niter=6, outpath='', grow=0, cosmic_ray=True, fsmode='convolve', cleantype='medmask', log=None): t_start = time.time() From 2871db88b5540be703573a49c401774456a7b255 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Fri, 21 Nov 2025 09:16:03 -0600 Subject: [PATCH 17/25] Adjusting memory allocation for image stacking --- potpyri/primitives/image_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 1129d9a..a5d621e 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -489,7 +489,7 @@ def stack_data(stacking_data, tel, masks, errors, mem_limit=8.0e9, log=None): print(f'Image size is {size}') # Get maximum size of chunk - max_chunk = int(np.floor(mem_limit / 4.0 / (size))) + max_chunk = int(np.floor(mem_limit / 2.0 / (size))) if max_chunk==0: max_chunk=1 if len(stacking_data)<=max_chunk: From 08375c67b8c47d4d071e9ea67a74395153a2f80f Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Tue, 25 Nov 2025 10:26:49 -0600 Subject: [PATCH 18/25] Fixing issue with memory allocation in image stacking --- potpyri/primitives/image_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index a5d621e..18ffee5 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -494,7 +494,7 @@ def stack_data(stacking_data, tel, masks, errors, mem_limit=8.0e9, log=None): if len(stacking_data)<=max_chunk: sci_med = combine(stacking_data, weights=weights, scale=scale, - method=stack_method, mem_limit=mem_limit) + method=stack_method, mem_limit=2*mem_limit) else: nimgs = len(stacking_data) * 1.0 nchunks = int(np.ceil(nimgs/(1.0*max_chunk))) From 5bd90e0620ab50f68fa10d32dbc9f142eede5474 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Mon, 8 Dec 2025 16:13:58 -0600 Subject: [PATCH 19/25] Updating PS1 catalog to cut on Kron magnitudes for point sources --- potpyri/primitives/absphot.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/potpyri/primitives/absphot.py b/potpyri/primitives/absphot.py index 3ee9217..93212f8 100755 --- a/potpyri/primitives/absphot.py +++ b/potpyri/primitives/absphot.py @@ -107,8 +107,13 @@ def get_catalog(self, coords, catalog, filt, log=None): seps = med_coord.separation(coords) max_sep = np.max(seps.to(u.deg).value) + + cols = [cat_ra, cat_dec, cat_mag, cat_err] + # Add Kron mag if the catalog is PS1 + if cat_ID=='II/349': + cols.append(f'{filt}Kmag') - vizier = Vizier(columns=[cat_ra, cat_dec, cat_mag, cat_err]) + vizier = Vizier(columns=cols) vizier.ROW_LIMIT = -1 if log: log.info(f'Getting {catalog} catalog with ID {cat_ID} in filt {filt}') @@ -123,6 +128,23 @@ def get_catalog(self, coords, catalog, filt, log=None): cat = cat[~np.isnan(cat[cat_mag])] cat = cat[cat[cat_err]>0.] + if cat_ID=='II/349': + if log: + log.info('Cutting on Kron magnitudes') + else: + print('Cutting on Kron magnitudes') + + nsources = len(cat) + cat_kron = f'{filt}Kmag' + mask = cat[cat_mag]-cat[cat_kron] < 0.1 + cat = cat[mask] + nkron = len(cat) + + if log: + log.info(f'Cut catalog from {nsources} to {nkron}') + else: + print(f'Cut catalog from {nsources} to {nkron}') + cat.rename_column(cat_ra, 'ra') cat.rename_column(cat_dec, 'dec') cat.rename_column(cat_mag, 'mag') @@ -306,3 +328,10 @@ def get_minmag(self, filt): def find_zeropoint(stack, tel, log=None): cal = absphot() cal.find_zeropoint(stack, tel, log=log) + +if __name__=="__main__": + from potpyri.instruments import instrument_getter + tel = instrument_getter('GMOS') + + cal = absphot() + cal.find_zeropoint('/Users/ckilpatrick/FRB20250428A/red/FRB20250428A.r.ut251129.12.22.stk.fits', tel) From 75e4eca14f8ab9d24fbaeebfe7d2d7fb2b64356e Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Mon, 15 Dec 2025 14:43:51 -0600 Subject: [PATCH 20/25] Fixing issue with Gaia WCS solution where new_wcs is calculated incorrectly with too few astrometric calibrators --- potpyri/primitives/absphot.py | 7 ------- potpyri/primitives/solve_wcs.py | 18 ++++++++++++++++-- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/potpyri/primitives/absphot.py b/potpyri/primitives/absphot.py index 93212f8..5430ae9 100755 --- a/potpyri/primitives/absphot.py +++ b/potpyri/primitives/absphot.py @@ -328,10 +328,3 @@ def get_minmag(self, filt): def find_zeropoint(stack, tel, log=None): cal = absphot() cal.find_zeropoint(stack, tel, log=log) - -if __name__=="__main__": - from potpyri.instruments import instrument_getter - tel = instrument_getter('GMOS') - - cal = absphot() - cal.find_zeropoint('/Users/ckilpatrick/FRB20250428A/red/FRB20250428A.r.ut251129.12.22.stk.fits', tel) diff --git a/potpyri/primitives/solve_wcs.py b/potpyri/primitives/solve_wcs.py index 89f7502..ab387e9 100755 --- a/potpyri/primitives/solve_wcs.py +++ b/potpyri/primitives/solve_wcs.py @@ -335,7 +335,7 @@ def solve_astrometry(file, tel, binn, paths, radius=0.5, replace=True, return(False) def align_to_gaia(file, tel, radius=0.5, max_search_radius=5.0*u.arcsec, - save_centroids=False, log=None): + save_centroids=False, min_gaia_match=5, log=None): cat = get_gaia_catalog(file, log=log) @@ -379,7 +379,7 @@ def align_to_gaia(file, tel, radius=0.5, max_search_radius=5.0*u.arcsec, else: print(f'Found {len(cat)} stars in the image') - if len(cat)<7: + if len(cat) Date: Mon, 15 Dec 2025 23:13:02 -0600 Subject: [PATCH 21/25] Refactoring chunking data for large memory usage --- potpyri/primitives/image_procs.py | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 18ffee5..9f2a412 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -444,7 +444,7 @@ def detrend_stack(stack): return(stack) -def stack_data(stacking_data, tel, masks, errors, mem_limit=8.0e9, log=None): +def stack_data(stacking_data, tel, masks, errors, mem_limit=16.0e9, log=None): stack_method = tel.stack_method if not stack_method: @@ -461,7 +461,7 @@ def stack_data(stacking_data, tel, masks, errors, mem_limit=8.0e9, log=None): mask = masks[i] ivar[mask.data.astype(bool)]=0. weights.append(ivar) - weights=np.array(weights) + #weights=np.array(weights) new_data = [] exptimes = [] @@ -481,23 +481,32 @@ def stack_data(stacking_data, tel, masks, errors, mem_limit=8.0e9, log=None): # Determine if intermediate stacks are needed # Get size of individual frame, account for data, noise, and mask exdata = stacking_data[0].data - size = 64.0/8.0 * exdata.shape[0]*exdata.shape[1] * 2 + exdata.shape[0]*exdata.shape[1] + size_of_image = exdata.data.nbytes + size_of_image += exdata.mask.nbytes + size_of_image += exdata.flags.nbytes if log: - log.info(f'Image size is {size}') + log.info(f'Image size is {size_of_image}') else: - print(f'Image size is {size}') + print(f'Image size is {size_of_image}') - # Get maximum size of chunk - max_chunk = int(np.floor(mem_limit / 2.0 / (size))) - if max_chunk==0: max_chunk=1 + no_of_img = len(stacking_data) - if len(stacking_data)<=max_chunk: + if stack_method=='median': + memory_factor = 3 + else: + memory_factor = 2 + + memory_factor *= 1.3 + + # Get number of chunks + nchunks = int((memory_factor * size_of_an_img * no_of_img) / mem_limit) + 1 + + if nchunks==1: sci_med = combine(stacking_data, weights=weights, scale=scale, - method=stack_method, mem_limit=2*mem_limit) + method=stack_method, mem_limit=mem_limit) else: nimgs = len(stacking_data) * 1.0 - nchunks = int(np.ceil(nimgs/(1.0*max_chunk))) if log: log.info(f'Splitting stacking into {nchunks} chunks') From acf4db1cc5fdac4effd87ca9a7fc465b755054f4 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Mon, 15 Dec 2025 23:49:29 -0600 Subject: [PATCH 22/25] Refactoring stacking in image_procs.py --- potpyri/primitives/image_procs.py | 52 +------------------------------ 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 9f2a412..f238e19 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -478,58 +478,8 @@ def stack_data(stacking_data, tel, masks, errors, mem_limit=16.0e9, log=None): if len(scale)==1: stack_method='average' - # Determine if intermediate stacks are needed - # Get size of individual frame, account for data, noise, and mask - exdata = stacking_data[0].data - size_of_image = exdata.data.nbytes - size_of_image += exdata.mask.nbytes - size_of_image += exdata.flags.nbytes - - if log: - log.info(f'Image size is {size_of_image}') - else: - print(f'Image size is {size_of_image}') - - no_of_img = len(stacking_data) - - if stack_method=='median': - memory_factor = 3 - else: - memory_factor = 2 - - memory_factor *= 1.3 - - # Get number of chunks - nchunks = int((memory_factor * size_of_an_img * no_of_img) / mem_limit) + 1 - - if nchunks==1: - sci_med = combine(stacking_data, weights=weights, scale=scale, + sci_med = combine(stacking_data, weights=weights, scale=scale, method=stack_method, mem_limit=mem_limit) - else: - nimgs = len(stacking_data) * 1.0 - - if log: - log.info(f'Splitting stacking into {nchunks} chunks') - else: - print(f'Splitting stacking into {nchunks} chunks') - - inter_med = [] - for i in np.arange(nchunks): - if log: - log.info(f'Stacking chunk {i+1}/{nchunks}') - else: - print(f'Stacking chunk {i+1}/{nchunks}') - start = i*max_chunk - end = (i+1)*max_chunk - if end>len(stacking_data): end=None - - sci_med = combine(stacking_data[start:end], - weights=weights[start:end], scale=scale[start:end], - method=stack_method, mem_limit=mem_limit) - - inter_med.append(sci_med) - - sci_med = combine(inter_med, method=stack_method) sci_med = sci_med.to_hdu() From b2afa0b917d709fdb8b0365ee5c96f209c21b34e Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Tue, 16 Dec 2025 00:16:23 -0600 Subject: [PATCH 23/25] Refactoring stacking in image_procs.py --- potpyri/primitives/image_procs.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index f238e19..f2b94a1 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -444,7 +444,7 @@ def detrend_stack(stack): return(stack) -def stack_data(stacking_data, tel, masks, errors, mem_limit=16.0e9, log=None): +def stack_data(stacking_data, tel, masks, errors, mem_limit=8.0e9, log=None): stack_method = tel.stack_method if not stack_method: @@ -461,24 +461,24 @@ def stack_data(stacking_data, tel, masks, errors, mem_limit=16.0e9, log=None): mask = masks[i] ivar[mask.data.astype(bool)]=0. weights.append(ivar) - #weights=np.array(weights) + weights=np.array(weights) - new_data = [] + all_data = [] exptimes = [] for i,stk in enumerate(stacking_data): mask = masks[i].data.astype(bool) stacking_data[i].data[mask] = np.nan exptimes.append(float(tel.get_exptime(stk.header))) - + all_data.append(stacking_data[i].data) + + all_data=np.array(new_data) exptimes = np.array(exptimes) scale = 1./exptimes - all_data = np.array([s.data for s in stacking_data]) - if len(scale)==1: stack_method='average' - sci_med = combine(stacking_data, weights=weights, scale=scale, + sci_med = combine(all_data, weights=weights, scale=scale, method=stack_method, mem_limit=mem_limit) sci_med = sci_med.to_hdu() From dcf1f2c3820b810af185198e2cc6b516e98fae26 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Tue, 16 Dec 2025 00:17:09 -0600 Subject: [PATCH 24/25] Refactoring stacking in image_procs.py --- potpyri/primitives/image_procs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index f2b94a1..825dd26 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -471,7 +471,7 @@ def stack_data(stacking_data, tel, masks, errors, mem_limit=8.0e9, log=None): exptimes.append(float(tel.get_exptime(stk.header))) all_data.append(stacking_data[i].data) - all_data=np.array(new_data) + all_data=np.array(all_data) exptimes = np.array(exptimes) scale = 1./exptimes From f68b762c5b6a35bdd7f5c23935d9fd5593eb4476 Mon Sep 17 00:00:00 2001 From: charliekilpatrick Date: Tue, 16 Dec 2025 01:06:11 -0600 Subject: [PATCH 25/25] Refactoring stacking in image_procs.py --- potpyri/primitives/image_procs.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/potpyri/primitives/image_procs.py b/potpyri/primitives/image_procs.py index 825dd26..27c49fc 100755 --- a/potpyri/primitives/image_procs.py +++ b/potpyri/primitives/image_procs.py @@ -455,30 +455,19 @@ def stack_data(stacking_data, tel, masks, errors, mem_limit=8.0e9, log=None): else: raise Exception('Could not get stacking method for these images') - weights = [] - for i,error in enumerate(errors): - ivar = 1./error.data**2 - mask = masks[i] - ivar[mask.data.astype(bool)]=0. - weights.append(ivar) - weights=np.array(weights) - - all_data = [] exptimes = [] for i,stk in enumerate(stacking_data): mask = masks[i].data.astype(bool) stacking_data[i].data[mask] = np.nan exptimes.append(float(tel.get_exptime(stk.header))) - all_data.append(stacking_data[i].data) - all_data=np.array(all_data) exptimes = np.array(exptimes) scale = 1./exptimes if len(scale)==1: stack_method='average' - sci_med = combine(all_data, weights=weights, scale=scale, + sci_med = combine(stacking_data, scale=scale, method=stack_method, mem_limit=mem_limit) sci_med = sci_med.to_hdu()