From 898517ce458f11aa3f8df1e7c3faf42d3335d2d6 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 23 Jun 2025 15:06:25 -0600 Subject: [PATCH 01/74] improved docstring --- src/astrohack/utils/gridding.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/astrohack/utils/gridding.py b/src/astrohack/utils/gridding.py index c626bf75..b38018f3 100644 --- a/src/astrohack/utils/gridding.py +++ b/src/astrohack/utils/gridding.py @@ -37,6 +37,7 @@ def grid_beam( chan_tol_fac: Frequency tolerance to chunk channels together telescope: Telescope object containing optical description of the telescope grid_interpolation_mode: linear, nearest, cubic or gaussian (convolution) + label: label to be used in messages Returns: The gridded beam, its time centroid, frequency axis, polarization axis, L and M axes and a boolean about the From 1f3123c4f5a47a66a3c6f5ae5763684d1bd2ed45 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 23 Jun 2025 15:10:01 -0600 Subject: [PATCH 02/74] improved docstring --- src/astrohack/utils/imaging.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/astrohack/utils/imaging.py b/src/astrohack/utils/imaging.py index b58cbc8f..f2747d84 100644 --- a/src/astrohack/utils/imaging.py +++ b/src/astrohack/utils/imaging.py @@ -105,12 +105,13 @@ def calculate_far_field_aperture( Args: grid (numpy.ndarray): gridded beam data - padding_factor (int, optional): Padding to apply to beam data grid before FFT. Padding is applied on outer edged of - each beam data grid and not between layers. Defaults to 20. + padding_factor (int, optional): Padding to apply to beam data grid before FFT. Padding is applied on outer edged + of each beam data grid and not between layers. Defaults to 20. freq: Beam grid frequency axis telescope: telescope object with optical parameters sky_cell_size: Sky cell size (radians) apply_grid_correction: Apply grid correction (True for gaussian convolution of the beam) + label: Data label for messages Returns: aperture grid, u-coordinate array, v-coordinate array, aperture cell size, representative wavelength @@ -160,6 +161,7 @@ def calculate_near_field_aperture( telescope: telescope object with optical parameters apply_grid_correction: Apply grid correction (True for gaussian convolution of the beam) apodize: Apodize beam to avoid boxing effects in the FFT (the dashed line cross) + label: Data label for messages Returns: aperture grid, u-coordinate array, v-coordinate array, aperture cell size, representative wavelength From f1242ed30eb516a8e870194f64ced318f3653b5c Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 23 Jun 2025 15:21:44 -0600 Subject: [PATCH 03/74] Renamed import that was named as a single letter. --- src/astrohack/utils/imaging.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/astrohack/utils/imaging.py b/src/astrohack/utils/imaging.py index f2747d84..c4ed1730 100644 --- a/src/astrohack/utils/imaging.py +++ b/src/astrohack/utils/imaging.py @@ -1,7 +1,7 @@ import math import scipy.ndimage import numpy as np -import astropy.units as u +import astropy.units as units import astropy.coordinates as coord from numba import njit import scipy.fftpack @@ -44,15 +44,15 @@ def calculate_parallactic_angle_chunk( """ observing_location = coord.EarthLocation.from_geocentric( - x=observing_location[0] * u.m, - y=observing_location[1] * u.m, - z=observing_location[2] * u.m, + x=observing_location[0] * units.m, + y=observing_location[1] * units.m, + z=observing_location[2] * units.m, ) direction = coord.SkyCoord( - ra=direction[:, 0] * u.rad, dec=direction[:, 1] * u.rad, frame=dir_frame.lower() + ra=direction[:, 0] * units.rad, dec=direction[:, 1] * units.rad, frame=dir_frame.lower() ) - zenith = coord.SkyCoord(0, 90, unit=u.deg, frame=zenith_frame.lower()) + zenith = coord.SkyCoord(0, 90, unit=units.deg, frame=zenith_frame.lower()) altaz_frame = coord.AltAz(location=observing_location, obstime=time_samples) zenith_altaz = zenith.transform_to(altaz_frame) From 211c3d9fe6881969a391baaf041fef27fbc04e13 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 23 Jun 2025 15:54:09 -0600 Subject: [PATCH 04/74] holog_mds now stores an AZ EL summary. --- src/astrohack/core/extract_holog.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index 12ade6af..56665d3f 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -1,5 +1,6 @@ import os import json + import numpy as np import xarray as xr import astropy @@ -446,6 +447,7 @@ def _create_holog_file( xds.attrs["parallactic_samples"] = parallactic_samples xds.attrs["telescope_name"] = telescope_name xds.attrs["antenna_name"] = ant_names[map_ant_index] + xds.attrs["az_el_information"] = _get_az_el_characteristics(pnt_map_dict[map_ant_tag], valid_data) xds.attrs["l_max"] = np.max(xds["DIRECTIONAL_COSINES"][:, 0].values) xds.attrs["l_min"] = np.min(xds["DIRECTIONAL_COSINES"][:, 0].values) @@ -874,3 +876,17 @@ def _get_time_index(data_time, i_time, time_axis, half_int): if i_time == time_axis.shape[0]: return -1 return i_time + + +def _get_az_el_characteristics(pnt_map_xds, valid_data): + az_el = pnt_map_xds["ENCODER"].values[valid_data, ...] + lm = pnt_map_xds["DIRECTIONAL_COSINES"].values[valid_data, ...] + rad2deg = 180/np.pi + mean_az_el = np.mean(az_el, axis=0)*rad2deg + median_az_el = np.median(az_el, axis=0)*rad2deg + ic = np.argmin((lm[:, 0]**2 + lm[:, 1])**2) + center_az_el = az_el[ic]*rad2deg + az_el_info = {'center': center_az_el.tolist(), + 'mean': mean_az_el.tolist(), + 'median': median_az_el.tolist()} + return az_el_info From df4dd35267321f5ee66986d72c977f714f34df0d Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 23 Jun 2025 16:02:38 -0600 Subject: [PATCH 05/74] holog now propagates AZ EL summary. --- src/astrohack/core/holog.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/astrohack/core/holog.py b/src/astrohack/core/holog.py index 28fae60b..2bb69afd 100644 --- a/src/astrohack/core/holog.py +++ b/src/astrohack/core/holog.py @@ -48,6 +48,7 @@ def process_holog_chunk(holog_chunk_params): telescope = _get_correct_telescope( ref_xds.attrs["antenna_name"], meta_data["telescope_name"] ) + az_el_info = ref_xds.attrs['az_el_information'] try: is_near_field = ref_xds.attrs["near_field"] except KeyError: @@ -195,6 +196,7 @@ def process_holog_chunk(holog_chunk_params): zernike_rms, zernike_n_order, holog_chunk_params["image_name"], + az_el_info ) logger.info(f"Finished processing {label}") @@ -293,6 +295,7 @@ def _export_to_xds( zernike_rms, zernike_n_order, image_name, + az_el_info, ): # Todo: Add Paralactic angle as a non-dimension coordinate dependant on time. xds = xr.Dataset() @@ -327,6 +330,7 @@ def _export_to_xds( xds.attrs["ddi"] = ddi xds.attrs["phase_fitting"] = phase_fit_results xds.attrs["zernike_N_order"] = zernike_n_order + xds.attrs["az_el_information"] = az_el_info coords = { "orig_pol": orig_pol_axis, From 9c4be17c4e7a04ba8ffab9f23a62872377c2b033 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 23 Jun 2025 16:06:59 -0600 Subject: [PATCH 06/74] panel now propagates AZ EL summary. --- src/astrohack/antenna/antenna_surface.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/astrohack/antenna/antenna_surface.py b/src/astrohack/antenna/antenna_surface.py index 971bb0b1..9406f39f 100644 --- a/src/astrohack/antenna/antenna_surface.py +++ b/src/astrohack/antenna/antenna_surface.py @@ -88,7 +88,7 @@ def __init__( self.pol_state = pol_state self._read_xds(inputxds) self.telescope = telescope - + if patch_phase: self.phase = phase_wrapping(self.phase) @@ -216,12 +216,14 @@ def _read_xds(self, inputxds): inputxds: X array dataset """ # Origin dependant reading + if self.reread: self._read_panel_xds(inputxds) else: self._read_holog_xds(inputxds) # Common elements + self.az_el_info = inputxds.attrs['az_el_information'] self.antenna_name = inputxds.attrs["ant_name"] self.ddi = inputxds.attrs["ddi"] self.label = create_dataset_label( @@ -919,6 +921,8 @@ def export_xds(self): xds.attrs["fitted"] = self.fitted xds.attrs["aperture_resolution"] = self.resolution xds.attrs["pol_state"] = self.pol_state + xds.attrs['az_el_information'] = self.az_el_info + xds["AMPLITUDE"] = xr.DataArray(self.amplitude, dims=["u", "v"]) xds["PHASE"] = xr.DataArray(self.phase, dims=["u", "v"]) xds["DEVIATION"] = xr.DataArray(self.deviation, dims=["u", "v"]) From 3c0c86dd084b5dfffcaf2ee3772aa5ca4c339429 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 11:14:19 -0600 Subject: [PATCH 07/74] Az El is now left in radians for convenient conversion at output. further restriction on fetch position of center limiting to the middle of the observation. --- src/astrohack/core/extract_holog.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index 56665d3f..b6213f08 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -881,11 +881,14 @@ def _get_time_index(data_time, i_time, time_axis, half_int): def _get_az_el_characteristics(pnt_map_xds, valid_data): az_el = pnt_map_xds["ENCODER"].values[valid_data, ...] lm = pnt_map_xds["DIRECTIONAL_COSINES"].values[valid_data, ...] - rad2deg = 180/np.pi - mean_az_el = np.mean(az_el, axis=0)*rad2deg - median_az_el = np.median(az_el, axis=0)*rad2deg - ic = np.argmin((lm[:, 0]**2 + lm[:, 1])**2) - center_az_el = az_el[ic]*rad2deg + mean_az_el = np.mean(az_el, axis=0) + median_az_el = np.median(az_el, axis=0) + lmmid = lm.shape[0]//2 + lmquart = lmmid//2 + ilow = lmmid-lmquart + iupper = lmmid+lmquart+1 + ic = np.argmin((lm[ilow:iupper, 0] ** 2 + lm[ilow:iupper, 1]) ** 2) + ilow + center_az_el = az_el[ic] az_el_info = {'center': center_az_el.tolist(), 'mean': mean_az_el.tolist(), 'median': median_az_el.tolist()} From 5f33100ed3fb2a4b843a2c39e48daccbf71d7b4e Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 11:15:53 -0600 Subject: [PATCH 08/74] Created a routine to format Az El information. --- src/astrohack/utils/text.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/astrohack/utils/text.py b/src/astrohack/utils/text.py index d0de913e..f3b862eb 100644 --- a/src/astrohack/utils/text.py +++ b/src/astrohack/utils/text.py @@ -674,3 +674,16 @@ def format_object_contents(obj): total_size += size outstr += f"Total size = {format_byte_size(total_size)}\n" return outstr + + +def format_az_el_information(az_el_dict, key='center', unit='deg', precision='.2g'): + if key == 'center': + prefix = '@ l,m = (0,0),' + elif key in ['mean', 'median']: + prefix = key.capitalize() + else: + raise ValueError(f"Unrecognized key: {key}") + + az_el = np.array(az_el_dict[key])*convert_unit('rad', unit, 'trigonometric') + az_el_label = f'{prefix} Az, El = ({az_el[0]:{precision}}, {az_el[1]:{precision}}) [{unit}]' + return az_el_label From 26bc96d612ea0776624fa8541a806a22ca88ad36 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 11:47:33 -0600 Subject: [PATCH 09/74] extract_holog.py now extracts and stores an observation summary. --- src/astrohack/core/extract_holog.py | 70 ++++++++++++++++++++++------- src/astrohack/extract_holog.py | 4 -- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index b6213f08..c191b24b 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -15,6 +15,7 @@ from astrohack.utils import create_dataset_label from astrohack.utils.imaging import calculate_parallactic_angle_chunk from astrohack.utils.algorithms import calculate_optimal_grid_parameters +from astrohack.utils.conversion import casa_time_to_mjd from astrohack.utils.file import load_point_file @@ -23,13 +24,16 @@ def process_extract_holog_chunk(extract_holog_params): """Perform data query on holography data chunk and get unique time and state_ids/ Args: + extract_holog_params: dictionary containing parameters + + Some of the parameters are: ms_name (str): Measurementset name data_column (str): Data column to extract. ddi (int): Data description id scan (int): Scan number map_ant_ids (numpy.narray): Array of antenna_id values corresponding to mapping data. ref_ant_ids (numpy.narray): Arry of antenna_id values corresponding to reference data. - sel_state_ids (list): List pf state_ids corresponding to holography data/ + sel_state_ids (list): List pf state_ids corresponding to holography data """ ms_name = extract_holog_params["ms_name"] @@ -43,7 +47,6 @@ def process_extract_holog_chunk(extract_holog_params): map_ant_name_tuple = extract_holog_params["map_ant_name_tuple"] holog_map_key = extract_holog_params["holog_map_key"] time_interval = extract_holog_params["time_smoothing_interval"] - telescope_name = extract_holog_params["telescope_name"] # This piece of information is no longer used leaving them here commented out for completeness # ref_ant_per_map_ant_name_tuple = extract_holog_params["ref_ant_per_map_ant_name_tuple"] @@ -68,14 +71,14 @@ def process_extract_holog_chunk(extract_holog_params): scans = [int(scan) for scan in scans] if sel_state_ids: ctb = ctables.taql( - "select %s, SCAN_NUMBER, ANTENNA1, ANTENNA2, TIME, TIME_CENTROID, WEIGHT, FLAG_ROW, FLAG from $table_obj " - "WHERE DATA_DESC_ID == %s AND SCAN_NUMBER in %s AND STATE_ID in %s" + "select %s, SCAN_NUMBER, ANTENNA1, ANTENNA2, TIME, TIME_CENTROID, WEIGHT, FLAG_ROW, FLAG, FIELD_ID from " + "$table_obj WHERE DATA_DESC_ID == %s AND SCAN_NUMBER in %s AND STATE_ID in %s" % (data_column, ddi, scans, list(sel_state_ids)) ) else: ctb = ctables.taql( - "select %s, SCAN_NUMBER, ANTENNA1, ANTENNA2, TIME, TIME_CENTROID, WEIGHT, FLAG_ROW, FLAG from $table_obj " - "WHERE DATA_DESC_ID == %s AND SCAN_NUMBER in %s" % (data_column, ddi, scans) + "select %s, SCAN_NUMBER, ANTENNA1, ANTENNA2, TIME, TIME_CENTROID, WEIGHT, FLAG_ROW, FLAG, FIELD_ID from " + "$table_obj WHERE DATA_DESC_ID == %s AND SCAN_NUMBER in %s" % (data_column, ddi, scans) ) vis_data = ctb.getcol(data_column) weight = ctb.getcol("WEIGHT") @@ -87,6 +90,9 @@ def process_extract_holog_chunk(extract_holog_params): flag = ctb.getcol("FLAG") flag_row = ctb.getcol("FLAG_ROW") scan_list = ctb.getcol("SCAN_NUMBER") + field_ids = ctb.getcol("FIELD_ID") + + obs_info = _get_obs_summary(ms_name, field_ids) # Here we use the median of the differences between dumps as this is a good proxy for the integration time if time_interval is None: @@ -115,7 +121,7 @@ def process_extract_holog_chunk(extract_holog_params): scan_list, ) - del vis_data, weight, ant1, ant2, time_vis_row, flag, flag_row + del vis_data, weight, ant1, ant2, time_vis_row, flag, flag_row, field_ids map_ant_name_list = list(map(str, map_ant_name_tuple)) @@ -131,7 +137,7 @@ def process_extract_holog_chunk(extract_holog_params): for ant_index in vis_map_dict.keys(): antenna_name = "_".join(("ant", ant_names[ant_index])) n_pix, cell_size = calculate_optimal_grid_parameters( - pnt_map_dict, antenna_name, Telescope(telescope_name).diam, chan_freq, ddi + pnt_map_dict, antenna_name, Telescope(obs_info["telescope_name"]).diam, chan_freq, ddi ) grid_params[antenna_name] = {"n_pix": n_pix, "cell_size": cell_size} @@ -158,6 +164,7 @@ def process_extract_holog_chunk(extract_holog_params): ant_names, grid_params, time_interval, + obs_info ) logger.info( @@ -167,6 +174,39 @@ def process_extract_holog_chunk(extract_holog_params): ) +def _get_obs_summary(ms_name, fields_ids): + unq_ids = np.unique(fields_ids) + field_tbl = ctables.table( + ms_name + "::FIELD", + readonly=True, + lockoptions={"option": "usernoread"}, + ack=False, + ) + i_src = int(unq_ids[0]) + src_name = field_tbl.getcol("NAME") + phase_center_fk5 = field_tbl.getcol("PHASE_DIR")[:, 0, :] + field_tbl.close() + + obs_table = ctables.table( + ms_name + "::OBSERVATION", + readonly=True, + lockoptions={"option": "usernoread"}, + ack=False, + ) + time_range = casa_time_to_mjd(obs_table.getcol("TIME_RANGE")[0]) + telescope_name = obs_table.getcol("TELESCOPE_NAME")[0] + obs_table.close() + + obs_info = { + "source": src_name[i_src], + "FK5 phase center": phase_center_fk5[i_src].tolist(), + "telescope_name": telescope_name, + "start time": time_range[0], + "stop time": time_range[-1] + } + return obs_info + + @njit(cache=False, nogil=True) def _get_time_intervals(time_vis_row, scan_list, time_interval): unq_scans = np.unique(scan_list) @@ -368,12 +408,14 @@ def _create_holog_file( ant_names, grid_params, time_interval, + obs_info ): """Create holog-structured, formatted output file and save to zarr. Args: holog_name (str): holog file name. - vis_map_dict (dict): a nested dictionary/map of weighted visibilities indexed as [antenna][time, chan, pol]; mainains time ordering. + vis_map_dict (dict): a nested dictionary/map of weighted visibilities indexed as [antenna][time, chan, pol]; \ + mainains time ordering. weight_map_dict (dict): weights dictionary/map for visibilites in vis_map_dict pnt_map_dict (dict): pointing table map dictionary time_vis (numpy.ndarray): time_vis values @@ -387,9 +429,6 @@ def _create_holog_file( ctb = ctables.table("/".join((ms_name, "ANTENNA")), ack=False) observing_location = ctb.getcol("POSITION") - ctb = ctables.table("/".join((ms_name, "OBSERVATION")), ack=False) - telescope_name = ctb.getcol("TELESCOPE_NAME")[0] - ctb.close() for map_ant_index in vis_map_dict.keys(): @@ -445,7 +484,7 @@ def _create_holog_file( xds.attrs["holog_map_key"] = holog_map_key xds.attrs["ddi"] = ddi xds.attrs["parallactic_samples"] = parallactic_samples - xds.attrs["telescope_name"] = telescope_name + xds.attrs["observation_information"] = obs_info xds.attrs["antenna_name"] = ant_names[map_ant_index] xds.attrs["az_el_information"] = _get_az_el_characteristics(pnt_map_dict[map_ant_tag], valid_data) @@ -813,7 +852,7 @@ def create_holog_meta_data(holog_file, holog_dict, input_params): cell_sizes.append(xds.attrs["grid_params"]["cell_size"]) n_pixs.append(xds.attrs["grid_params"]["n_pix"]) - telescope_names.append(xds.attrs["telescope_name"]) + telescope_names.append(xds.attrs["observation_information"]["telescope_name"]) # cell_sizes_sigfigs = significant_figures_round(cell_sizes, digits=3) @@ -834,7 +873,8 @@ def create_holog_meta_data(holog_file, holog_dict, input_params): # meta_data["cell_size"] = \ # astrohack.utils.algorithms.calculate_suggested_grid_parameter(parameter=np.array(cell_sizes)) # - # logger.info("The suggested cell size is calculated to be: {cell_size}".format(cell_size=meta_data["cell_size"])) + # logger.info("The suggested cell size is calculated to be: {cell_size}".format(cell_size=meta_data["cell_size"] + # )) # # if not (len(set(n_pixs)) == 1): # logger.warning('Number of pixels not consistent: ' + str(n_pixs)) diff --git a/src/astrohack/extract_holog.py b/src/astrohack/extract_holog.py index 9231caed..33a86ed1 100644 --- a/src/astrohack/extract_holog.py +++ b/src/astrohack/extract_holog.py @@ -616,8 +616,6 @@ def extract_holog( pol_ctb.getcol("CORR_TYPE", startrow=pol_setup_id, nrow=1)[0, :] ] - extract_holog_params["telescope_name"] = obs_ctb.getcol("TELESCOPE_NAME")[0] - # Loop over all beam_scan_ids, a beam_scan_id can consist of more than one scan in a measurement set (this is # the case for the VLA pointed mosaics). for holog_map_key in holog_obs_dict[ddi_name].keys(): @@ -722,8 +720,6 @@ def extract_holog( file=extract_holog_params["holog_name"], dask_load=True, load_pnt_dict=False ) - extract_holog_params["telescope_name"] = telescope_name - meta_data = create_holog_meta_data( holog_file=extract_holog_params["holog_name"], holog_dict=holog_dict, From 94fd2ac38b2deffbe1511c3d669b17a1eb2fcf65 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 12:01:08 -0600 Subject: [PATCH 10/74] holog.py now propagates observation information --- src/astrohack/core/holog.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/astrohack/core/holog.py b/src/astrohack/core/holog.py index 2bb69afd..38969246 100644 --- a/src/astrohack/core/holog.py +++ b/src/astrohack/core/holog.py @@ -45,10 +45,12 @@ def process_holog_chunk(holog_chunk_params): ddi = holog_chunk_params["this_ddi"] convert_to_stokes = holog_chunk_params["to_stokes"] ref_xds = ant_data_dict[ddi]["map_0"] + az_el_info = ref_xds.attrs['az_el_information'] + obs_info = ref_xds.attrs['observation_information'] + telescope = _get_correct_telescope( - ref_xds.attrs["antenna_name"], meta_data["telescope_name"] + ref_xds.attrs["antenna_name"], obs_info["telescope_name"] ) - az_el_info = ref_xds.attrs['az_el_information'] try: is_near_field = ref_xds.attrs["near_field"] except KeyError: @@ -177,7 +179,6 @@ def process_holog_chunk(holog_chunk_params): aperture_resolution, holog_chunk_params["this_ant"], ant_data_dict[ddi]["map_0"].attrs["antenna_name"], - meta_data["telescope_name"], time_centroid, ddi, phase_fit_results, @@ -196,7 +197,8 @@ def process_holog_chunk(holog_chunk_params): zernike_rms, zernike_n_order, holog_chunk_params["image_name"], - az_el_info + az_el_info, + obs_info ) logger.info(f"Finished processing {label}") @@ -276,7 +278,6 @@ def _export_to_xds( aperture_resolution, ant_id, ant_name, - telescope_name, time_centroid, ddi, phase_fit_results, @@ -296,6 +297,7 @@ def _export_to_xds( zernike_n_order, image_name, az_el_info, + obs_info ): # Todo: Add Paralactic angle as a non-dimension coordinate dependant on time. xds = xr.Dataset() @@ -325,12 +327,12 @@ def _export_to_xds( xds.attrs["aperture_resolution"] = aperture_resolution xds.attrs["ant_id"] = ant_id xds.attrs["ant_name"] = ant_name - xds.attrs["telescope_name"] = telescope_name xds.attrs["time_centroid"] = np.array(time_centroid) xds.attrs["ddi"] = ddi xds.attrs["phase_fitting"] = phase_fit_results xds.attrs["zernike_N_order"] = zernike_n_order xds.attrs["az_el_information"] = az_el_info + xds.attrs["observation_information"] = obs_info coords = { "orig_pol": orig_pol_axis, From 09bf00df95b589c571410a4c88b19e5630665cfc Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 12:09:02 -0600 Subject: [PATCH 11/74] panel now propagates observation information --- src/astrohack/antenna/antenna_surface.py | 2 ++ src/astrohack/antenna/telescope.py | 11 ++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/astrohack/antenna/antenna_surface.py b/src/astrohack/antenna/antenna_surface.py index 9406f39f..5aaf6e5f 100644 --- a/src/astrohack/antenna/antenna_surface.py +++ b/src/astrohack/antenna/antenna_surface.py @@ -224,6 +224,7 @@ def _read_xds(self, inputxds): # Common elements self.az_el_info = inputxds.attrs['az_el_information'] + self.obs_info = inputxds.attrs['observation_information'] self.antenna_name = inputxds.attrs["ant_name"] self.ddi = inputxds.attrs["ddi"] self.label = create_dataset_label( @@ -922,6 +923,7 @@ def export_xds(self): xds.attrs["aperture_resolution"] = self.resolution xds.attrs["pol_state"] = self.pol_state xds.attrs['az_el_information'] = self.az_el_info + xds.attrs['observation_information'] = self.obs_info xds["AMPLITUDE"] = xr.DataArray(self.amplitude, dims=["u", "v"]) xds["PHASE"] = xr.DataArray(self.phase, dims=["u", "v"]) diff --git a/src/astrohack/antenna/telescope.py b/src/astrohack/antenna/telescope.py index e8ff5297..a550da99 100644 --- a/src/astrohack/antenna/telescope.py +++ b/src/astrohack/antenna/telescope.py @@ -51,20 +51,21 @@ def __init__(self, name: str, path=None): @classmethod def from_xds(cls, xds): - if xds.attrs["telescope_name"] == "ALMA": + tel_name = xds.attrs["observation_information"]["telescope_name"] + if tel_name == "ALMA": telescope_name = "_".join( - (xds.attrs["telescope_name"], xds.attrs["ant_name"][0:2]) + (tel_name, xds.attrs["ant_name"][0:2]) ) return cls(telescope_name) elif ( - xds.attrs["telescope_name"] == "EVLA" - or xds.attrs["telescope_name"] == "VLA" + tel_name == "EVLA" + or tel_name == "VLA" ): telescope_name = "VLA" return cls(telescope_name) else: raise ValueError( - "Unsupported telescope {0:s}".format(xds.attrs["telescope_name"]) + "Unsupported telescope {0:s}".format(tel_name) ) @staticmethod From 0f409f25ccdcd4a619ee2d4351ec9597632438fd Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 12:18:33 -0600 Subject: [PATCH 12/74] extract_holog.py now stores a summary of frequency information. --- src/astrohack/core/extract_holog.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index c191b24b..610fb6f1 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -487,6 +487,7 @@ def _create_holog_file( xds.attrs["observation_information"] = obs_info xds.attrs["antenna_name"] = ant_names[map_ant_index] xds.attrs["az_el_information"] = _get_az_el_characteristics(pnt_map_dict[map_ant_tag], valid_data) + xds.attrs["frequency_information"] = _get_freq_summary(chan) xds.attrs["l_max"] = np.max(xds["DIRECTIONAL_COSINES"][:, 0].values) xds.attrs["l_min"] = np.min(xds["DIRECTIONAL_COSINES"][:, 0].values) @@ -933,3 +934,15 @@ def _get_az_el_characteristics(pnt_map_xds, valid_data): 'mean': mean_az_el.tolist(), 'median': median_az_el.tolist()} return az_el_info + + +def _get_freq_summary(chan_axis): + chan_width = np.abs(chan_axis[1]-chan_axis[0]) + freq_info = { + "channel width": chan_width, + "number of channels": chan_axis.shape[0], + "range": [chan_axis[0]-chan_width/2, chan_axis[-1]+chan_width/2], + "representative": chan_axis[chan_axis.shape[0]//2] + } + + return freq_info From 7a5803a26f71b52e09fb048f6a28b6de4a87bac7 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 12:26:05 -0600 Subject: [PATCH 13/74] holog now propagates summary of frequency information. --- src/astrohack/core/holog.py | 10 +++++++--- src/astrohack/utils/gridding.py | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/astrohack/core/holog.py b/src/astrohack/core/holog.py index 38969246..cc21532c 100644 --- a/src/astrohack/core/holog.py +++ b/src/astrohack/core/holog.py @@ -56,7 +56,7 @@ def process_holog_chunk(holog_chunk_params): except KeyError: is_near_field = False - beam_grid, time_centroid, freq_axis, pol_axis, l_axis, m_axis, grid_corr = ( + beam_grid, time_centroid, freq_axis, pol_axis, l_axis, m_axis, grid_corr, freq_info = ( grid_beam( ant_ddi_dict=ant_data_dict[ddi], grid_size=holog_chunk_params["grid_size"], @@ -65,6 +65,7 @@ def process_holog_chunk(holog_chunk_params): chan_tol_fac=holog_chunk_params["chan_tolerance_factor"], telescope=telescope, grid_interpolation_mode=holog_chunk_params["grid_interpolation_mode"], + frequency_information=ref_xds.attrs["frequency_information"], label=label, ) ) @@ -198,7 +199,8 @@ def process_holog_chunk(holog_chunk_params): zernike_n_order, holog_chunk_params["image_name"], az_el_info, - obs_info + obs_info, + freq_info ) logger.info(f"Finished processing {label}") @@ -297,7 +299,8 @@ def _export_to_xds( zernike_n_order, image_name, az_el_info, - obs_info + obs_info, + freq_info ): # Todo: Add Paralactic angle as a non-dimension coordinate dependant on time. xds = xr.Dataset() @@ -333,6 +336,7 @@ def _export_to_xds( xds.attrs["zernike_N_order"] = zernike_n_order xds.attrs["az_el_information"] = az_el_info xds.attrs["observation_information"] = obs_info + xds.attrs["frequency_information"] = freq_info coords = { "orig_pol": orig_pol_axis, diff --git a/src/astrohack/utils/gridding.py b/src/astrohack/utils/gridding.py index b38018f3..2c49ee46 100644 --- a/src/astrohack/utils/gridding.py +++ b/src/astrohack/utils/gridding.py @@ -24,6 +24,7 @@ def grid_beam( chan_tol_fac, telescope, grid_interpolation_mode, + frequency_information, label, ): """ @@ -37,6 +38,7 @@ def grid_beam( chan_tol_fac: Frequency tolerance to chunk channels together telescope: Telescope object containing optical description of the telescope grid_interpolation_mode: linear, nearest, cubic or gaussian (convolution) + frequency_information: Dictionaty containing a summary of frequency information. label: label to be used in messages Returns: @@ -56,6 +58,8 @@ def grid_beam( n_chan = 1 avg_chan_map, avg_freq_axis = _create_average_chan_map(freq_axis, chan_tol_fac) output_freq_axis = [np.mean(avg_freq_axis)] + frequency_information["channel width"] *= frequency_information["number of channels"] + frequency_information["number of channels"] = 1 else: avg_chan_map = None avg_freq_axis = None @@ -133,6 +137,7 @@ def grid_beam( l_axis, m_axis, grid_corr, + frequency_information ) From 542a5880ba98caf99eac835b2919a3bfef89fde5 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 12:28:36 -0600 Subject: [PATCH 14/74] panel now propagates summary of frequency information. --- src/astrohack/antenna/antenna_surface.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/astrohack/antenna/antenna_surface.py b/src/astrohack/antenna/antenna_surface.py index 5aaf6e5f..e483f87f 100644 --- a/src/astrohack/antenna/antenna_surface.py +++ b/src/astrohack/antenna/antenna_surface.py @@ -226,6 +226,7 @@ def _read_xds(self, inputxds): self.az_el_info = inputxds.attrs['az_el_information'] self.obs_info = inputxds.attrs['observation_information'] self.antenna_name = inputxds.attrs["ant_name"] + self.freq_info = inputxds.attrs["frequency_information"] self.ddi = inputxds.attrs["ddi"] self.label = create_dataset_label( inputxds.attrs["ant_name"], inputxds.attrs["ddi"] @@ -924,6 +925,7 @@ def export_xds(self): xds.attrs["pol_state"] = self.pol_state xds.attrs['az_el_information'] = self.az_el_info xds.attrs['observation_information'] = self.obs_info + xds.attrs['frequency_information'] = self.freq_info xds["AMPLITUDE"] = xr.DataArray(self.amplitude, dims=["u", "v"]) xds["PHASE"] = xr.DataArray(self.phase, dims=["u", "v"]) From dd2a2a56a2634039d9e9fed0eaef54b7724ddbbc Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 12:49:34 -0600 Subject: [PATCH 15/74] Fixed right ascension negative values --- src/astrohack/core/extract_holog.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index 610fb6f1..832dc2e0 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -14,8 +14,9 @@ from astrohack.antenna import Telescope from astrohack.utils import create_dataset_label from astrohack.utils.imaging import calculate_parallactic_angle_chunk -from astrohack.utils.algorithms import calculate_optimal_grid_parameters +from astrohack.utils.algorithms import calculate_optimal_grid_parameters, phase_wrapping from astrohack.utils.conversion import casa_time_to_mjd +from astrohack.utils.constants import twopi from astrohack.utils.file import load_point_file @@ -197,6 +198,12 @@ def _get_obs_summary(ms_name, fields_ids): telescope_name = obs_table.getcol("TELESCOPE_NAME")[0] obs_table.close() + phase_center_fk5[:, 0] = np.where( + phase_center_fk5[:, 0] < 0, + phase_center_fk5[:, 0] + twopi, + phase_center_fk5[:, 0], + ) + obs_info = { "source": src_name[i_src], "FK5 phase center": phase_center_fk5[i_src].tolist(), From 0c5f144f7f4167921a7648d88624304983fde526 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 12:58:58 -0600 Subject: [PATCH 16/74] Added routine to format observation summary --- src/astrohack/utils/text.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/astrohack/utils/text.py b/src/astrohack/utils/text.py index f3b862eb..dd1901d3 100644 --- a/src/astrohack/utils/text.py +++ b/src/astrohack/utils/text.py @@ -3,7 +3,7 @@ import textwrap import numpy as np - +from astropy.time import Time from prettytable import PrettyTable from toolviper.utils import logger as logger @@ -676,7 +676,7 @@ def format_object_contents(obj): return outstr -def format_az_el_information(az_el_dict, key='center', unit='deg', precision='.2g'): +def format_az_el_information(az_el_dict, key='center', unit='deg', precision='.1f'): if key == 'center': prefix = '@ l,m = (0,0),' elif key in ['mean', 'median']: @@ -685,5 +685,28 @@ def format_az_el_information(az_el_dict, key='center', unit='deg', precision='.2 raise ValueError(f"Unrecognized key: {key}") az_el = np.array(az_el_dict[key])*convert_unit('rad', unit, 'trigonometric') - az_el_label = f'{prefix} Az, El = ({az_el[0]:{precision}}, {az_el[1]:{precision}}) [{unit}]' + prefix += ' Az, El' + az_el_label = f'{prefix:16s} = ({az_el[0]:{precision}}, {az_el[1]:{precision}}) [{unit}]' return az_el_label + + +def format_observation_information(obs_dict, az_el_dict, az_el_key='mean', phase_center_unit='radec', az_el_unit='deg', + time_format="%d %h %Y, %H:%M:%S", precision='.1f', tab=' '): + outstr = 'Observation Summary:\n' + for key, item in obs_dict.items(): + outstr += f'{tab}{key.capitalize().replace('_', ' '):16s} = ' + if 'FK5' in key: + if phase_center_unit == 'radec': + outstr += f'{rad_to_hour_str(item[0])} {rad_to_deg_str(item[1])} [FK5]' + else: + fac = convert_unit('rad', phase_center_unit, 'trigonometric') + outstr += f'({fac*item[0]:{precision}}, {fac*item[1]:{precision}}) [{phase_center_unit}]' + elif 'time' in key: + date = Time(item, format='mjd').to_datetime() + outstr += f'{date.strftime(time_format)} (UTC)' + else: + outstr += str(item) + outstr += '\n' + + outstr += f'{tab}{format_az_el_information(az_el_dict, az_el_key, unit=az_el_unit, precision=precision)}\n' + return outstr From 90fbba7862c2b69729d5d52feafeda2898280d6d Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 13:06:42 -0600 Subject: [PATCH 17/74] Added routine to format spectral summary --- src/astrohack/utils/text.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/astrohack/utils/text.py b/src/astrohack/utils/text.py index dd1901d3..5c30e0c0 100644 --- a/src/astrohack/utils/text.py +++ b/src/astrohack/utils/text.py @@ -710,3 +710,20 @@ def format_observation_information(obs_dict, az_el_dict, az_el_key='mean', phase outstr += f'{tab}{format_az_el_information(az_el_dict, az_el_key, unit=az_el_unit, precision=precision)}\n' return outstr + + +def format_spectral_information(freq_dict, tab=' '): + outstr = 'Spectral Summary:\n' + for key, item in freq_dict.items(): + outstr += f'{tab}{key.capitalize().replace('_', ' '):16s} = ' + if 'range' in key: + outstr += f'{format_frequency(item[0], decimal_places=3)} to {format_frequency(item[1], decimal_places=3)}' + elif 'number' in key: + outstr += f'{item}' + else: + outstr += format_frequency(item, decimal_places=3) + outstr += '\n' + + return outstr + + From 2480f1edf74f92176bf190ae24c72a84d1fb25b9 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Thu, 26 Jun 2025 13:14:40 -0600 Subject: [PATCH 18/74] cosmetic alignment. --- src/astrohack/utils/text.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/astrohack/utils/text.py b/src/astrohack/utils/text.py index 5c30e0c0..2c5327bf 100644 --- a/src/astrohack/utils/text.py +++ b/src/astrohack/utils/text.py @@ -686,7 +686,7 @@ def format_az_el_information(az_el_dict, key='center', unit='deg', precision='.1 az_el = np.array(az_el_dict[key])*convert_unit('rad', unit, 'trigonometric') prefix += ' Az, El' - az_el_label = f'{prefix:16s} = ({az_el[0]:{precision}}, {az_el[1]:{precision}}) [{unit}]' + az_el_label = f'{prefix:21s} = ({az_el[0]:{precision}}, {az_el[1]:{precision}}) [{unit}]' return az_el_label @@ -694,7 +694,7 @@ def format_observation_information(obs_dict, az_el_dict, az_el_key='mean', phase time_format="%d %h %Y, %H:%M:%S", precision='.1f', tab=' '): outstr = 'Observation Summary:\n' for key, item in obs_dict.items(): - outstr += f'{tab}{key.capitalize().replace('_', ' '):16s} = ' + outstr += f'{tab}{key.capitalize().replace('_', ' '):21s} = ' if 'FK5' in key: if phase_center_unit == 'radec': outstr += f'{rad_to_hour_str(item[0])} {rad_to_deg_str(item[1])} [FK5]' @@ -715,7 +715,7 @@ def format_observation_information(obs_dict, az_el_dict, az_el_key='mean', phase def format_spectral_information(freq_dict, tab=' '): outstr = 'Spectral Summary:\n' for key, item in freq_dict.items(): - outstr += f'{tab}{key.capitalize().replace('_', ' '):16s} = ' + outstr += f'{tab}{key.capitalize().replace('_', ' '):21s} = ' if 'range' in key: outstr += f'{format_frequency(item[0], decimal_places=3)} to {format_frequency(item[1], decimal_places=3)}' elif 'number' in key: From c1b9d9f3ba9196a51666395a76e6425e26d536ff Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 15:07:13 -0600 Subject: [PATCH 19/74] calculate_optimal_grid_parameters now outputs len 2 arrays rather than single values for both directions. --- src/astrohack/utils/algorithms.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/astrohack/utils/algorithms.py b/src/astrohack/utils/algorithms.py index 326ac546..55884034 100644 --- a/src/astrohack/utils/algorithms.py +++ b/src/astrohack/utils/algorithms.py @@ -92,8 +92,8 @@ def calc_coords(image_size, cell_size): """Calculate the center pixel of the image given a cell and image size Args: - image_size (float): image size - cell_size (float): cell size + image_size (np.array): image size + cell_size (np.array): cell size Returns: float, float: center pixel location in coordinates x, y @@ -347,22 +347,17 @@ def calculate_optimal_grid_parameters( # Since this is just an estimate for the situation where the user doesn't specify a values, I am picking # a values according to the developer heuristic, i.e. it seems to be good. cell_size = 0.85 * reference_lambda / telescope_diameter - + lm = pnt_map_dict[antenna_name].POINTING_OFFSET.values # Get data range - data_range = ( - pnt_map_dict[antenna_name].POINTING_OFFSET.values[:, 1].max() - - pnt_map_dict[antenna_name].POINTING_OFFSET.values[:, 1].min() - ) + data_range = np.array([lm[:, 0].max()-lm[:, 0].min(), lm[:, 1].max()-lm[:, 1].min()]) logger.info( f"{create_dataset_label(antenna_name, ddi)}: Cell size {format_angular_distance(cell_size)}, " - f"FOV: {format_angular_distance(data_range)}" + f"FOV: ({format_angular_distance(data_range[0])}, {format_angular_distance(data_range[1])})" ) - # logger.info(f"cell_size: {cell_size}") - # logger.info(f"data_range: {data_range}") try: - n_pix = int(np.ceil(data_range / cell_size)) ** 2 + n_pix = np.ceil(data_range / cell_size) except ZeroDivisionError: logger.error( @@ -371,7 +366,7 @@ def calculate_optimal_grid_parameters( ) raise ZeroDivisionError - return n_pix, cell_size + return n_pix.tolist(), cell_size.tolist() def compute_average_stokes_visibilities(vis, stokes): From f0fff58ace2c30e8fece8a7ad8fad515d5f03ad4 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 15:08:10 -0600 Subject: [PATCH 20/74] short term fix. --- src/astrohack/holog.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/astrohack/holog.py b/src/astrohack/holog.py index 68ebcc75..ee4ba2cf 100644 --- a/src/astrohack/holog.py +++ b/src/astrohack/holog.py @@ -216,8 +216,7 @@ def holog( return None else: - n_pix = int(np.sqrt(meta_data["n_pix"])) - holog_params["grid_size"] = np.array([n_pix, n_pix]) + holog_params["grid_size"] = np.array([meta_data["n_pix"], meta_data["n_pix"]], dtype=int) else: logger.debug("Using user specified grid size.", holog_params["grid_size"]) From 4cd8062a4848828c2e42dc802800415089ebfad4 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 15:08:55 -0600 Subject: [PATCH 21/74] implemented a more complete summary information dictionary. --- src/astrohack/core/extract_holog.py | 60 ++++++++++++++++++++++++++--- 1 file changed, 54 insertions(+), 6 deletions(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index 832dc2e0..3f6cc1b6 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -102,6 +102,8 @@ def process_extract_holog_chunk(extract_holog_params): ctb.close() table_obj.close() + map_ref_dict = _get_map_ref_dict(map_ant_tuple, ref_ant_per_map_ant_tuple, ant_names) + ( time_vis, vis_map_dict, @@ -165,7 +167,8 @@ def process_extract_holog_chunk(extract_holog_params): ant_names, grid_params, time_interval, - obs_info + obs_info, + map_ref_dict ) logger.info( @@ -175,8 +178,20 @@ def process_extract_holog_chunk(extract_holog_params): ) -def _get_obs_summary(ms_name, fields_ids): - unq_ids = np.unique(fields_ids) +def _get_map_ref_dict(map_ant_tuple, ref_ant_per_map_ant_tuple, ant_names): + map_dict = {} + for ii, map_id in enumerate(map_ant_tuple): + map_name = ant_names[map_id] + ref_list = [] + for ref_id in ref_ant_per_map_ant_tuple[ii]: + ref_list.append(ant_names[ref_id]) + map_dict[map_name] = ref_list + return map_dict + + + +def _get_obs_summary(ms_name, field_ids): + unq_ids = np.unique(field_ids) field_tbl = ctables.table( ms_name + "::FIELD", readonly=True, @@ -206,7 +221,7 @@ def _get_obs_summary(ms_name, fields_ids): obs_info = { "source": src_name[i_src], - "FK5 phase center": phase_center_fk5[i_src].tolist(), + "phase center": phase_center_fk5[i_src].tolist(), "telescope_name": telescope_name, "start time": time_range[0], "stop time": time_range[-1] @@ -415,7 +430,8 @@ def _create_holog_file( ant_names, grid_params, time_interval, - obs_info + obs_info, + map_ref_dict ): """Create holog-structured, formatted output file and save to zarr. @@ -435,7 +451,6 @@ def _create_holog_file( ctb = ctables.table("/".join((ms_name, "ANTENNA")), ack=False) observing_location = ctb.getcol("POSITION") - ctb.close() for map_ant_index in vis_map_dict.keys(): @@ -504,6 +519,10 @@ def _create_holog_file( xds.attrs["grid_params"] = grid_params[map_ant_tag] xds.attrs["time_smoothing_interval"] = time_interval + xds.attrs["summary"] = _crate_observation_summary(ant_names[map_ant_index], obs_info, grid_params, + xds["DIRECTIONAL_COSINES"].values, chan, + pnt_map_dict[map_ant_tag], valid_data, map_ref_dict) + holog_file = holog_name logger.debug( @@ -953,3 +972,32 @@ def _get_freq_summary(chan_axis): } return freq_info + + +def _crate_observation_summary(antenna_name, obs_info, grid_params, lm, chan_axis, pnt_map_xds, valid_data, + map_ref_dict): + spw_info = _get_freq_summary(chan_axis) + obs_info['az el info'] = _get_az_el_characteristics(pnt_map_xds, valid_data) + obs_info['reference antennas'] = map_ref_dict[antenna_name] + obs_info['antenna name'] = antenna_name + + l_max = np.max(lm[:, 0]) + l_min = np.min(lm[:, 0]) + m_max = np.max(lm[:, 1]) + m_min = np.min(lm[:, 1]) + + beam_info = { + 'grid size': grid_params[f'ant_{antenna_name}']['n_pix'], + 'cell size': grid_params[f'ant_{antenna_name}']['cell_size'], + 'l extent': [l_min, l_max], + 'm extent': [m_min, m_max], + } + + summary = { + "spectral": spw_info, + "beam": beam_info, + "general": obs_info, + "aperture": None + } + return summary + From a6f2c9789f09801f8c1d63357229709bbd77e565 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 15:09:19 -0600 Subject: [PATCH 22/74] small change caused by key name change --- src/astrohack/utils/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/astrohack/utils/text.py b/src/astrohack/utils/text.py index 2c5327bf..e98cadb5 100644 --- a/src/astrohack/utils/text.py +++ b/src/astrohack/utils/text.py @@ -695,7 +695,7 @@ def format_observation_information(obs_dict, az_el_dict, az_el_key='mean', phase outstr = 'Observation Summary:\n' for key, item in obs_dict.items(): outstr += f'{tab}{key.capitalize().replace('_', ' '):21s} = ' - if 'FK5' in key: + if 'phase center' in key: if phase_center_unit == 'radec': outstr += f'{rad_to_hour_str(item[0])} {rad_to_deg_str(item[1])} [FK5]' else: From c77a87b85788022e83bb07e30b112a14d98dc24b Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 15:23:47 -0600 Subject: [PATCH 23/74] Aggressively removing holog_mds meta data as it is heavily superseded by the new observation summary. --- src/astrohack/core/extract_holog.py | 96 ----------------------------- src/astrohack/extract_holog.py | 14 +---- src/astrohack/mds.py | 11 ---- 3 files changed, 1 insertion(+), 120 deletions(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index 3f6cc1b6..42a0842b 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -506,17 +506,6 @@ def _create_holog_file( xds.attrs["holog_map_key"] = holog_map_key xds.attrs["ddi"] = ddi xds.attrs["parallactic_samples"] = parallactic_samples - xds.attrs["observation_information"] = obs_info - xds.attrs["antenna_name"] = ant_names[map_ant_index] - xds.attrs["az_el_information"] = _get_az_el_characteristics(pnt_map_dict[map_ant_tag], valid_data) - xds.attrs["frequency_information"] = _get_freq_summary(chan) - - xds.attrs["l_max"] = np.max(xds["DIRECTIONAL_COSINES"][:, 0].values) - xds.attrs["l_min"] = np.min(xds["DIRECTIONAL_COSINES"][:, 0].values) - xds.attrs["m_max"] = np.max(xds["DIRECTIONAL_COSINES"][:, 1].values) - xds.attrs["m_min"] = np.min(xds["DIRECTIONAL_COSINES"][:, 1].values) - - xds.attrs["grid_params"] = grid_params[map_ant_tag] xds.attrs["time_smoothing_interval"] = time_interval xds.attrs["summary"] = _crate_observation_summary(ant_names[map_ant_index], obs_info, grid_params, @@ -849,91 +838,6 @@ def _interpolate_pointing(time_vis, pnt_time, dire, dir_cos, enc, pnt_off, tgt): return avg_dir, avg_dir_cos, avg_enc, avg_pnt_off, avg_tgt -def create_holog_meta_data(holog_file, holog_dict, input_params): - """Save holog file meta information to json file with the transformation - of the ordering (ddi, holog_map, ant) --> (ant, ddi, holog_map). - - Args: - input_params (): - holog_file (str): holog file name. - holog_dict (dict): Dictionary containing msdx data. - """ - - ant_holog_dict = {} - cell_sizes = [] - n_pixs = [] - telescope_names = [] - - for ddi, map_dict in holog_dict.items(): - if "ddi_" in ddi: - for mapping, ant_dict in map_dict.items(): - if "map_" in mapping: - for ant, xds in ant_dict.items(): - if "ant_" in ant: - if ant not in ant_holog_dict: - ant_holog_dict[ant] = {ddi: {mapping: {}}} - elif ddi not in ant_holog_dict[ant]: - ant_holog_dict[ant][ddi] = {mapping: {}} - - ant_holog_dict[ant][ddi][mapping] = xds.to_dict(data=False) - - cell_sizes.append(xds.attrs["grid_params"]["cell_size"]) - n_pixs.append(xds.attrs["grid_params"]["n_pix"]) - telescope_names.append(xds.attrs["observation_information"]["telescope_name"]) - - # cell_sizes_sigfigs = significant_figures_round(cell_sizes, digits=3) - - meta_data = { - "cell_size": np.min(cell_sizes), - "n_pix": np.max(n_pixs), - "telescope_name": telescope_names[0], - } - - # Commented out tests on grid_size and cell_size as they do not help us in catching problems since grid_size and - # cell_size should be free to vary between antennas and DDIs, e.g. DDIs at different frequencies and arrays with - # antennas of different sizes - - # if not (len(set(cell_sizes_sigfigs)) == 1): - # logger.warning('Cell size not consistent: ' + str(cell_sizes)) - # logger.warning('Calculating suggested cell size ...') - # - # meta_data["cell_size"] = \ - # astrohack.utils.algorithms.calculate_suggested_grid_parameter(parameter=np.array(cell_sizes)) - # - # logger.info("The suggested cell size is calculated to be: {cell_size}".format(cell_size=meta_data["cell_size"] - # )) - # - # if not (len(set(n_pixs)) == 1): - # logger.warning('Number of pixels not consistent: ' + str(n_pixs)) - # logger.warning('Calculating suggested number of pixels ...') - # - # meta_data['n_pix'] = int( - # astrohack.utils.algorithms.calculate_suggested_grid_parameter(parameter=np.array(n_pixs))) - # - # logger.info("The suggested number of pixels is calculated to be: {n_pix} (grid: {points} x {points})".format( - # n_pix=meta_data["n_pix"], points=int(np.sqrt(meta_data["n_pix"])) - # )) - - if not (len(set(telescope_names)) == 1): - logger.error("Telescope name not consistent: " + str(telescope_names)) - meta_data["telescope_name"] = None - - output_meta_file = "{name}/{ext}".format(name=holog_file, ext=".holog_json") - - try: - with open(output_meta_file, "w") as json_file: - json.dump(ant_holog_dict, json_file) - - except Exception as error: - logger.error(f"{error}") - - raise Exception(error) - - meta_data.update(input_params) - - return meta_data - - @njit(cache=False, nogil=True) def _get_time_index(data_time, i_time, time_axis, half_int): if i_time == time_axis.shape[0]: diff --git a/src/astrohack/extract_holog.py b/src/astrohack/extract_holog.py index 33a86ed1..af7924b6 100644 --- a/src/astrohack/extract_holog.py +++ b/src/astrohack/extract_holog.py @@ -9,13 +9,13 @@ import toolviper.utils.parameter import dask + import astrohack import psutil import numpy as np import toolviper.utils.logger as logger -from astropy.time import Time from casacore import tables as ctables from rich.console import Console from rich.table import Table @@ -26,7 +26,6 @@ from astrohack.utils.file import load_holog_file from astrohack.utils.file import load_point_file from astrohack.utils.data import write_meta_data -from astrohack.core.extract_holog import create_holog_meta_data from astrohack.core.extract_holog import create_holog_obs_dict from astrohack.core.extract_holog import process_extract_holog_chunk from astrohack.utils.tools import get_valid_state_ids @@ -720,17 +719,6 @@ def extract_holog( file=extract_holog_params["holog_name"], dask_load=True, load_pnt_dict=False ) - meta_data = create_holog_meta_data( - holog_file=extract_holog_params["holog_name"], - holog_dict=holog_dict, - input_params=extract_holog_params.copy(), - ) - - holog_attr_file = "{name}/{ext}".format( - name=extract_holog_params["holog_name"], ext=".holog_attr" - ) - write_meta_data(holog_attr_file, meta_data) - holog_attr_file = "{name}/{ext}".format( name=extract_holog_params["holog_name"], ext=".holog_input" ) diff --git a/src/astrohack/mds.py b/src/astrohack/mds.py index acc66c22..20e4e0e6 100644 --- a/src/astrohack/mds.py +++ b/src/astrohack/mds.py @@ -572,7 +572,6 @@ def open(self, file: str = None, dask_load: bool = True) -> bool: load_holog_file( file=file, dask_load=dask_load, load_pnt_dict=False, holog_dict=self ) - self._meta_data = read_meta_data(file + "/.holog_attr") self._input_pars = read_meta_data(file + "/.holog_input") self._file_is_open = True @@ -621,16 +620,6 @@ def select(self, ddi: int, map_id: int, ant: str) -> object: else: return self[ddi][map_id][ant] - @property - def meta_data(self): - """Retrieve AstrohackHologFile JSON metadata. - - :return: JSON metadata for this AstrohackHologFile object - :rtype: dict - """ - - return self._meta_data - @toolviper.utils.parameter.validate(custom_checker=custom_plots_checker) def plot_diagnostics( self, From a7c6275033c671b4f8d1aadfabfe20a200aea6fb Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 15:34:41 -0600 Subject: [PATCH 24/74] improved message --- src/astrohack/utils/algorithms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/astrohack/utils/algorithms.py b/src/astrohack/utils/algorithms.py index 55884034..2ae98398 100644 --- a/src/astrohack/utils/algorithms.py +++ b/src/astrohack/utils/algorithms.py @@ -352,7 +352,7 @@ def calculate_optimal_grid_parameters( data_range = np.array([lm[:, 0].max()-lm[:, 0].min(), lm[:, 1].max()-lm[:, 1].min()]) logger.info( - f"{create_dataset_label(antenna_name, ddi)}: Cell size {format_angular_distance(cell_size)}, " + f"{create_dataset_label(antenna_name, ddi)}: Suggested cell size {format_angular_distance(cell_size)}, " f"FOV: ({format_angular_distance(data_range[0])}, {format_angular_distance(data_range[1])})" ) From 0d69f60d175e9c36d2b57778dc38bb5002b7b0ad Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 15:35:19 -0600 Subject: [PATCH 25/74] Brought back writting of holog_json. --- src/astrohack/core/extract_holog.py | 37 +++++++++++++++++++++++++++++ src/astrohack/extract_holog.py | 4 +++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index 42a0842b..47688a73 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -905,3 +905,40 @@ def _crate_observation_summary(antenna_name, obs_info, grid_params, lm, chan_axi } return summary + +def create_holog_json(holog_file, holog_dict): + """Save holog file meta information to json file with the transformation + of the ordering (ddi, holog_map, ant) --> (ant, ddi, holog_map). + + Args: + input_params (): + holog_file (str): holog file name. + holog_dict (dict): Dictionary containing msdx data. + """ + + ant_holog_dict = {} + + for ddi, map_dict in holog_dict.items(): + if "ddi_" in ddi: + for mapping, ant_dict in map_dict.items(): + if "map_" in mapping: + for ant, xds in ant_dict.items(): + if "ant_" in ant: + if ant not in ant_holog_dict: + ant_holog_dict[ant] = {ddi: {mapping: {}}} + elif ddi not in ant_holog_dict[ant]: + ant_holog_dict[ant][ddi] = {mapping: {}} + + ant_holog_dict[ant][ddi][mapping] = xds.to_dict(data=False) + + output_meta_file = "{name}/{ext}".format(name=holog_file, ext=".holog_json") + + try: + with open(output_meta_file, "w") as json_file: + json.dump(ant_holog_dict, json_file) + + except Exception as error: + logger.error(f"{error}") + + raise Exception(error) + diff --git a/src/astrohack/extract_holog.py b/src/astrohack/extract_holog.py index af7924b6..b9fbc4da 100644 --- a/src/astrohack/extract_holog.py +++ b/src/astrohack/extract_holog.py @@ -26,7 +26,7 @@ from astrohack.utils.file import load_holog_file from astrohack.utils.file import load_point_file from astrohack.utils.data import write_meta_data -from astrohack.core.extract_holog import create_holog_obs_dict +from astrohack.core.extract_holog import create_holog_obs_dict, create_holog_json from astrohack.core.extract_holog import process_extract_holog_chunk from astrohack.utils.tools import get_valid_state_ids from astrohack.utils.text import get_default_file_name @@ -719,6 +719,8 @@ def extract_holog( file=extract_holog_params["holog_name"], dask_load=True, load_pnt_dict=False ) + create_holog_json(extract_holog_params["holog_name"], holog_dict) + holog_attr_file = "{name}/{ext}".format( name=extract_holog_params["holog_name"], ext=".holog_input" ) From 9e1bd2114b46e2c8929a9b9a68cf2a774fcd2dbf Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 15:42:34 -0600 Subject: [PATCH 26/74] some reorganization. --- src/astrohack/core/extract_holog.py | 79 ++++++++++++++--------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index 47688a73..7801dd23 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -189,46 +189,6 @@ def _get_map_ref_dict(map_ant_tuple, ref_ant_per_map_ant_tuple, ant_names): return map_dict - -def _get_obs_summary(ms_name, field_ids): - unq_ids = np.unique(field_ids) - field_tbl = ctables.table( - ms_name + "::FIELD", - readonly=True, - lockoptions={"option": "usernoread"}, - ack=False, - ) - i_src = int(unq_ids[0]) - src_name = field_tbl.getcol("NAME") - phase_center_fk5 = field_tbl.getcol("PHASE_DIR")[:, 0, :] - field_tbl.close() - - obs_table = ctables.table( - ms_name + "::OBSERVATION", - readonly=True, - lockoptions={"option": "usernoread"}, - ack=False, - ) - time_range = casa_time_to_mjd(obs_table.getcol("TIME_RANGE")[0]) - telescope_name = obs_table.getcol("TELESCOPE_NAME")[0] - obs_table.close() - - phase_center_fk5[:, 0] = np.where( - phase_center_fk5[:, 0] < 0, - phase_center_fk5[:, 0] + twopi, - phase_center_fk5[:, 0], - ) - - obs_info = { - "source": src_name[i_src], - "phase center": phase_center_fk5[i_src].tolist(), - "telescope_name": telescope_name, - "start time": time_range[0], - "stop time": time_range[-1] - } - return obs_info - - @njit(cache=False, nogil=True) def _get_time_intervals(time_vis_row, scan_list, time_interval): unq_scans = np.unique(scan_list) @@ -849,6 +809,45 @@ def _get_time_index(data_time, i_time, time_axis, half_int): return i_time +def _get_obs_summary(ms_name, field_ids): + unq_ids = np.unique(field_ids) + field_tbl = ctables.table( + ms_name + "::FIELD", + readonly=True, + lockoptions={"option": "usernoread"}, + ack=False, + ) + i_src = int(unq_ids[0]) + src_name = field_tbl.getcol("NAME") + phase_center_fk5 = field_tbl.getcol("PHASE_DIR")[:, 0, :] + field_tbl.close() + + obs_table = ctables.table( + ms_name + "::OBSERVATION", + readonly=True, + lockoptions={"option": "usernoread"}, + ack=False, + ) + time_range = casa_time_to_mjd(obs_table.getcol("TIME_RANGE")[0]) + telescope_name = obs_table.getcol("TELESCOPE_NAME")[0] + obs_table.close() + + phase_center_fk5[:, 0] = np.where( + phase_center_fk5[:, 0] < 0, + phase_center_fk5[:, 0] + twopi, + phase_center_fk5[:, 0], + ) + + obs_info = { + "source": src_name[i_src], + "phase center": phase_center_fk5[i_src].tolist(), + "telescope name": telescope_name, + "start time": time_range[0], + "stop time": time_range[-1] + } + return obs_info + + def _get_az_el_characteristics(pnt_map_xds, valid_data): az_el = pnt_map_xds["ENCODER"].values[valid_data, ...] lm = pnt_map_xds["DIRECTIONAL_COSINES"].values[valid_data, ...] From 1b6e8bf023e7434915534192c2b963031687d25c Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 15:57:43 -0600 Subject: [PATCH 27/74] Corrected a typo --- src/astrohack/core/extract_holog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index 7801dd23..ea242d07 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -140,7 +140,7 @@ def process_extract_holog_chunk(extract_holog_params): for ant_index in vis_map_dict.keys(): antenna_name = "_".join(("ant", ant_names[ant_index])) n_pix, cell_size = calculate_optimal_grid_parameters( - pnt_map_dict, antenna_name, Telescope(obs_info["telescope_name"]).diam, chan_freq, ddi + pnt_map_dict, antenna_name, Telescope(obs_info["telescope name"]).diam, chan_freq, ddi ) grid_params[antenna_name] = {"n_pix": n_pix, "cell_size": cell_size} From 5930210ede773328f27df39463c11c3f8f264543 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 16:17:33 -0600 Subject: [PATCH 28/74] holog now uses the data in the summary and passes it along. --- src/astrohack/core/holog.py | 84 +++++++++++++++++++++++-------------- src/astrohack/holog.py | 60 +------------------------- 2 files changed, 54 insertions(+), 90 deletions(-) diff --git a/src/astrohack/core/holog.py b/src/astrohack/core/holog.py index cc21532c..23545579 100644 --- a/src/astrohack/core/holog.py +++ b/src/astrohack/core/holog.py @@ -2,11 +2,11 @@ import xarray as xr from astrohack.antenna.telescope import Telescope +from astrohack.utils import format_angular_distance from astrohack.utils.text import create_dataset_label from astrohack.utils.conversion import convert_5d_grid_to_stokes from astrohack.utils.algorithms import phase_wrapping from astrohack.utils.zernike_aperture_fitting import fit_zernike_coefficients -from astrohack.utils.data import read_meta_data from astrohack.utils.file import load_holog_file from astrohack.utils.imaging import ( calculate_far_field_aperture, @@ -38,34 +38,58 @@ def process_holog_chunk(holog_chunk_params): ddi_id=holog_chunk_params["this_ddi"], ) label = create_dataset_label( - holog_chunk_params["this_ant"], holog_chunk_params["this_ddi"] + holog_chunk_params["this_ant"], holog_chunk_params["this_ddi"], separator=',' ) logger.info(f"Processing {label}") - meta_data = read_meta_data(holog_chunk_params["holog_name"] + "/.holog_attr") ddi = holog_chunk_params["this_ddi"] convert_to_stokes = holog_chunk_params["to_stokes"] ref_xds = ant_data_dict[ddi]["map_0"] - az_el_info = ref_xds.attrs['az_el_information'] - obs_info = ref_xds.attrs['observation_information'] + summary = ref_xds.attrs['summary'] + + user_grid_size = holog_chunk_params["grid_size"] + + if user_grid_size is None: + grid_size = np.array(summary['beam']['grid size']) + elif isinstance(user_grid_size, int): + grid_size = np.array([user_grid_size, user_grid_size]) + elif isinstance(user_grid_size, (list, np.ndarray)): + grid_size = user_grid_size + else: + raise Exception(f"Don't know what due with grid size of type {type(user_grid_size)}") + + logger.info(f'{label}: Using a grid of {int(grid_size[0])} by {int(grid_size[1])} pixels for the beam') + + user_cell_size = holog_chunk_params["cell_size"] + if user_cell_size is None: + cell_size = np.array([summary['beam']['cell size'], summary['beam']['cell size']]) + elif isinstance(user_cell_size, (int, float)): + cell_size = np.array([user_cell_size, user_cell_size]) + elif isinstance(user_cell_size, (list, np.ndarray)): + cell_size = user_cell_size + else: + raise Exception(f"Don't know what due with cell size of type {type(user_cell_size)}") + + logger.info(f'{label}: Using a cell size of {format_angular_distance(cell_size[0])} by ' + f'{format_angular_distance(cell_size[1])} for the beam') telescope = _get_correct_telescope( - ref_xds.attrs["antenna_name"], obs_info["telescope_name"] + summary["general"]["antenna name"], summary["general"]["telescope name"] ) try: is_near_field = ref_xds.attrs["near_field"] except KeyError: is_near_field = False - beam_grid, time_centroid, freq_axis, pol_axis, l_axis, m_axis, grid_corr, freq_info = ( + beam_grid, time_centroid, freq_axis, pol_axis, l_axis, m_axis, grid_corr, summary["spectral"] = ( grid_beam( ant_ddi_dict=ant_data_dict[ddi], - grid_size=holog_chunk_params["grid_size"], - sky_cell_size=holog_chunk_params["cell_size"], + grid_size=grid_size, + sky_cell_size=cell_size, avg_chan=holog_chunk_params["chan_average"], chan_tol_fac=holog_chunk_params["chan_tolerance_factor"], telescope=telescope, grid_interpolation_mode=holog_chunk_params["grid_interpolation_mode"], - frequency_information=ref_xds.attrs["frequency_information"], + frequency_information=summary["spectral"], label=label, ) ) @@ -103,7 +127,7 @@ def process_holog_chunk(holog_chunk_params): padding_factor=holog_chunk_params["padding_factor"], freq=freq_axis, telescope=telescope, - sky_cell_size=holog_chunk_params["cell_size"], + sky_cell_size=cell_size, apply_grid_correction=grid_corr, label=label, ) @@ -170,16 +194,15 @@ def process_holog_chunk(holog_chunk_params): logger.error(f"Unsupported phase fitting engine: {phase_fit_engine}") raise ValueError - aperture_resolution = _compute_aperture_resolution(l_axis, m_axis, used_wavelength) + summary['aperture'] = _get_aperture_summary(u_axis, v_axis, + _compute_aperture_resolution(l_axis, m_axis, used_wavelength)) _export_to_xds( beam_grid, aperture_grid, amplitude, phase_corrected_angle, - aperture_resolution, holog_chunk_params["this_ant"], - ant_data_dict[ddi]["map_0"].attrs["antenna_name"], time_centroid, ddi, phase_fit_results, @@ -198,9 +221,7 @@ def process_holog_chunk(holog_chunk_params): zernike_rms, zernike_n_order, holog_chunk_params["image_name"], - az_el_info, - obs_info, - freq_info + summary ) logger.info(f"Finished processing {label}") @@ -249,16 +270,16 @@ def _crop_and_split_aperture(aperture_grid, u_axis, v_axis, telescope, scaling=1 end_cut = center_pixel + radius amplitude = np.absolute( - aperture_grid[..., start_cut[0] : end_cut[0], start_cut[1] : end_cut[1]] + aperture_grid[..., start_cut[0]: end_cut[0], start_cut[1]: end_cut[1]] ) phase = np.angle( - aperture_grid[..., start_cut[0] : end_cut[0], start_cut[1] : end_cut[1]] + aperture_grid[..., start_cut[0]: end_cut[0], start_cut[1]: end_cut[1]] ) return ( amplitude, phase, - u_axis[start_cut[0] : end_cut[0]], - v_axis[start_cut[1] : end_cut[1]], + u_axis[start_cut[0]: end_cut[0]], + v_axis[start_cut[1]: end_cut[1]], ) @@ -277,9 +298,7 @@ def _export_to_xds( aperture_grid, amplitude, phase_corrected_angle, - aperture_resolution, ant_id, - ant_name, time_centroid, ddi, phase_fit_results, @@ -298,9 +317,7 @@ def _export_to_xds( zernike_rms, zernike_n_order, image_name, - az_el_info, - obs_info, - freq_info + summary ): # Todo: Add Paralactic angle as a non-dimension coordinate dependant on time. xds = xr.Dataset() @@ -327,16 +344,12 @@ def _export_to_xds( zernike_rms, dims=["time", "chan", "orig_pol"] ) - xds.attrs["aperture_resolution"] = aperture_resolution xds.attrs["ant_id"] = ant_id - xds.attrs["ant_name"] = ant_name xds.attrs["time_centroid"] = np.array(time_centroid) xds.attrs["ddi"] = ddi xds.attrs["phase_fitting"] = phase_fit_results xds.attrs["zernike_N_order"] = zernike_n_order - xds.attrs["az_el_information"] = az_el_info - xds.attrs["observation_information"] = obs_info - xds.attrs["frequency_information"] = freq_info + xds.attrs["summary"] = summary coords = { "orig_pol": orig_pol_axis, @@ -354,3 +367,12 @@ def _export_to_xds( xds.to_zarr( f"{image_name}/{ant_id}/{ddi}", mode="w", compute=True, consolidated=True ) + + +def _get_aperture_summary(u_axis, v_axis, aperture_resolution): + aperture_dict = { + 'grid size': [u_axis.shape[0], v_axis.shape[0]], + 'cell size': [u_axis[1]-u_axis[0], v_axis[1]-v_axis[0]], + 'resolution': aperture_resolution.tolist() + } + return aperture_dict diff --git a/src/astrohack/holog.py b/src/astrohack/holog.py index ee4ba2cf..d31b33d7 100644 --- a/src/astrohack/holog.py +++ b/src/astrohack/holog.py @@ -10,7 +10,6 @@ from astrohack.utils.graph import compute_graph from astrohack.utils.file import overwrite_file -from astrohack.utils.data import read_meta_data from astrohack.utils.data import write_meta_data from astrohack.core.holog import process_holog_chunk from astrohack.utils.text import get_default_file_name @@ -24,7 +23,7 @@ def holog( holog_name: str, grid_size: Union[int, Array, List] = None, - cell_size: Union[int, Array, List] = None, + cell_size: Union[float, Array, List] = None, image_name: str = None, padding_factor: int = 10, grid_interpolation_mode: str = "gaussian", @@ -181,63 +180,6 @@ def holog( with open(json_data, "r") as json_file: holog_json = json.load(json_file) - meta_data = read_meta_data(holog_params["holog_name"] + "/.holog_attr") - - # If cell size is None, fill from metadata if it exists - if holog_params["cell_size"] is None: - if meta_data["cell_size"] is None: - logger.error( - "Cell size meta data not found. There was likely an issue with the holography data extraction. Fix\ - extract data or provide cell_size as argument." - ) - logger.error("There was an error, see log above for more info.") - - return None - - else: - holog_params["cell_size"] = np.array( - [-meta_data["cell_size"], meta_data["cell_size"]] - ) - - else: - holog_params["cell_size"] = _convert_gridding_parameter( - gridding_parameter=holog_params["cell_size"], reflect_on_axis=True - ) - - # If grid size is None, create it from n_pix. - if holog_params["grid_size"] is None: - if meta_data["n_pix"] is None: - logger.error( - "Grid size meta data not found. There was likely an issue with the holography data extraction. Fix \ - extract data or provide grid_size as argument." - ) - logger.error("There was an error, see log above for more info.") - - return None - - else: - holog_params["grid_size"] = np.array([meta_data["n_pix"], meta_data["n_pix"]], dtype=int) - - else: - logger.debug("Using user specified grid size.", holog_params["grid_size"]) - holog_params["grid_size"] = _convert_gridding_parameter( - gridding_parameter=holog_params["grid_size"], reflect_on_axis=False - ) - - logger.info( - "Cell size: {cell_size}, Grid size {grid_size}".format( - cell_size=holog_params["cell_size"], grid_size=holog_params["grid_size"] - ) - ) - - json_data = { - "cell_size": holog_params["cell_size"].tolist(), - "grid_size": holog_params["grid_size"].tolist(), - } - - with open(".holog_diagnostic.json", "w") as out_file: - json.dump(json_data, out_file) - if compute_graph( holog_json, process_holog_chunk, holog_params, ["ant", "ddi"], parallel=parallel ): From f69d14db8c7db15a33309d198a874869bad7760e Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 16:29:03 -0600 Subject: [PATCH 29/74] grid size is now a list of integers from the get go --- src/astrohack/core/holog.py | 2 +- src/astrohack/utils/algorithms.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/astrohack/core/holog.py b/src/astrohack/core/holog.py index 23545579..404bb967 100644 --- a/src/astrohack/core/holog.py +++ b/src/astrohack/core/holog.py @@ -57,7 +57,7 @@ def process_holog_chunk(holog_chunk_params): else: raise Exception(f"Don't know what due with grid size of type {type(user_grid_size)}") - logger.info(f'{label}: Using a grid of {int(grid_size[0])} by {int(grid_size[1])} pixels for the beam') + logger.info(f'{label}: Using a grid of {grid_size[0]} by {grid_size[1]} pixels for the beam') user_cell_size = holog_chunk_params["cell_size"] if user_cell_size is None: diff --git a/src/astrohack/utils/algorithms.py b/src/astrohack/utils/algorithms.py index 2ae98398..3652a7f5 100644 --- a/src/astrohack/utils/algorithms.py +++ b/src/astrohack/utils/algorithms.py @@ -366,7 +366,7 @@ def calculate_optimal_grid_parameters( ) raise ZeroDivisionError - return n_pix.tolist(), cell_size.tolist() + return [int(n_pix[0]), int(n_pix[1])], cell_size.tolist() def compute_average_stokes_visibilities(vis, stokes): From 3ae695e6ff986cd87e9fe893c74deb05f1029fe5 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 16:30:05 -0600 Subject: [PATCH 30/74] panel now uses the summary functionality. --- src/astrohack/antenna/antenna_surface.py | 44 ++++-------------------- src/astrohack/antenna/telescope.py | 2 +- 2 files changed, 7 insertions(+), 39 deletions(-) diff --git a/src/astrohack/antenna/antenna_surface.py b/src/astrohack/antenna/antenna_surface.py index e483f87f..dd03a72d 100644 --- a/src/astrohack/antenna/antenna_surface.py +++ b/src/astrohack/antenna/antenna_surface.py @@ -140,16 +140,6 @@ def _read_holog_xds(self, inputxds): self.v_axis = inputxds.v_prime.values self.computephase = False - try: - self.resolution = inputxds.attrs["aperture_resolution"] - except KeyError: - - logger.warning("holog image does not have resolution information") - logger.warning( - "Rerun holog with astrohack v>0.1.5 for aperture resolution information" - ) - self.resolution = None - def _read_panel_xds(self, inputxds): self.wavelength = inputxds.attrs["wavelength"] self.amp_unit = inputxds.attrs["amp_unit"] @@ -171,24 +161,7 @@ def _read_panel_xds(self, inputxds): self.u_axis = inputxds.u.values self.v_axis = inputxds.u.values self.panel_distribution = inputxds["PANEL_DISTRIBUTION"].values - try: - self.amplitude_noise = inputxds["AMP_NOISE"].values - except KeyError: - logger.warning( - "Input panel file does not have amplitude noise information, noise statistics will be " - "flawed" - ) - self.amplitude_noise = np.full_like(self.amplitude, np.nan) - - try: - self.resolution = inputxds.attrs["aperture_resolution"] - except KeyError: - - logger.warning("Input panel file does not have resolution information") - logger.warning( - "Rerun holog with astrohack v>0.1.5 for aperture resolution information" - ) - self.resolution = None + self.amplitude_noise = inputxds["AMP_NOISE"].values if self.solved: self.panel_fallback = inputxds["PANEL_FALLBACK"].values @@ -223,13 +196,12 @@ def _read_xds(self, inputxds): self._read_holog_xds(inputxds) # Common elements - self.az_el_info = inputxds.attrs['az_el_information'] - self.obs_info = inputxds.attrs['observation_information'] - self.antenna_name = inputxds.attrs["ant_name"] - self.freq_info = inputxds.attrs["frequency_information"] + self.summary = inputxds.attrs['summary'] + self.antenna_name = inputxds.attrs["summary"]["general"]["antenna name"] + self.resolution = inputxds.summary["aperture"]["resolution"] self.ddi = inputxds.attrs["ddi"] self.label = create_dataset_label( - inputxds.attrs["ant_name"], inputxds.attrs["ddi"] + self.antenna_name, inputxds.attrs["ddi"] ) def _define_amp_clip(self, clip_type, clip_level): @@ -911,8 +883,6 @@ def export_xds(self): xds = xr.Dataset() gains = self.gains() rms = self.get_rms(unit="m") - xds.attrs["telescope_name"] = self.telescope.name - xds.attrs["ant_name"] = self.antenna_name xds.attrs["ddi"] = self.ddi xds.attrs["wavelength"] = self.wavelength xds.attrs["amp_unit"] = self.amp_unit @@ -923,9 +893,7 @@ def export_xds(self): xds.attrs["fitted"] = self.fitted xds.attrs["aperture_resolution"] = self.resolution xds.attrs["pol_state"] = self.pol_state - xds.attrs['az_el_information'] = self.az_el_info - xds.attrs['observation_information'] = self.obs_info - xds.attrs['frequency_information'] = self.freq_info + xds.attrs['summary'] = self.summary xds["AMPLITUDE"] = xr.DataArray(self.amplitude, dims=["u", "v"]) xds["PHASE"] = xr.DataArray(self.phase, dims=["u", "v"]) diff --git a/src/astrohack/antenna/telescope.py b/src/astrohack/antenna/telescope.py index a550da99..cc1285a7 100644 --- a/src/astrohack/antenna/telescope.py +++ b/src/astrohack/antenna/telescope.py @@ -51,7 +51,7 @@ def __init__(self, name: str, path=None): @classmethod def from_xds(cls, xds): - tel_name = xds.attrs["observation_information"]["telescope_name"] + tel_name = xds.attrs["summary"]["general"]["telescope name"] if tel_name == "ALMA": telescope_name = "_".join( (tel_name, xds.attrs["ant_name"][0:2]) From 5e9270a00c6b24182c9ab1c2ad1363330ba31710 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 17:01:21 -0600 Subject: [PATCH 31/74] added formating of beam information --- src/astrohack/utils/text.py | 69 ++++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/src/astrohack/utils/text.py b/src/astrohack/utils/text.py index e98cadb5..096c5403 100644 --- a/src/astrohack/utils/text.py +++ b/src/astrohack/utils/text.py @@ -457,8 +457,9 @@ def format_wavelength(wave_value, unit="m", decimal_places=2): return format_value_unit(fac * wave_value, unitout, decimal_places) -def format_angular_distance(dist_value, unit="rad", decimal_places=2): +def format_angular_distance(user_value, unit="rad", decimal_places=2): one_deg = np.pi / 180 + dist_value = np.abs(user_value) if dist_value >= np.pi / 180: unitout = "deg" elif dist_value >= one_deg / 60: @@ -470,7 +471,7 @@ def format_angular_distance(dist_value, unit="rad", decimal_places=2): else: unitout = "uasec" fac = convert_unit(unit, unitout, "trigonometric") - return format_value_unit(fac * dist_value, unitout, decimal_places) + return format_value_unit(fac * user_value, unitout, decimal_places) def format_label(label, separators=("_", "\n"), new_separator=" "): @@ -686,36 +687,39 @@ def format_az_el_information(az_el_dict, key='center', unit='deg', precision='.1 az_el = np.array(az_el_dict[key])*convert_unit('rad', unit, 'trigonometric') prefix += ' Az, El' - az_el_label = f'{prefix:21s} = ({az_el[0]:{precision}}, {az_el[1]:{precision}}) [{unit}]' + az_el_label = f'{prefix} = ({az_el[0]:{precision}}, {az_el[1]:{precision}}) [{unit}]' return az_el_label -def format_observation_information(obs_dict, az_el_dict, az_el_key='mean', phase_center_unit='radec', az_el_unit='deg', - time_format="%d %h %Y, %H:%M:%S", precision='.1f', tab=' '): - outstr = 'Observation Summary:\n' +def format_observation_information(obs_dict, tab, ident, key_size, az_el_key='mean', phase_center_unit='radec', az_el_unit='deg', + time_format="%d %h %Y, %H:%M:%S", precision='.1f'): + outstr = f'{ident}General:\n' + tab = tab+ident for key, item in obs_dict.items(): - outstr += f'{tab}{key.capitalize().replace('_', ' '):21s} = ' + line = f'{tab}{key.capitalize().replace('_', ' '):{key_size}s} => ' if 'phase center' in key: if phase_center_unit == 'radec': - outstr += f'{rad_to_hour_str(item[0])} {rad_to_deg_str(item[1])} [FK5]' + line += f'{rad_to_hour_str(item[0])} {rad_to_deg_str(item[1])} [FK5]' else: fac = convert_unit('rad', phase_center_unit, 'trigonometric') - outstr += f'({fac*item[0]:{precision}}, {fac*item[1]:{precision}}) [{phase_center_unit}]' + line += f'({fac*item[0]:{precision}}, {fac*item[1]:{precision}}) [{phase_center_unit}]' elif 'time' in key: date = Time(item, format='mjd').to_datetime() - outstr += f'{date.strftime(time_format)} (UTC)' + line += f'{date.strftime(time_format)} (UTC)' + elif 'az el info' in key: + line += f'{format_az_el_information(item, az_el_key, unit=az_el_unit, precision=precision)}' else: - outstr += str(item) - outstr += '\n' + line += str(item) + outstr += f'{line}\n' - outstr += f'{tab}{format_az_el_information(az_el_dict, az_el_key, unit=az_el_unit, precision=precision)}\n' return outstr -def format_spectral_information(freq_dict, tab=' '): - outstr = 'Spectral Summary:\n' +def format_spectral_information(freq_dict, tab, ident, key_size): + outstr = f'{ident}Spectral:\n' + tab += ident for key, item in freq_dict.items(): - outstr += f'{tab}{key.capitalize().replace('_', ' '):21s} = ' + outstr += f'{tab}{key.capitalize().replace('_', ' '):{key_size}s} => ' if 'range' in key: outstr += f'{format_frequency(item[0], decimal_places=3)} to {format_frequency(item[1], decimal_places=3)}' elif 'number' in key: @@ -727,3 +731,36 @@ def format_spectral_information(freq_dict, tab=' '): return outstr +def format_beam_information(beam_dict, tab, ident, key_size): + outstr = f'{ident}Beam:\n' + tab += ident + for key, item in beam_dict.items(): + outstr += f'{tab}{key.capitalize().replace('_', ' '):{key_size}s} => ' + if key == 'cell size': + outstr += format_angular_distance(item) + elif key == 'grid size': + outstr += f'{item[0]} by {item[1]} pixels' + else: + outstr += f'From {format_angular_distance(item[0])} to {format_angular_distance(item[1])}' + outstr += '\n' + return outstr + + +def format_observation_summary(obs_sum, tab_size=3, tab_count=0, az_el_key='mean', phase_center_unit='radec', + az_el_unit='deg', time_format="%d %h %Y, %H:%M:%S", precision='.1f', key_size=18): + spc = ' ' + major_tab = tab_count*tab_size*spc + one_tab = tab_size*spc + ident = one_tab+major_tab + outstr = f'{major_tab}Observation Summary:\n\n' + outstr += format_observation_information(obs_sum["general"], az_el_key=az_el_key, + phase_center_unit=phase_center_unit, az_el_unit=az_el_unit, + time_format=time_format, precision=precision, tab=one_tab, + ident=one_tab+major_tab, key_size=key_size) + outstr += '\n' + outstr += format_spectral_information(obs_sum["spectral"], one_tab, ident, key_size) + + outstr += '\n' + outstr += format_beam_information(obs_sum["beam"], one_tab, ident, key_size) + + return outstr From 41ef0183ea0cd92622b1337165afbc3cb90f241e Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Mon, 30 Jun 2025 17:15:06 -0600 Subject: [PATCH 32/74] added formating of aperture information --- src/astrohack/utils/text.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/astrohack/utils/text.py b/src/astrohack/utils/text.py index 096c5403..64556c4c 100644 --- a/src/astrohack/utils/text.py +++ b/src/astrohack/utils/text.py @@ -746,6 +746,19 @@ def format_beam_information(beam_dict, tab, ident, key_size): return outstr +def format_aperture_information(aperture_dict, tab, ident, key_size, resolution_unit='cm'): + outstr = f'{ident}Aperture:\n' + tab += ident + for key, item in aperture_dict.items(): + outstr += f'{tab}{key.capitalize().replace('_', ' '):{key_size}s} => ' + if key == 'grid size': + outstr += f'{item[0]} by {item[1]} pixels' + else: + outstr += f'{format_wavelength(item[0])} by {format_wavelength(item[1])}' + outstr += '\n' + return outstr + + def format_observation_summary(obs_sum, tab_size=3, tab_count=0, az_el_key='mean', phase_center_unit='radec', az_el_unit='deg', time_format="%d %h %Y, %H:%M:%S", precision='.1f', key_size=18): spc = ' ' @@ -763,4 +776,7 @@ def format_observation_summary(obs_sum, tab_size=3, tab_count=0, az_el_key='mean outstr += '\n' outstr += format_beam_information(obs_sum["beam"], one_tab, ident, key_size) + if obs_sum["aperture"] is not None: + outstr += '\n' + outstr += format_aperture_information(obs_sum["aperture"], one_tab, ident, key_size) return outstr From 811dabd820463e57cc540520e80f73b7af086379 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 11:58:24 -0600 Subject: [PATCH 33/74] Some formatting changes, added a function to make a centralized header. --- src/astrohack/utils/text.py | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/src/astrohack/utils/text.py b/src/astrohack/utils/text.py index 64556c4c..2820159a 100644 --- a/src/astrohack/utils/text.py +++ b/src/astrohack/utils/text.py @@ -764,12 +764,12 @@ def format_observation_summary(obs_sum, tab_size=3, tab_count=0, az_el_key='mean spc = ' ' major_tab = tab_count*tab_size*spc one_tab = tab_size*spc - ident = one_tab+major_tab - outstr = f'{major_tab}Observation Summary:\n\n' - outstr += format_observation_information(obs_sum["general"], az_el_key=az_el_key, - phase_center_unit=phase_center_unit, az_el_unit=az_el_unit, - time_format=time_format, precision=precision, tab=one_tab, - ident=one_tab+major_tab, key_size=key_size) + ident = major_tab + + outstr = format_observation_information(obs_sum["general"], az_el_key=az_el_key, + phase_center_unit=phase_center_unit, az_el_unit=az_el_unit, + time_format=time_format, precision=precision, tab=one_tab, + ident=ident, key_size=key_size) outstr += '\n' outstr += format_spectral_information(obs_sum["spectral"], one_tab, ident, key_size) @@ -780,3 +780,21 @@ def format_observation_summary(obs_sum, tab_size=3, tab_count=0, az_el_key='mean outstr += '\n' outstr += format_aperture_information(obs_sum["aperture"], one_tab, ident, key_size) return outstr + + +def make_header(heading, separator, header_width, buffer_width): + spc = ' ' + sep_line = f'{header_width * separator}\n' + len_head = len(heading) + before_blank = (header_width - 2*buffer_width - len_head)//2 + if 2*buffer_width+len_head+2*before_blank < header_width: + after_blank = before_blank+1 + else: + after_blank = before_blank + outstr = sep_line + buffer = buffer_width*separator + outstr += f'{buffer}{before_blank*spc}{heading}{after_blank*spc}{buffer}\n' + outstr += sep_line+'\n' + return outstr + + From 3f8b7cc8d7ca92ae7e2065b93cb85f4c0bb1daf6 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 12:01:30 -0600 Subject: [PATCH 34/74] Added a generalized routine capable of printing observation summary. --- src/astrohack/visualization/textual_data.py | 38 ++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/astrohack/visualization/textual_data.py b/src/astrohack/visualization/textual_data.py index 11d12188..4d64e5ea 100644 --- a/src/astrohack/visualization/textual_data.py +++ b/src/astrohack/visualization/textual_data.py @@ -1,7 +1,8 @@ import numpy as np from astrohack.core.image_comparison_tool import extract_rms_from_xds -from astrohack.utils import rad_to_deg_str, twopi, fixed_format_error, dynamic_format +from astrohack.utils import rad_to_deg_str, twopi, fixed_format_error, dynamic_format, format_observation_summary, \ + make_header from astrohack.antenna import Telescope, AntennaSurface from astrohack.utils import ( convert_unit, @@ -542,3 +543,38 @@ def create_fits_comparison_rms_table(parameters, xdt): if parameters["print_table"]: print(table) return + + +def generate_observation_summary(parm_dict): + antenna = parm_dict["this_ant"] + ddi = parm_dict["this_ddi"] + try: + map_id = parm_dict["this_map"] + is_holog_zarr = True + except KeyError: + map_id = None + is_holog_zarr = False + + xds = parm_dict["xds_data"] + obs_sum = xds.attrs["summary"] + + tab_size = parm_dict['tab_size'] + tab_count = 1 + + if is_holog_zarr: + header = f'{ddi}, {map_id}, {antenna}' + else: + header = f'{antenna}, {ddi}' + + outstr = make_header(header, '#', 60, 3) + + outstr += format_observation_summary(obs_sum, tab_size, tab_count, az_el_key=parm_dict['az_el_key'], + phase_center_unit=parm_dict['phase_center_unit'], + az_el_unit=parm_dict['az_el_unit'], + time_format=parm_dict["time_format"])+'\n' + + if parm_dict["print_summary"]: + print(outstr) + + return outstr + From 2b8e0e3a2a51854d8a271c925487ae9a8062a4ab Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 12:29:44 -0600 Subject: [PATCH 35/74] It is now possible to retrieve returns when using compute_graph. --- src/astrohack/utils/graph.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/astrohack/utils/graph.py b/src/astrohack/utils/graph.py index eb74bb21..470a27e4 100644 --- a/src/astrohack/utils/graph.py +++ b/src/astrohack/utils/graph.py @@ -27,8 +27,7 @@ def _construct_general_graph_recursively( delayed_list.append(dask.delayed(chunk_function)(dask.delayed(param_dict))) else: - delayed_list.append(0) - chunk_function(param_dict) + delayed_list.append((chunk_function, param_dict)) else: key = key_order[0] @@ -58,7 +57,7 @@ def _construct_general_graph_recursively( logger.warning(f"{item} is not present for {oneup}") -def compute_graph(looping_dict, chunk_function, param_dict, key_order, parallel=False): +def compute_graph(looping_dict, chunk_function, param_dict, key_order, parallel=False, fetch_returns=False): """ General tool for looping over the data and constructing graphs for dask parallel processing Args: @@ -67,6 +66,7 @@ def compute_graph(looping_dict, chunk_function, param_dict, key_order, parallel= param_dict: The parameter dictionary for the chunk function key_order: The order over which to loop over the keys inside the looping dictionary parallel: Are loops to be executed in parallel? + fetch_returns: retrieve returns and return them to the caller Returns: True if processing has occurred, False if no data was processed @@ -89,8 +89,16 @@ def compute_graph(looping_dict, chunk_function, param_dict, key_order, parallel= else: if parallel: - dask.compute(delayed_list) - return True + return_list = dask.compute(delayed_list)[0] + else: + return_list = [] + for pair in delayed_list: + return_list.append(pair[0](pair[1])) + + if fetch_returns: + return True, return_list + else: + return True def compute_graph_from_lists( From fc5acf0d87738e18a8bf02fdee03c35d02613276 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 12:32:21 -0600 Subject: [PATCH 36/74] Removed print toggle --- src/astrohack/visualization/textual_data.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/astrohack/visualization/textual_data.py b/src/astrohack/visualization/textual_data.py index 4d64e5ea..e3739bcc 100644 --- a/src/astrohack/visualization/textual_data.py +++ b/src/astrohack/visualization/textual_data.py @@ -573,8 +573,5 @@ def generate_observation_summary(parm_dict): az_el_unit=parm_dict['az_el_unit'], time_format=parm_dict["time_format"])+'\n' - if parm_dict["print_summary"]: - print(outstr) - return outstr From 3db4869a1d7765d6f55766ce62f7f00a824db585 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 12:38:28 -0600 Subject: [PATCH 37/74] Added observation summary method to AstrohackHologFile Class. --- src/astrohack/mds.py | 63 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/src/astrohack/mds.py b/src/astrohack/mds.py index 20e4e0e6..4bf6a45e 100644 --- a/src/astrohack/mds.py +++ b/src/astrohack/mds.py @@ -3,6 +3,7 @@ import numpy as np import toolviper.utils.logger as logger +from joblib.testing import param from toolviper.utils.console import Colorize @@ -41,7 +42,7 @@ export_phase_fit_chunk, print_array_configuration, export_to_parminator, - export_zernike_fit_chunk, + export_zernike_fit_chunk, generate_observation_summary, ) from astrohack.visualization.fits import ( export_to_fits_panel_chunk, @@ -785,6 +786,66 @@ def export_to_aips( compute_graph(self, export_to_aips, param_dict, key_order, parallel) return + def observation_summary( + self, + summary_file: str, + ant: Union[str, List[str]] = "all", + ddi: Union[int, List[int]] = "all", + map_id: Union[int, List[int]] = "all", + az_el_key: str = 'mean', + phase_center_unit: str = 'radec', + az_el_unit: str = 'deg', + time_format: str = "%d %h %Y, %H:%M:%S", + tab_size: int = 3, + print_summary: bool = True, + parallel: bool = False, + ) -> None: + """ Create a Summary of observation information + + :param summary_file: Text file to put the observation summary + :type summary_file: str + :param ant: antenna ID to use in subselection, defaults to "all" when None, ex. ea25 + :type ant: list or str, optional + :param ddi: data description ID to use in subselection, defaults to "all" when None, ex. 0 + :type ddi: list or int, optional + :param map_id: map ID to use in subselection. This relates to which antenna are in the mapping vs. scanning \ + configuration, defaults to "all" when None, ex. 0 + :type map_id: list or int, optional + :param az_el_key: What type of Azimuth & Elevation information to print, 'mean', 'median' or 'center', default\ + is 'mean' + :type az_el_key: str, optional + :param phase_center_unit: What unit to display phase center coordinates, 'radec' and angle units supported, \ + default is 'radec' + :type phase_center_unit: str, optional + :param az_el_unit: Angle unit used to display Azimuth & Elevation information, default is 'deg' + :type az_el_unit: str, optional + :param time_format: datetime time format for the start and end dates of observation, default is \ + "%d %h %Y, %H:%M:%S" + :type time_format: str, optional + :param tab_size: Number of spaces in the tab levels, default is 3 + :type tab_size: int, optional + :param print_summary: Print the summary at the end of execution, default is True + :type print_summary: bool, optional + :param parallel: Run in parallel, defaults to False + :type parallel: bool, optional + + **Additional Information** + + This method produces a summary of the data in the AstrohackHologFile displaying general information, + spectral information and suggested beam image characteristics. + """ + + param_dict = locals() + param_dict["map"] = map_id + key_order = ["ddi", "map", "ant"] + execution, summary = compute_graph(self, generate_observation_summary, param_dict, key_order, parallel, + fetch_returns=True) + summary = "".join(summary) + with open(summary_file, 'w') as output_file: + output_file.write(summary) + if print_summary: + print(summary) + class AstrohackPanelFile(dict): """Data class for holography panel data. From 4d25753f468801e6a9c37d29e26905a4a745db60 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 12:39:41 -0600 Subject: [PATCH 38/74] Added missing methods to the AstrohackHologFile Class summary. --- src/astrohack/mds.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/astrohack/mds.py b/src/astrohack/mds.py index 4bf6a45e..03f77b72 100644 --- a/src/astrohack/mds.py +++ b/src/astrohack/mds.py @@ -593,6 +593,8 @@ def summary(self) -> None: self.select, self.plot_diagnostics, self.plot_lm_sky_coverage, + self.export_to_aips, + self.observation_summary ] ) From d638ac01ed089914bf7b4ec735c1cecd57c094f9 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 12:42:56 -0600 Subject: [PATCH 39/74] Added observation summary to AstrohackPanelFile Class. --- src/astrohack/mds.py | 56 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/src/astrohack/mds.py b/src/astrohack/mds.py index 03f77b72..26a2c891 100644 --- a/src/astrohack/mds.py +++ b/src/astrohack/mds.py @@ -921,6 +921,7 @@ def summary(self) -> None: self.export_to_fits, self.plot_antennas, self.export_gain_tables, + self.observation_summary ] ) @@ -1195,6 +1196,61 @@ def export_gain_tables( parallel=parallel, ) + def observation_summary( + self, + summary_file: str, + ant: Union[str, List[str]] = "all", + ddi: Union[int, List[int]] = "all", + az_el_key: str = 'mean', + phase_center_unit: str = 'radec', + az_el_unit: str = 'deg', + time_format: str = "%d %h %Y, %H:%M:%S", + tab_size: int = 3, + print_summary: bool = True, + parallel: bool = False, + ) -> None: + """ Create a Summary of observation information + + :param summary_file: Text file to put the observation summary + :type summary_file: str + :param ant: antenna ID to use in subselection, defaults to "all" when None, ex. ea25 + :type ant: list or str, optional + :param ddi: data description ID to use in subselection, defaults to "all" when None, ex. 0 + :type ddi: list or int, optional + :param az_el_key: What type of Azimuth & Elevation information to print, 'mean', 'median' or 'center', default\ + is 'mean' + :type az_el_key: str, optional + :param phase_center_unit: What unit to display phase center coordinates, 'radec' and angle units supported, \ + default is 'radec' + :type phase_center_unit: str, optional + :param az_el_unit: Angle unit used to display Azimuth & Elevation information, default is 'deg' + :type az_el_unit: str, optional + :param time_format: datetime time format for the start and end dates of observation, default is \ + "%d %h %Y, %H:%M:%S" + :type time_format: str, optional + :param tab_size: Number of spaces in the tab levels, default is 3 + :type tab_size: int, optional + :param print_summary: Print the summary at the end of execution, default is True + :type print_summary: bool, optional + :param parallel: Run in parallel, defaults to False + :type parallel: bool, optional + + **Additional Information** + + This method produces a summary of the data in the AstrohackPanelFile displaying general information, + spectral information, beam image characteristics and aperture image characteristics. + """ + + param_dict = locals() + key_order = ["ant", "ddi"] + execution, summary = compute_graph(self, generate_observation_summary, param_dict, key_order, parallel, + fetch_returns=True) + summary = "".join(summary) + with open(summary_file, 'w') as output_file: + output_file.write(summary) + if print_summary: + print(summary) + class AstrohackPointFile(dict): """Data Class for holography pointing data.""" From 8a53c7c29f977e81ec688ee435b6124025812aa3 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 12:45:55 -0600 Subject: [PATCH 40/74] Added observation summary to AstrohackImageFile Class. --- src/astrohack/mds.py | 80 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/src/astrohack/mds.py b/src/astrohack/mds.py index 26a2c891..4adcac32 100644 --- a/src/astrohack/mds.py +++ b/src/astrohack/mds.py @@ -199,6 +199,9 @@ def summary(self): self.plot_beams, self.plot_apertures, self.export_phase_fit_results, + self.export_zernike_fit_results, + self.plot_zernike_model, + self.observation_summary ] ) @@ -518,6 +521,61 @@ def plot_zernike_model( parallel=parallel, ) + def observation_summary( + self, + summary_file: str, + ant: Union[str, List[str]] = "all", + ddi: Union[int, List[int]] = "all", + az_el_key: str = 'mean', + phase_center_unit: str = 'radec', + az_el_unit: str = 'deg', + time_format: str = "%d %h %Y, %H:%M:%S", + tab_size: int = 3, + print_summary: bool = True, + parallel: bool = False, + ) -> None: + """ Create a Summary of observation information + + :param summary_file: Text file to put the observation summary + :type summary_file: str + :param ant: antenna ID to use in subselection, defaults to "all" when None, ex. ea25 + :type ant: list or str, optional + :param ddi: data description ID to use in subselection, defaults to "all" when None, ex. 0 + :type ddi: list or int, optional + :param az_el_key: What type of Azimuth & Elevation information to print, 'mean', 'median' or 'center', default\ + is 'mean' + :type az_el_key: str, optional + :param phase_center_unit: What unit to display phase center coordinates, 'radec' and angle units supported, \ + default is 'radec' + :type phase_center_unit: str, optional + :param az_el_unit: Angle unit used to display Azimuth & Elevation information, default is 'deg' + :type az_el_unit: str, optional + :param time_format: datetime time format for the start and end dates of observation, default is \ + "%d %h %Y, %H:%M:%S" + :type time_format: str, optional + :param tab_size: Number of spaces in the tab levels, default is 3 + :type tab_size: int, optional + :param print_summary: Print the summary at the end of execution, default is True + :type print_summary: bool, optional + :param parallel: Run in parallel, defaults to False + :type parallel: bool, optional + + **Additional Information** + + This method produces a summary of the data in the AstrohackImageFile displaying general information, + spectral information, beam image characteristics and aperture image characteristics. + """ + + param_dict = locals() + key_order = ["ant", "ddi"] + execution, summary = compute_graph(self, generate_observation_summary, param_dict, key_order, parallel, + fetch_returns=True) + summary = "".join(summary) + with open(summary_file, 'w') as output_file: + output_file.write(summary) + if print_summary: + print(summary) + class AstrohackHologFile(dict): """Data Class for extracted holography data @@ -1197,17 +1255,17 @@ def export_gain_tables( ) def observation_summary( - self, - summary_file: str, - ant: Union[str, List[str]] = "all", - ddi: Union[int, List[int]] = "all", - az_el_key: str = 'mean', - phase_center_unit: str = 'radec', - az_el_unit: str = 'deg', - time_format: str = "%d %h %Y, %H:%M:%S", - tab_size: int = 3, - print_summary: bool = True, - parallel: bool = False, + self, + summary_file: str, + ant: Union[str, List[str]] = "all", + ddi: Union[int, List[int]] = "all", + az_el_key: str = 'mean', + phase_center_unit: str = 'radec', + az_el_unit: str = 'deg', + time_format: str = "%d %h %Y, %H:%M:%S", + tab_size: int = 3, + print_summary: bool = True, + parallel: bool = False, ) -> None: """ Create a Summary of observation information From a76e9a17c6d3e4d076b4b43fec81ad163d77b657 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 15:50:10 -0600 Subject: [PATCH 41/74] Added duration to observation summary and renamed some routines. --- src/astrohack/core/extract_holog.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index ea242d07..a24c3803 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -93,7 +93,7 @@ def process_extract_holog_chunk(extract_holog_params): scan_list = ctb.getcol("SCAN_NUMBER") field_ids = ctb.getcol("FIELD_ID") - obs_info = _get_obs_summary(ms_name, field_ids) + gen_info = _get_general_summary(ms_name, field_ids) # Here we use the median of the differences between dumps as this is a good proxy for the integration time if time_interval is None: @@ -140,7 +140,7 @@ def process_extract_holog_chunk(extract_holog_params): for ant_index in vis_map_dict.keys(): antenna_name = "_".join(("ant", ant_names[ant_index])) n_pix, cell_size = calculate_optimal_grid_parameters( - pnt_map_dict, antenna_name, Telescope(obs_info["telescope name"]).diam, chan_freq, ddi + pnt_map_dict, antenna_name, Telescope(gen_info["telescope name"]).diam, chan_freq, ddi ) grid_params[antenna_name] = {"n_pix": n_pix, "cell_size": cell_size} @@ -167,7 +167,7 @@ def process_extract_holog_chunk(extract_holog_params): ant_names, grid_params, time_interval, - obs_info, + gen_info, map_ref_dict ) @@ -390,7 +390,7 @@ def _create_holog_file( ant_names, grid_params, time_interval, - obs_info, + gen_info, map_ref_dict ): """Create holog-structured, formatted output file and save to zarr. @@ -468,7 +468,7 @@ def _create_holog_file( xds.attrs["parallactic_samples"] = parallactic_samples xds.attrs["time_smoothing_interval"] = time_interval - xds.attrs["summary"] = _crate_observation_summary(ant_names[map_ant_index], obs_info, grid_params, + xds.attrs["summary"] = _crate_observation_summary(ant_names[map_ant_index], gen_info, grid_params, xds["DIRECTIONAL_COSINES"].values, chan, pnt_map_dict[map_ant_tag], valid_data, map_ref_dict) @@ -809,7 +809,7 @@ def _get_time_index(data_time, i_time, time_axis, half_int): return i_time -def _get_obs_summary(ms_name, field_ids): +def _get_general_summary(ms_name, field_ids): unq_ids = np.unique(field_ids) field_tbl = ctables.table( ms_name + "::FIELD", @@ -838,14 +838,15 @@ def _get_obs_summary(ms_name, field_ids): phase_center_fk5[:, 0], ) - obs_info = { + gen_info = { "source": src_name[i_src], "phase center": phase_center_fk5[i_src].tolist(), "telescope name": telescope_name, "start time": time_range[0], - "stop time": time_range[-1] + "stop time": time_range[-1], + "duration": (time_range[-1]-time_range[0])*86400 # Store it in seconds rather than days } - return obs_info + return gen_info def _get_az_el_characteristics(pnt_map_xds, valid_data): From 49ccb3123941ab87d17799a7da510164f8afcfa9 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 16:02:45 -0600 Subject: [PATCH 42/74] Added formating of duration in seconds to a more comprehensible string. --- src/astrohack/utils/text.py | 55 +++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/src/astrohack/utils/text.py b/src/astrohack/utils/text.py index 2820159a..f5cf4b3a 100644 --- a/src/astrohack/utils/text.py +++ b/src/astrohack/utils/text.py @@ -457,6 +457,43 @@ def format_wavelength(wave_value, unit="m", decimal_places=2): return format_value_unit(fac * wave_value, unitout, decimal_places) +def format_duration(duration, unit='sec', decimal_places=2): + duration = np.abs(duration*convert_unit(unit, 'sec', 'time')) + oneminu = convert_unit('min', 'sec', 'time') + onehour = convert_unit('hour', 'sec', 'time') + oneday = convert_unit('day', 'sec', 'time') + + if duration < 1: + if duration < 1e-6: + unitout = 'nsec' + elif duration < 1e-3: + unitout = 'usec' + else: + unitout = 'msec' + fac = convert_unit('sec', unitout, "time") + return format_value_unit(fac * duration, unitout, decimal_places) + elif duration < oneminu: + return format_value_unit(duration, 'sec', decimal_places) + elif oneminu <= duration < onehour: + minu = int(np.floor(duration/oneminu)) + seco = duration - minu*oneminu + return f'{minu} min, {format_value_unit(seco, 'sec', decimal_places)}' + elif onehour <= duration < oneday: + hour = int(np.floor(duration/onehour)) + rest = duration - hour*onehour + minu = int(np.floor(rest/oneminu)) + seco = rest - minu*oneminu + return f'{hour} hour, {minu} min, {format_value_unit(seco, 'sec', decimal_places)}' + else: + day = int(np.floor(duration/oneday)) + rest = duration - day*oneday + hour = int(np.floor(rest/onehour)) + rest -= hour*onehour + minu = int(np.floor(rest/oneminu)) + seco = rest - minu*oneminu + return f'{day} day, {hour} hour, {minu} min, {format_value_unit(seco, 'sec', decimal_places)}' + + def format_angular_distance(user_value, unit="rad", decimal_places=2): one_deg = np.pi / 180 dist_value = np.abs(user_value) @@ -691,8 +728,8 @@ def format_az_el_information(az_el_dict, key='center', unit='deg', precision='.1 return az_el_label -def format_observation_information(obs_dict, tab, ident, key_size, az_el_key='mean', phase_center_unit='radec', az_el_unit='deg', - time_format="%d %h %Y, %H:%M:%S", precision='.1f'): +def format_general_information(obs_dict, tab, ident, key_size, az_el_key='mean', phase_center_unit='radec', + az_el_unit='deg', time_format="%d %h %Y, %H:%M:%S", precision='.1f'): outstr = f'{ident}General:\n' tab = tab+ident for key, item in obs_dict.items(): @@ -708,6 +745,8 @@ def format_observation_information(obs_dict, tab, ident, key_size, az_el_key='me line += f'{date.strftime(time_format)} (UTC)' elif 'az el info' in key: line += f'{format_az_el_information(item, az_el_key, unit=az_el_unit, precision=precision)}' + elif 'duration' == key: + line += f'{format_duration(item)}' else: line += str(item) outstr += f'{line}\n' @@ -746,7 +785,7 @@ def format_beam_information(beam_dict, tab, ident, key_size): return outstr -def format_aperture_information(aperture_dict, tab, ident, key_size, resolution_unit='cm'): +def format_aperture_information(aperture_dict, tab, ident, key_size): outstr = f'{ident}Aperture:\n' tab += ident for key, item in aperture_dict.items(): @@ -766,10 +805,10 @@ def format_observation_summary(obs_sum, tab_size=3, tab_count=0, az_el_key='mean one_tab = tab_size*spc ident = major_tab - outstr = format_observation_information(obs_sum["general"], az_el_key=az_el_key, - phase_center_unit=phase_center_unit, az_el_unit=az_el_unit, - time_format=time_format, precision=precision, tab=one_tab, - ident=ident, key_size=key_size) + outstr = format_general_information(obs_sum["general"], az_el_key=az_el_key, + phase_center_unit=phase_center_unit, az_el_unit=az_el_unit, + time_format=time_format, precision=precision, tab=one_tab, + ident=ident, key_size=key_size) outstr += '\n' outstr += format_spectral_information(obs_sum["spectral"], one_tab, ident, key_size) @@ -796,5 +835,3 @@ def make_header(heading, separator, header_width, buffer_width): outstr += f'{buffer}{before_blank*spc}{heading}{after_blank*spc}{buffer}\n' outstr += sep_line+'\n' return outstr - - From 9cffbbed8864ded0f1709554935abef94fa05d36 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 16:05:46 -0600 Subject: [PATCH 43/74] Added a comment on the dates being MJD in days --- src/astrohack/core/extract_holog.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/astrohack/core/extract_holog.py b/src/astrohack/core/extract_holog.py index a24c3803..85e27a1c 100644 --- a/src/astrohack/core/extract_holog.py +++ b/src/astrohack/core/extract_holog.py @@ -842,9 +842,9 @@ def _get_general_summary(ms_name, field_ids): "source": src_name[i_src], "phase center": phase_center_fk5[i_src].tolist(), "telescope name": telescope_name, - "start time": time_range[0], - "stop time": time_range[-1], - "duration": (time_range[-1]-time_range[0])*86400 # Store it in seconds rather than days + "start time": time_range[0], # start time is in MJD in days + "stop time": time_range[-1], # stop time is in MJD in days + "duration": (time_range[-1]-time_range[0])*86400 # Store it in seconds rather than days } return gen_info From fa0fd7634f0af27935e927b7b366834af4b6ac75 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 16:25:46 -0600 Subject: [PATCH 44/74] Added units.radec to custom_unit_checker --- src/astrohack/utils/validation.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/astrohack/utils/validation.py b/src/astrohack/utils/validation.py index 30bb8cfd..9ef33424 100644 --- a/src/astrohack/utils/validation.py +++ b/src/astrohack/utils/validation.py @@ -28,7 +28,10 @@ def custom_unit_checker(unit_type): elif unit_type == "units.frequency": return freq_units - + elif unit_type == 'units.radec': + valid_units = trigo_units.copy() + valid_units.append('radec') + return valid_units else: return "Not found" From 128c0d9a4708536a725cf841d0f7e2ae7ec0b9b6 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 16:28:10 -0600 Subject: [PATCH 45/74] Added parameter checking to observation summary. --- src/astrohack/config/mds.param.json | 233 +++++++++++++++++++++++++++- src/astrohack/mds.py | 4 +- 2 files changed, 235 insertions(+), 2 deletions(-) diff --git a/src/astrohack/config/mds.param.json b/src/astrohack/config/mds.param.json index 3a2e2904..f9d43d02 100644 --- a/src/astrohack/config/mds.param.json +++ b/src/astrohack/config/mds.param.json @@ -518,7 +518,79 @@ ] } }, - + "AstrohackImageFile.observation_summary": { + "summary_file": { + "nullable": false, + "required": true, + "type": [ + "string" + ] + }, + "ant": { + "nullable": false, + "required": false, + "struct_type": [ + "str" + ], + "minlength": 1, + "type": [ + "string", + "list" + ] + }, + "ddi": { + "nullable": false, + "required": false, + "struct_type": [ + "int" + ], + "minlength": 1, + "type": [ + "int", + "list", + "string" + ] + }, + "az_el_key": { + "nullable": false, + "required": true, + "type": ["string"], + "allowed": ["mean", "median", "center"] + }, + "phase_center_unit": { + "nullable": false, + "required": false, + "type": ["string"], + "check allowed with": "units.radec" + }, + "az_el_unit": { + "nullable": false, + "required": false, + "type": ["string"], + "check allowed with": "units.trig" + }, + "time_format":{ + "nullable": false, + "required": false, + "type": ["string"] + }, + "tab_size": { + "nullable": false, + "required": false, + "type": ["int"], + "min": 0 + }, + "print_summary": { + "nullable": false, + "required": false, + "type": ["boolean"] + }, + "parallel": { + "nullable": false, + "required": false, + "type": ["boolean"] + } + }, "AstrohackHologFile.select": { "ddi": { @@ -825,6 +897,92 @@ ] } }, + "AstrohackHologFile.observation_summary": { + "summary_file": { + "nullable": false, + "required": true, + "type": [ + "string" + ] + }, + "map_id": { + "nullable": false, + "required": false, + "struct_type": [ + "int" + ], + "minlength": 1, + "type": [ + "int", + "list", + "string" + ] + }, + "ant": { + "nullable": false, + "required": false, + "struct_type": [ + "str" + ], + "minlength": 1, + "type": [ + "string", + "list" + ] + }, + "ddi": { + "nullable": false, + "required": false, + "struct_type": [ + "int" + ], + "minlength": 1, + "type": [ + "int", + "list", + "string" + ] + }, + "az_el_key": { + "nullable": false, + "required": true, + "type": ["string"], + "allowed": ["mean", "median", "center"] + }, + "phase_center_unit": { + "nullable": false, + "required": false, + "type": ["string"], + "check allowed with": "units.radec" + }, + "az_el_unit": { + "nullable": false, + "required": false, + "type": ["string"], + "check allowed with": "units.trig" + }, + "time_format":{ + "nullable": false, + "required": false, + "type": ["string"] + }, + "tab_size": { + "nullable": false, + "required": false, + "type": ["int"], + "min": 0 + }, + "print_summary": { + "nullable": false, + "required": false, + "type": ["boolean"] + }, + "parallel": { + "nullable": false, + "required": false, + "type": ["boolean"] + } + }, "AstrohackPanelFile.get_antenna": { "ant":{ @@ -1087,6 +1245,79 @@ "type": ["boolean"] } }, + "AstrohackPanelFile.observation_summary": { + "summary_file": { + "nullable": false, + "required": true, + "type": [ + "string" + ] + }, + "ant": { + "nullable": false, + "required": false, + "struct_type": [ + "str" + ], + "minlength": 1, + "type": [ + "string", + "list" + ] + }, + "ddi": { + "nullable": false, + "required": false, + "struct_type": [ + "int" + ], + "minlength": 1, + "type": [ + "int", + "list", + "string" + ] + }, + "az_el_key": { + "nullable": false, + "required": true, + "type": ["string"], + "allowed": ["mean", "median", "center"] + }, + "phase_center_unit": { + "nullable": false, + "required": false, + "type": ["string"], + "check allowed with": "units.radec" + }, + "az_el_unit": { + "nullable": false, + "required": false, + "type": ["string"], + "check allowed with": "units.trig" + }, + "time_format":{ + "nullable": false, + "required": false, + "type": ["string"] + }, + "tab_size": { + "nullable": false, + "required": false, + "type": ["int"], + "min": 0 + }, + "print_summary": { + "nullable": false, + "required": false, + "type": ["boolean"] + }, + "parallel": { + "nullable": false, + "required": false, + "type": ["boolean"] + } + }, "AstrohackLocitFile.print_array_configuration":{ "relative":{ diff --git a/src/astrohack/mds.py b/src/astrohack/mds.py index 4adcac32..d246faa0 100644 --- a/src/astrohack/mds.py +++ b/src/astrohack/mds.py @@ -3,7 +3,6 @@ import numpy as np import toolviper.utils.logger as logger -from joblib.testing import param from toolviper.utils.console import Colorize @@ -521,6 +520,7 @@ def plot_zernike_model( parallel=parallel, ) + @toolviper.utils.parameter.validate(custom_checker=custom_unit_checker) def observation_summary( self, summary_file: str, @@ -846,6 +846,7 @@ def export_to_aips( compute_graph(self, export_to_aips, param_dict, key_order, parallel) return + @toolviper.utils.parameter.validate(custom_checker=custom_unit_checker) def observation_summary( self, summary_file: str, @@ -1254,6 +1255,7 @@ def export_gain_tables( parallel=parallel, ) + @toolviper.utils.parameter.validate(custom_checker=custom_unit_checker) def observation_summary( self, summary_file: str, From d1ee86ecf176e04656b9adb5437bec2b5aaca4b6 Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 16:43:53 -0600 Subject: [PATCH 46/74] Fixed bug introduced when trying to open a telescope from a xds. --- src/astrohack/visualization/diagnostics.py | 2 +- src/astrohack/visualization/textual_data.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/astrohack/visualization/diagnostics.py b/src/astrohack/visualization/diagnostics.py index 30e2d3f6..81cfcb0b 100644 --- a/src/astrohack/visualization/diagnostics.py +++ b/src/astrohack/visualization/diagnostics.py @@ -780,7 +780,7 @@ def plot_antenna_chunk(parm_dict): plot_type = parm_dict["plot_type"] basename = f"{destination}/{antenna}_{ddi}" xds = parm_dict["xds_data"] - telescope = Telescope(xds.attrs["telescope_name"]) + telescope = Telescope.from_xds(xds) surface = AntennaSurface(xds, telescope, reread=True) if plot_type == plot_types[0]: # deviation plot surface.plot_deviation(basename, "panel", parm_dict) diff --git a/src/astrohack/visualization/textual_data.py b/src/astrohack/visualization/textual_data.py index e3739bcc..b9b3d1b0 100644 --- a/src/astrohack/visualization/textual_data.py +++ b/src/astrohack/visualization/textual_data.py @@ -280,7 +280,7 @@ def export_screws_chunk(parm_dict): ddi = parm_dict["this_ddi"] export_name = parm_dict["destination"] + f"/panel_screws_{antenna}_{ddi}." xds = parm_dict["xds_data"] - telescope = Telescope(xds.attrs["telescope_name"]) + telescope = Telescope.from_xds(xds) surface = AntennaSurface(xds, telescope, reread=True) surface.export_screws(export_name + "txt", unit=parm_dict["unit"]) surface.plot_screw_adjustments(export_name + "png", parm_dict) From e1e99e2aa5c19d2a7384b35fdff4923b8121052f Mon Sep 17 00:00:00 2001 From: Victor de Souza magalhaes Date: Tue, 1 Jul 2025 16:44:22 -0600 Subject: [PATCH 47/74] Updated tutorial_vla.ipynb to use the new observation summaries. --- docs/tutorial_vla.ipynb | 1610 +++++++++++++++++++++++++++------------ 1 file changed, 1129 insertions(+), 481 deletions(-) diff --git a/docs/tutorial_vla.ipynb b/docs/tutorial_vla.ipynb index 897a79ba..1671be19 100644 --- a/docs/tutorial_vla.ipynb +++ b/docs/tutorial_vla.ipynb @@ -41,12 +41,6 @@ "execution_count": 1, "id": "7db6f868-030c-41ee-8188-c236aa675c27", "metadata": { - "execution": { - "iopub.execute_input": "2025-06-04T16:18:37.911073Z", - "iopub.status.busy": "2025-06-04T16:18:37.910824Z", - "iopub.status.idle": "2025-06-04T16:18:40.926016Z", - "shell.execute_reply": "2025-06-04T16:18:40.923455Z" - }, "tags": [] }, "outputs": [ @@ -54,7 +48,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:38,376\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m viperlog: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n" + "AstroHACK version 0.7.1 already installed.\n" ] }, { @@ -158,12 +152,6 @@ "execution_count": 2, "id": "ffb79bcd", "metadata": { - "execution": { - "iopub.execute_input": "2025-06-04T16:18:40.928267Z", - "iopub.status.busy": "2025-06-04T16:18:40.927773Z", - "iopub.status.idle": "2025-06-04T16:18:43.383806Z", - "shell.execute_reply": "2025-06-04T16:18:43.383273Z" - }, "tags": [] }, "outputs": [ @@ -171,21 +159,37 @@ "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:40,929\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper/utils/data/.dropbox\u001b[0m \n" + "[\u001b[38;2;128;05;128m2025-07-01 16:42:16,030\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper/utils/data/.dropbox\u001b[0m \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:16,031\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Updating file metadata information ... \n", + " " ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:40,929\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Updating file metadata information ... \n" - ] + "data": { + "text/html": [ + "
                                       \n",
+       "  Download List                        \n",
+       " ───────────────────────────────────── \n",
+       "  ea25_cal_small_after_fixed.split.ms  \n",
+       "                                       \n",
+       "
\n" + ], + "text/plain": [ + " \n", + " \u001b[1m \u001b[0m\u001b[1mDownload List \u001b[0m\u001b[1m \u001b[0m \n", + " ───────────────────────────────────── \n", + " \u001b[35mea25_cal_small_after_fixed.split.ms\u001b[0m \n", + " \n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ - " " + "[\u001b[38;2;128;05;128m2025-07-01 16:42:17,241\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: data/ea25_cal_small_after_fixed.split.ms \n" ] }, { @@ -291,78 +295,93 @@ "cell_type": "code", "execution_count": 3, "id": "7fb0902f-f274-4d47-a48e-61c0c2561ca7", - "metadata": { - "execution": { - "iopub.execute_input": "2025-06-04T16:18:43.387848Z", - "iopub.status.busy": "2025-06-04T16:18:43.387343Z", - "iopub.status.idle": "2025-06-04T16:18:50.451784Z", - "shell.execute_reply": "2025-06-04T16:18:50.451381Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:43,388\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:48,304\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Finished processing \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:48,348\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Writing distance matrix to /export/home/figs/vdesouza/work/Holography-1022/astrohack/docs/.baseline_distance_matrix.csv ... \n" + "[\u001b[38;2;128;05;128m2025-07-01 16:42:17,246\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:21,526\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Finished processing \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:21,612\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Writing distance matrix to /export/home/figs/vdesouza/work/Holography-1022/astrohack/docs/.baseline_distance_matrix.csv ... \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:21,620\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper/utils/data/.dropbox\u001b[0m \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:21,624\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Updating file metadata information ... \n", + " " ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:48,352\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper/utils/data/.dropbox\u001b[0m \n" - ] + "data": { + "text/html": [ + "
                   \n",
+       "  Download List    \n",
+       " ───────────────── \n",
+       "  heuristic_model  \n",
+       "                   \n",
+       "
\n" + ], + "text/plain": [ + " \n", + " \u001b[1m \u001b[0m\u001b[1mDownload List \u001b[0m\u001b[1m \u001b[0m \n", + " ───────────────── \n", + " \u001b[35mheuristic_model\u001b[0m \n", + " \n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:48,352\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Updating file metadata information ... \n" + " " ] }, { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - " " + "model/elastic.model: 100%|██████████████████████████████████████████████████████████████████████████████| 556/556 [00:00<00:00, 2.69MiB/s]\n", + "/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/sklearn/base.py:380: InconsistentVersionWarning: Trying to unpickle estimator ElasticNet from version 1.3.2 when using version 1.6.1. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n", + "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n", + " warnings.warn(\n" ] }, { "data": { "text/html": [ - "
                   \n",
-       "  Download List    \n",
-       " ───────────────── \n",
-       "  heuristic_model  \n",
-       "                   \n",
+       "
                                      System Info                                       \n",
+       "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃ N-cores ┃ Available memory (MB) ┃ Total memory (MB) ┃ Suggested memory per core (MB) ┃\n",
+       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       4 │ 18966                 │ 31703             │                          11911 │\n",
+       "└─────────┴───────────────────────┴───────────────────┴────────────────────────────────┘\n",
+       "    Available memory: represents the system memory available without going into swap    \n",
        "
\n"
       ],
       "text/plain": [
-       "                   \n",
-       " \u001b[1m \u001b[0m\u001b[1mDownload List  \u001b[0m\u001b[1m \u001b[0m \n",
-       " ───────────────── \n",
-       "  \u001b[35mheuristic_model\u001b[0m  \n",
-       "                   \n"
+       "\u001b[3m                                      System Info                                       \u001b[0m\n",
+       "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃\u001b[1m \u001b[0m\u001b[1mN-cores\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAvailable memory (MB)\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mTotal memory (MB)\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mSuggested memory per core (MB)\u001b[0m\u001b[1m \u001b[0m┃\n",
+       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│\u001b[34m \u001b[0m\u001b[34m      4\u001b[0m\u001b[34m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m18966                \u001b[0m\u001b[35m \u001b[0m│\u001b[36m \u001b[0m\u001b[36m31703            \u001b[0m\u001b[36m \u001b[0m│\u001b[32m \u001b[0m\u001b[32m                         11911\u001b[0m\u001b[32m \u001b[0m│\n",
+       "└─────────┴───────────────────────┴───────────────────┴────────────────────────────────┘\n",
+       "\u001b[2;3m    Available memory: represents the system memory available without going into swap    \u001b[0m\n"
       ]
      },
      "metadata": {},
      "output_type": "display_data"
     },
+    {
+     "data": {
+      "text/plain": [
+       "11911"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
     {
      "name": "stdout",
      "output_type": "stream",
@@ -443,12 +462,6 @@
    "execution_count": 4,
    "id": "10dffc63-1907-497f-b025-392b5813eac9",
    "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-06-04T16:18:50.453501Z",
-     "iopub.status.busy": "2025-06-04T16:18:50.453006Z",
-     "iopub.status.idle": "2025-06-04T16:18:52.068383Z",
-     "shell.execute_reply": "2025-06-04T16:18:52.067796Z"
-    },
     "tags": []
    },
    "outputs": [
@@ -456,63 +469,503 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:50,454\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper\u001b[0m \n"
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:23,706\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper\u001b[0m \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:23,712\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m It is recommended that the local cache directory be set using the \u001b[38;2;50;50;205mdask_local_dir\u001b[0m parameter. \n"
      ]
     },
     {
-     "name": "stdout",
+     "name": "stderr",
      "output_type": "stream",
      "text": [
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:50,461\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m It is recommended that the local cache directory be set using the \u001b[38;2;50;50;205mdask_local_dir\u001b[0m parameter. \n"
+      "/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/distributed/node.py:187: UserWarning: Port 8787 is already in use.\n",
+      "Perhaps you already have a cluster running?\n",
+      "Hosting the HTTP server on port 35271 instead\n",
+      "  warnings.warn(\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,937\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: slurm -- Success \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mdask_jobqueue\u001b[0m is available \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: dask_ssh -- Fail \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205masyncssh\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mjupyter_server_proxy\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mparamiko\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: CUDA -- Fail \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,939\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mdask_cuda\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,939\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m \u001b[38;2;46;139;87mAvailable functions of this environment\u001b[0m: slurm \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: slurm -- Success \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: slurm -- Success \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mdask_jobqueue\u001b[0m is available \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mdask_jobqueue\u001b[0m is available \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: dask_ssh -- Fail \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205masyncssh\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: dask_ssh -- Fail \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mjupyter_server_proxy\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mparamiko\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205masyncssh\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: CUDA -- Fail \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mjupyter_server_proxy\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mparamiko\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mdask_cuda\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m \u001b[38;2;46;139;87mAvailable functions of this environment\u001b[0m: slurm \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: CUDA -- Fail \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mdask_cuda\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m \u001b[38;2;46;139;87mAvailable functions of this environment\u001b[0m: slurm \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: slurm -- Success \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mdask_jobqueue\u001b[0m is available \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,945\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: dask_ssh -- Fail \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,945\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205masyncssh\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,945\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mjupyter_server_proxy\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,945\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mparamiko\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,946\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Loading module: CUDA -- Fail \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,946\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m    \u001b[38;2;50;50;205mdask_cuda\u001b[0m is unavailable \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,946\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m \u001b[38;2;46;139;87mAvailable functions of this environment\u001b[0m: slurm \n",
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,948\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Client  \n"
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:26,465\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Client  \n"
      ]
     },
+    {
+     "data": {
+      "text/html": [
+       "
\n",
+       "    
 
\n",
+       "    
\n",
+       "        
Client
\n",
+       "        
MenrvaClient-a8c7c1b4-56cc-11f0-a921-0c37967bc0e9
\n",
+       "        \n",
+       "\n",
+       "        \n",
+       "        \n",
+       "            \n",
+       "            \n",
+       "        \n",
+       "        \n",
+       "\n",
+       "        \n",
+       "            \n",
+       "                \n",
+       "                \n",
+       "            \n",
+       "        \n",
+       "\n",
+       "        
Connection method: Cluster objectCluster type: distributed.LocalCluster
\n",
+       "                    Dashboard:  http://127.0.0.1:35271/status\n",
+       "                
\n",
+       "\n",
+       "        \n",
+       "\n",
+       "        \n",
+       "            
\n",
+       "            
Cluster Info
\n",
+       "            
\n",
+       "    
\n",
+       "    
\n",
+       "    
\n",
+       "        
LocalCluster
\n",
+       "        
594d734b
\n",
+       "        \n",
+       "            \n",
+       "                \n",
+       "                \n",
+       "            \n",
+       "            \n",
+       "                \n",
+       "                \n",
+       "            \n",
+       "            \n",
+       "            \n",
+       "    \n",
+       "    \n",
+       "\n",
+       "\n",
+       "            \n",
+       "        
\n",
+       "                    Dashboard: http://127.0.0.1:35271/status\n",
+       "                \n",
+       "                    Workers: 4\n",
+       "                
\n",
+       "                    Total threads: 4\n",
+       "                \n",
+       "                    Total memory: 30.96 GiB\n",
+       "                
Status: runningUsing processes: True
\n",
+       "\n",
+       "        
\n",
+       "            \n",
+       "                
Scheduler Info
\n",
+       "            \n",
+       "\n",
+       "            
\n",
+       "    
\n",
+       "        
 
\n",
+       "        
\n",
+       "            
Scheduler
\n",
+       "            
Scheduler-9987407a-6cc6-4c6f-b5b1-976f7cc90d32
\n",
+       "            \n",
+       "                \n",
+       "                    \n",
+       "                    \n",
+       "                \n",
+       "                \n",
+       "                    \n",
+       "                    \n",
+       "                \n",
+       "                \n",
+       "                    \n",
+       "                    \n",
+       "                \n",
+       "            
\n",
+       "                        Comm: tcp://127.0.0.1:46589\n",
+       "                    \n",
+       "                        Workers: 4 \n",
+       "                    
\n",
+       "                        Dashboard: http://127.0.0.1:35271/status\n",
+       "                    \n",
+       "                        Total threads: 4\n",
+       "                    
\n",
+       "                        Started: Just now\n",
+       "                    \n",
+       "                        Total memory: 30.96 GiB\n",
+       "                    
\n",
+       "        
\n",
+       "    
\n",
+       "\n",
+       "    
\n",
+       "        \n",
+       "            
Workers
\n",
+       "        \n",
+       "\n",
+       "        \n",
+       "        
\n",
+       "            
 
\n",
+       "            
\n",
+       "            
\n",
+       "                \n",
+       "                    
Worker: 0
\n",
+       "                \n",
+       "                \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                    \n",
+       "\n",
+       "                    \n",
+       "\n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "\n",
+       "                
\n",
+       "                            Comm:  tcp://127.0.0.1:37205\n",
+       "                        \n",
+       "                            Total threads:  1\n",
+       "                        
\n",
+       "                            Dashboard:  http://127.0.0.1:44653/status\n",
+       "                        \n",
+       "                            Memory:  7.74 GiB\n",
+       "                        
\n",
+       "                            Nanny:  tcp://127.0.0.1:46231\n",
+       "                        
\n",
+       "                            Local directory:  /tmp/dask-scratch-space/worker-c9r4n_ir\n",
+       "                        
\n",
+       "                            Tasks executing:  \n",
+       "                        \n",
+       "                            Tasks in memory:  \n",
+       "                        
\n",
+       "                            Tasks ready:  \n",
+       "                        \n",
+       "                            Tasks in flight: \n",
+       "                        
\n",
+       "                            CPU usage: 0.0%\n",
+       "                        \n",
+       "                            Last seen:  Just now\n",
+       "                        
\n",
+       "                            Memory usage:  63.73 MiB\n",
+       "                        \n",
+       "                            Spilled bytes:  0 B\n",
+       "                        
\n",
+       "                            Read bytes:  0.0 B\n",
+       "                        \n",
+       "                            Write bytes:  0.0 B\n",
+       "                        
\n",
+       "            
\n",
+       "            
\n",
+       "        
\n",
+       "        \n",
+       "        
\n",
+       "            
 
\n",
+       "            
\n",
+       "            
\n",
+       "                \n",
+       "                    
Worker: 1
\n",
+       "                \n",
+       "                \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                    \n",
+       "\n",
+       "                    \n",
+       "\n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "\n",
+       "                
\n",
+       "                            Comm:  tcp://127.0.0.1:43595\n",
+       "                        \n",
+       "                            Total threads:  1\n",
+       "                        
\n",
+       "                            Dashboard:  http://127.0.0.1:40663/status\n",
+       "                        \n",
+       "                            Memory:  7.74 GiB\n",
+       "                        
\n",
+       "                            Nanny:  tcp://127.0.0.1:39425\n",
+       "                        
\n",
+       "                            Local directory:  /tmp/dask-scratch-space/worker-py5wco57\n",
+       "                        
\n",
+       "                            Tasks executing:  \n",
+       "                        \n",
+       "                            Tasks in memory:  \n",
+       "                        
\n",
+       "                            Tasks ready:  \n",
+       "                        \n",
+       "                            Tasks in flight: \n",
+       "                        
\n",
+       "                            CPU usage: 0.0%\n",
+       "                        \n",
+       "                            Last seen:  Just now\n",
+       "                        
\n",
+       "                            Memory usage:  63.33 MiB\n",
+       "                        \n",
+       "                            Spilled bytes:  0 B\n",
+       "                        
\n",
+       "                            Read bytes:  0.0 B\n",
+       "                        \n",
+       "                            Write bytes:  0.0 B\n",
+       "                        
\n",
+       "            
\n",
+       "            
\n",
+       "        
\n",
+       "        \n",
+       "        
\n",
+       "            
 
\n",
+       "            
\n",
+       "            
\n",
+       "                \n",
+       "                    
Worker: 2
\n",
+       "                \n",
+       "                \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                    \n",
+       "\n",
+       "                    \n",
+       "\n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "\n",
+       "                
\n",
+       "                            Comm:  tcp://127.0.0.1:43897\n",
+       "                        \n",
+       "                            Total threads:  1\n",
+       "                        
\n",
+       "                            Dashboard:  http://127.0.0.1:41813/status\n",
+       "                        \n",
+       "                            Memory:  7.74 GiB\n",
+       "                        
\n",
+       "                            Nanny:  tcp://127.0.0.1:40629\n",
+       "                        
\n",
+       "                            Local directory:  /tmp/dask-scratch-space/worker-a1ahw76j\n",
+       "                        
\n",
+       "                            Tasks executing:  \n",
+       "                        \n",
+       "                            Tasks in memory:  \n",
+       "                        
\n",
+       "                            Tasks ready:  \n",
+       "                        \n",
+       "                            Tasks in flight: \n",
+       "                        
\n",
+       "                            CPU usage: 0.0%\n",
+       "                        \n",
+       "                            Last seen:  Just now\n",
+       "                        
\n",
+       "                            Memory usage:  63.75 MiB\n",
+       "                        \n",
+       "                            Spilled bytes:  0 B\n",
+       "                        
\n",
+       "                            Read bytes:  0.0 B\n",
+       "                        \n",
+       "                            Write bytes:  0.0 B\n",
+       "                        
\n",
+       "            
\n",
+       "            
\n",
+       "        
\n",
+       "        \n",
+       "        
\n",
+       "            
 
\n",
+       "            
\n",
+       "            
\n",
+       "                \n",
+       "                    
Worker: 3
\n",
+       "                \n",
+       "                \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                    \n",
+       "\n",
+       "                    \n",
+       "\n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "                        \n",
+       "                        \n",
+       "                    \n",
+       "                    \n",
+       "\n",
+       "                
\n",
+       "                            Comm:  tcp://127.0.0.1:41981\n",
+       "                        \n",
+       "                            Total threads:  1\n",
+       "                        
\n",
+       "                            Dashboard:  http://127.0.0.1:37365/status\n",
+       "                        \n",
+       "                            Memory:  7.74 GiB\n",
+       "                        
\n",
+       "                            Nanny:  tcp://127.0.0.1:38545\n",
+       "                        
\n",
+       "                            Local directory:  /tmp/dask-scratch-space/worker-bgmaxbqf\n",
+       "                        
\n",
+       "                            Tasks executing:  \n",
+       "                        \n",
+       "                            Tasks in memory:  \n",
+       "                        
\n",
+       "                            Tasks ready:  \n",
+       "                        \n",
+       "                            Tasks in flight: \n",
+       "                        
\n",
+       "                            CPU usage: 0.0%\n",
+       "                        \n",
+       "                            Last seen:  Just now\n",
+       "                        
\n",
+       "                            Memory usage:  64.02 MiB\n",
+       "                        \n",
+       "                            Spilled bytes:  0 B\n",
+       "                        
\n",
+       "                            Read bytes:  0.0 B\n",
+       "                        \n",
+       "                            Write bytes:  0.0 B\n",
+       "                        
\n",
+       "            
\n",
+       "            
\n",
+       "        
\n",
+       "        \n",
+       "\n",
+       "    
\n",
+       "
\n",
+       "\n",
+       "        
\n",
+       "    
\n",
+       "
\n",
+       "            
\n",
+       "        \n",
+       "\n",
+       "    
\n",
+       "
"
+      ],
+      "text/plain": [
+       ""
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
     {
      "data": {
       "text/html": [
@@ -632,12 +1085,6 @@
    "execution_count": 5,
    "id": "0f6f37a0-4994-48b6-b930-7fb61e0d8db7",
    "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-06-04T16:18:52.070545Z",
-     "iopub.status.busy": "2025-06-04T16:18:52.070349Z",
-     "iopub.status.idle": "2025-06-04T16:19:09.447310Z",
-     "shell.execute_reply": "2025-06-04T16:19:09.446741Z"
-    },
     "tags": []
    },
    "outputs": [
@@ -645,7 +1092,21 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "[\u001b[38;2;128;05;128m2025-06-04 10:18:52,071\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n"
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:26,498\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:26,501\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m data/ea25_cal_small_after_fixed.split.point.zarr will be overwritten. \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,724\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Finished processing \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,760\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,768\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Creating output file name: data/ea25_cal_small_after_fixed.split.holog.zarr \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,771\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m data/ea25_cal_small_after_fixed.split.holog.zarr will be overwritten. \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,826\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Processing ddi: 0, scans: [8 ... 57] \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,828\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Processing ddi: 1, scans: [8 ... 57] \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:50,573\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m    worker_3: \u001b[0m EA25: DDI 0: Suggested cell size 2.47 amin, FOV: (1.11 deg, 1.11 deg) \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:50,579\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m    worker_3: \u001b[0m EA06: DDI 0: Suggested cell size 2.47 amin, FOV: (1.11 deg, 1.11 deg) \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:51,244\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m    worker_2: \u001b[0m EA25: DDI 1: Suggested cell size 2.20 amin, FOV: (1.11 deg, 1.11 deg) \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:51,250\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m    worker_2: \u001b[0m EA06: DDI 1: Suggested cell size 2.20 amin, FOV: (1.11 deg, 1.11 deg) \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:52,688\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m    worker_3: \u001b[0m Finished extracting holography chunk for ddi: 0 holog_map_key: map_0 \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:53,485\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m    worker_2: \u001b[0m Finished extracting holography chunk for ddi: 1 holog_map_key: map_0 \n",
+      "[\u001b[38;2;128;05;128m2025-07-01 16:42:53,495\u001b[0m] \u001b[38;2;50;50;205m    INFO\u001b[0m\u001b[38;2;112;128;144m   astrohack: \u001b[0m Finished processing \n"
      ]
     },
     {
@@ -791,12 +1252,6 @@
    "execution_count": 6,
    "id": "f4ce2575-5925-4822-b426-fc8b67580e9e",
    "metadata": {
-    "execution": {
-     "iopub.execute_input": "2025-06-04T16:19:09.449502Z",
-     "iopub.status.busy": "2025-06-04T16:19:09.448979Z",
-     "iopub.status.idle": "2025-06-04T16:19:09.480920Z",
-     "shell.execute_reply": "2025-06-04T16:19:09.480484Z"
-    },
     "tags": []
    },
    "outputs": [
@@ -1186,17 +1641,11 @@
        "    VIS                        (time, chan, pol) complex128 37MB dask.array<chunksize=(2229, 16, 2), meta=np.ndarray>\n",
        "    WEIGHT                     (time, chan, pol) float64 18MB dask.array<chunksize=(2229, 16, 2), meta=np.ndarray>\n",
        "Attributes:\n",
-       "    antenna_name:             ea25\n",
        "    ddi:                      0\n",
-       "    grid_params:              {'cell_size': 0.000719484970141879, 'n_pix': 729}\n",
        "    holog_map_key:            map_0\n",
-       "    l_max:                    0.009675024871632517\n",
-       "    l_min:                    -0.009674930411087166\n",
-       "    m_max:                    0.009698666161358812\n",
-       "    m_min:                    -0.00975229251000802\n",
        "    parallactic_samples:      [5.308157433326323, 5.357028871639436, 5.502977...\n",
-       "    telescope_name:           EVLA\n",
-       "    time_smoothing_interval:  1.0