From 564d5943cecb85619bef7fcb9d9b3e6e056bdc41 Mon Sep 17 00:00:00 2001 From: Christina Hedges Date: Fri, 14 Feb 2025 15:50:43 -0500 Subject: [PATCH] adding capability to search TICA ffis... --- README.rst | 1 + pyproject.toml | 2 +- src/tesscube/__init__.py | 13 ++- src/tesscube/cube.py | 40 +++++++-- src/tesscube/fits.py | 83 ++++++++++-------- src/tesscube/query.py | 184 +++++++++++++++++++++++++++++++++------ src/tesscube/wcs.py | 8 +- 7 files changed, 255 insertions(+), 76 deletions(-) diff --git a/README.rst b/README.rst index 00487de..8bad065 100644 --- a/README.rst +++ b/README.rst @@ -175,6 +175,7 @@ Please include a self-contained example that fully demonstrates your problem or Changelog: ========== + - Added ability to use "TICA" FFIs. This is experimental and might be buggy. - Patch removes the un-needed `fitsio` dependency - Initial v1.0.0 release of `tesscube`. diff --git a/pyproject.toml b/pyproject.toml index 60c6726..e702f8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "tesscube" -version = "1.0.5" +version = "1.1.0" description = "" authors = ["TESS Science Support Center ", "Christina Hedges "] license = "MIT" diff --git a/src/tesscube/__init__.py b/src/tesscube/__init__.py index e72e127..04b1488 100644 --- a/src/tesscube/__init__.py +++ b/src/tesscube/__init__.py @@ -1,10 +1,21 @@ -__version__ = "1.0.5" # Standard library import os # noqa import tempfile PACKAGEDIR = os.path.abspath(os.path.dirname(__file__)) +from importlib.metadata import version, PackageNotFoundError # noqa + + +def get_version(): + try: + return version("tesscube") + except PackageNotFoundError: + return "unknown" + + +__version__ = get_version() + HDR_SIZE = 2880 # bytes BYTES_PER_PIX = 4 # float32 MAX_CONCURRENT_DOWNLOADS = 10 diff --git a/src/tesscube/cube.py b/src/tesscube/cube.py index 68329b0..30ad73b 100644 --- a/src/tesscube/cube.py +++ b/src/tesscube/cube.py @@ -7,7 +7,7 @@ from astropy.time import Time from astropy.wcs.utils import fit_wcs_from_points -from . import BYTES_PER_PIX, DATA_OFFSET, log +from . import BYTES_PER_PIX, DATA_OFFSET, log, HDR_SIZE from .fits import ( get_header_dict, get_output_first_extention_header, @@ -38,9 +38,17 @@ class TESSCube(QueryMixin, WCSMixin): The CCD number (1-4). """ - def __init__(self, sector: int, camera: int, ccd: int): + def __init__(self, sector: int, camera: int, ccd: int, tica=False): self.sector, self.camera, self.ccd = sector, camera, ccd - self.object_key = f"tess/public/mast/tess-s{sector:04}-{camera}-{ccd}-cube.fits" + self.tica = tica + if self.tica: + self.object_key = ( + f"tess/public/mast/tica/tica-s{sector:04}-{camera}-{ccd}-cube.fits" + ) + else: + self.object_key = ( + f"tess/public/mast/tess-s{sector:04}-{camera}-{ccd}-cube.fits" + ) ( self.nsets, self.nframes, @@ -115,6 +123,8 @@ def last_hdu(self): DATA_OFFSET + (self.ncolumns * self.nframes * self.nsets * self.nrows) * BYTES_PER_PIX ) + if self.tica: + end += HDR_SIZE * 4 return get_last_hdu(object_key=self.object_key, end=end) @cached_property @@ -124,15 +134,18 @@ def ffi_names(self): @cached_property def tstart(self): - return self.last_hdu.data["TSTART"] + return self.last_hdu.data["TSTART" if not self.tica else "STARTTJD"] @cached_property def tstop(self): - return self.last_hdu.data["TSTOP"] + return self.last_hdu.data["TSTOP" if not self.tica else "ENDTJD"] @cached_property def telapse(self): - return self.last_hdu.data["TELAPSE"] + if not self.tica: + return self.last_hdu.data["TELAPSE"] + else: + return self.tstop - self.tstart @lru_cache(maxsize=4) def get_ffi( @@ -474,12 +487,18 @@ def time(self): @cached_property def timecorr(self): """Barycentric time correction for the center of the FFI.""" - return self.last_hdu.data["BARYCORR"] + if not self.tica: + return self.last_hdu.data["BARYCORR"] + else: + return self.time * 0 @cached_property def quality(self): """SPOC provided quality flags for each cadence.""" - return self.last_hdu.data["DQUALITY"] + if not self.tica: + return self.last_hdu.data["DQUALITY"] + else: + return self.last_hdu.data["QUAL_BIT"] @cached_property def cadence_number(self): @@ -492,7 +511,10 @@ def cadence_number(self): @cached_property def exposure_time(self): """Exposure time in days""" - return self.last_hdu.data["EXPOSURE"][0] + if not self.tica: + return self.last_hdu.data["EXPOSURE"][0] + else: + return self.last_hdu.data["INT_TIME"] / 86400 @property def shape(self): diff --git a/src/tesscube/fits.py b/src/tesscube/fits.py index 68a073a..c825aac 100644 --- a/src/tesscube/fits.py +++ b/src/tesscube/fits.py @@ -59,23 +59,29 @@ def get_header_dict(cube): ) header_dict["PROCVER"] = fits.Card("PROCVER", __version__, "software version") header_dict["TSTART"] = fits.Card( - "TSTART", - cube.last_hdu.data["TSTART"][0], + "TSTART" if not cube.tica else "STARTTJD", + cube.last_hdu.data["TSTART" if not cube.tica else "STARTTJD"][0], "observation start time in TJD of first FFI", ) header_dict["TSTOP"] = fits.Card( - "TSTOP", - cube.last_hdu.data["TSTOP"][-1], + "TSTOP" if not cube.tica else "ENDTJD", + cube.last_hdu.data["TSTOP" if not cube.tica else "ENDTJD"][-1], "observation stop time in TJD of last FFI", ) header_dict["DATE-OBS"] = fits.Card( "DATE-OBS", - Time(cube.last_hdu.data["TSTART"][0] + 2457000, format="jd").isot, + Time( + cube.last_hdu.data["TSTART" if not cube.tica else "STARTTJD"][0] + 2457000, + format="jd", + ).isot, "TSTART as UTC calendar date", ) header_dict["DATE-END"] = fits.Card( "DATE-END", - Time(cube.last_hdu.data["TSTOP"][-1] + 2457000, format="jd").isot, + Time( + cube.last_hdu.data["TSTOP" if not cube.tica else "ENDTJD"][-1] + 2457000, + format="jd", + ).isot, "TSTOP as UTC calendar date", ) return header_dict @@ -118,8 +124,10 @@ def get_output_primary_hdu(cube): "TICVER", "CRMITEN", "CRBLKSZ", - "CRSPOC", ] + if not cube.tica: + for a in ["CRSPOC"]: + keys.append(a) header_dict = cube.header_dict return fits.PrimaryHDU(header=fits.Header(cards=[header_dict[key] for key in keys])) @@ -135,7 +143,6 @@ def get_output_first_extention_header(cube): "RA_OBJ", "DEC_OBJ", "EQUINOX", - "EXPOSURE", "TIMEREF", "TASSIGN", "TIMESYS", @@ -143,37 +150,43 @@ def get_output_first_extention_header(cube): "BJDREFF", "TIMEUNIT", "TELAPSE", - "LIVETIME", "TSTART", "TSTOP", "DATE-OBS", "DATE-END", - "DEADC", - "TIMEPIXR", - "TIERRELA", - "INT_TIME", - "READTIME", - "FRAMETIM", - "NUM_FRM", - "TIMEDEL", - "BACKAPP", - "DEADAPP", - "VIGNAPP", - "GAINA", - "GAINB", - "GAINC", - "GAIND", - "READNOIA", - "READNOIB", - "READNOIC", - "READNOID", - "NREADOUT", - "FXDOFF", - "MEANBLCA", - "MEANBLCB", - "MEANBLCC", - "MEANBLCD", ] + if not cube.tica: + for a in [ + "EXPOSURE", + "LIVETIME", + "DEADC", + "TIMEPIXR", + "TIERRELA", + "INT_TIME", + "READTIME", + "FRAMETIM", + "NUM_FRM", + "TIMEDEL", + "BACKAPP", + "DEADAPP", + "VIGNAPP", + "GAINA", + "GAINB", + "GAINC", + "GAIND", + "READNOIA", + "READNOIB", + "READNOIC", + "READNOID", + "NREADOUT", + "FXDOFF", + "MEANBLCA", + "MEANBLCB", + "MEANBLCC", + "MEANBLCD", + ]: + keys.append(a) + header_dict = cube.header_dict return fits.Header(cards=[header_dict[key] for key in keys]) @@ -371,7 +384,7 @@ def _fix_primary_hdu(hdu): hdr["TICID"] = (None, "unique tess target identifier") delete_kwds_wildcards = [ - "NAXIS*" "SC_*", + "NAXIS*SC_*", "RMS*", "A_*", "AP_*", diff --git a/src/tesscube/query.py b/src/tesscube/query.py index bd6687f..db41410 100644 --- a/src/tesscube/query.py +++ b/src/tesscube/query.py @@ -15,6 +15,8 @@ from botocore import UNSIGNED from botocore.config import Config +from astropy.time import Time + from . import ( BUCKET_NAME, BYTES_PER_PIX, @@ -366,9 +368,14 @@ async def async_get_flux( nrows=nrows, frame_range=frame_range, ) - values = np.asarray(values).reshape((nrows, ncolumns, nframes, self.nsets)) - flux, flux_err = np.asarray(values).transpose([3, 2, 0, 1]) - return flux, flux_err + if self.nsets == 1: + values = np.asarray(values).reshape((nrows, ncolumns, nframes)) + flux = np.asarray(values).transpose([2, 0, 1]) + return flux, flux * 0 + else: + values = np.asarray(values).reshape((nrows, ncolumns, nframes, self.nsets)) + flux, flux_err = np.asarray(values).transpose([3, 2, 0, 1]) + return flux, flux_err @lru_cache(maxsize=128) def get_flux( @@ -443,29 +450,149 @@ async def async_get_primary_hdu(object_key: str) -> fits.PrimaryHDU: async with get_session().create_client( "s3", config=Config(signature_version=UNSIGNED) ) as s3: - # Retrieve the cube header - response = await s3.get_object( - Bucket=BUCKET_NAME, - Key=object_key, - Range=f"bytes=0-{HDR_SIZE * 2-1}", - ) - first_bytes = await response["Body"].read() - with warnings.catch_warnings(): - # Ignore "File may have been truncated" warning - warnings.simplefilter("ignore", AstropyUserWarning) - with fits.open(BytesIO(first_bytes)) as hdulist: - ( - hdulist[0].header["NAXIS1"], - hdulist[0].header["NAXIS2"], - hdulist[0].header["NAXIS3"], - hdulist[0].header["NAXIS4"], - ) = ( - hdulist[1].header["NAXIS1"], - hdulist[1].header["NAXIS2"], - hdulist[1].header["NAXIS3"], - hdulist[1].header["NAXIS4"], - ) - return hdulist[0] + if "tica" in object_key.lower(): + # Retrieve the cube header + response = await s3.get_object( + Bucket=BUCKET_NAME, + Key=object_key, + Range=f"bytes=0-{(HDR_SIZE * 3) * 2 - 1}", + ) + first_bytes = await response["Body"].read() + with warnings.catch_warnings(): + # Ignore "File may have been truncated" warning + warnings.simplefilter("ignore", AstropyUserWarning) + with fits.open(BytesIO(first_bytes)) as hdulist: + ( + hdulist[0].header["NAXIS1"], + hdulist[0].header["NAXIS2"], + hdulist[0].header["NAXIS3"], + hdulist[0].header["NAXIS4"], + ) = ( + hdulist[1].header["NAXIS1"], + hdulist[1].header["NAXIS2"], + hdulist[1].header["NAXIS3"], + hdulist[1].header["NAXIS4"], + ) + + hdulist[0].header["BJDREFI"] = ( + 2457000, + "integer part of BTJD reference date", + ) + hdulist[0].header["BJDREFF"] = ( + 0.00000000, + "fraction of the day in BTJD reference date", + ) + hdulist[0].header["TIMEUNIT"] = ( + "d", + "time unit for TIME, TSTART and TSTOP", + ) + + # Adding some missing kwds not in TICA (but in Ames-produced SPOC ffis) + hdulist[0].header["EXTVER"] = ( + "1", + "extension version number (not format version)", + ) + hdulist[0].header["SIMDATA"] = ( + False, + "file is based on simulated data", + ) + hdulist[0].header["NEXTEND"] = ( + "2", + "number of standard extensions", + ) + hdulist[0].header["TSTART"] = ( + hdulist[0].header["STARTTJD"], + "observation start time in TJD of first FFI", + ) + hdulist[0].header["TSTOP"] = ( + hdulist[0].header["ENDTJD"], + "observation stop time in TJD of last FFI", + ) + hdulist[0].header["CAMERA"] = ( + hdulist[0].header["CAMNUM"], + "Camera number", + ) + hdulist[0].header["CCD"] = ( + hdulist[0].header["CCDNUM"], + "CCD chip number", + ) + hdulist[0].header["ASTATE"] = ( + None, + "archive state F indicates single orbit processing", + ) + hdulist[0].header["CRMITEN"] = ( + hdulist[0].header["CRM"], + "spacecraft cosmic ray mitigation enabled", + ) + hdulist[0].header["CRBLKSZ"] = ( + None, + "[exposures] s/c cosmic ray mitigation block siz", + ) + hdulist[0].header["FFIINDEX"] = ( + hdulist[0].header["CADENCE"], + "number of FFI cadence interval of first FFI", + ) + hdulist[0].header["DATA_REL"] = ( + None, + "data release version number", + ) + + date_obs = Time( + hdulist[0].header["TSTART"] + hdulist[0].header["BJDREFI"], + format="jd", + ).iso + date_end = Time( + hdulist[0].header["TSTOP"] + hdulist[0].header["BJDREFI"], + format="jd", + ).iso + hdulist[0].header["DATE-OBS"] = ( + date_obs, + "TSTART as UTC calendar date of first FFI", + ) + hdulist[0].header["DATE-END"] = ( + date_end, + "TSTOP as UTC calendar date of last FFI", + ) + + hdulist[0].header["FILEVER"] = (None, "file format version") + hdulist[0].header["RADESYS"] = ( + None, + "reference frame of celestial coordinates", + ) + hdulist[0].header["SCCONFIG"] = ( + None, + "spacecraft configuration ID", + ) + hdulist[0].header["TIMVERSN"] = ( + None, + "OGIP memo number for file format", + ) + + return hdulist[0] + else: + # Retrieve the cube header + response = await s3.get_object( + Bucket=BUCKET_NAME, + Key=object_key, + Range=f"bytes=0-{HDR_SIZE * 2 - 1}", + ) + first_bytes = await response["Body"].read() + with warnings.catch_warnings(): + # Ignore "File may have been truncated" warning + warnings.simplefilter("ignore", AstropyUserWarning) + with fits.open(BytesIO(first_bytes)) as hdulist: + ( + hdulist[0].header["NAXIS1"], + hdulist[0].header["NAXIS2"], + hdulist[0].header["NAXIS3"], + hdulist[0].header["NAXIS4"], + ) = ( + hdulist[1].header["NAXIS1"], + hdulist[1].header["NAXIS2"], + hdulist[1].header["NAXIS3"], + hdulist[1].header["NAXIS4"], + ) + return hdulist[0] async def async_get_last_hdu(object_key: str, end: int) -> fits.PrimaryHDU: @@ -495,11 +622,12 @@ async def async_get_last_hdu(object_key: str, end: int) -> fits.PrimaryHDU: # n_pixels = len(first_bytes) // BYTES_PER_PIX # values = np.asarray(struct.unpack(">" + "f" * n_pixels, first_bytes)) # return values - return fits.open( + hdulist = fits.open( BytesIO(first_bytes.lstrip(b"\x00")), ignore_missing_simple=True, lazy_load_hdus=False, - )[0] + ) + return hdulist[0] async def async_get_ffi(ffi_name: str) -> fits.HDUList: diff --git a/src/tesscube/wcs.py b/src/tesscube/wcs.py index 1e1ae24..165efb3 100644 --- a/src/tesscube/wcs.py +++ b/src/tesscube/wcs.py @@ -56,8 +56,12 @@ def _extract_average_WCS(hdu): value = data[idx] idx += 1 wcs_hdu.header[attr] = value - wcs_hdu.header["WCSAXES"] = int(wcs_hdu.header["WCSAXES"]) - wcs_hdu.header["WCSAXESP"] = int(wcs_hdu.header["WCSAXESP"]) + wcs_hdu.header["WCSAXES"] = int( + wcs_hdu.header["WCSAXES"] if "WCSAXES" in wcs_hdu.header.keys() else 2 + ) + wcs_hdu.header["WCSAXESP"] = int( + wcs_hdu.header["WCSAXESP"] if "WCSAXESP" in wcs_hdu.header.keys() else 2 + ) return WCS(wcs_hdu.header)