diff --git a/.coveragerc b/.coveragerc index 56f1ae3b34..55ea46ef23 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,5 @@ [run] source = - astrodata geminidr gemini_instruments gempy diff --git a/Jenkinsfile b/Jenkinsfile index aceac7b30c..c744fafc9f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -92,7 +92,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "unit_tests_outputs/" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/unit/" } steps { @@ -134,7 +134,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "regression_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/regr/" } steps { @@ -176,7 +176,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "f2_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/f2/" } steps { @@ -213,7 +213,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "gsaoi_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/gsaoi/" } steps { @@ -250,7 +250,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "niri_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/niri/" } steps { @@ -287,7 +287,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "gnirs_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/gnirs/" } steps { @@ -324,7 +324,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "gmos_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/gmos/" } steps { @@ -364,7 +364,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "wavecal_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/wavecal/" } steps { @@ -404,7 +404,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "gmosls_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/gmosls/" } steps { @@ -442,7 +442,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "slow_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/slow/" } steps { @@ -480,7 +480,7 @@ pipeline { environment { MPLBACKEND = "agg" DRAGONS_TEST_OUT = "ghost_tests_outputs" - TOX_ARGS = "astrodata geminidr gemini_instruments gempy recipe_system" + TOX_ARGS = "geminidr gemini_instruments gempy recipe_system" TMPDIR = "${env.WORKSPACE}/.tmp/ghost/" } steps { diff --git a/astrodata/.notempty b/astrodata/.notempty deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/astrodata/.readthedocs.yaml b/astrodata/.readthedocs.yaml deleted file mode 100644 index e80232f0c8..0000000000 --- a/astrodata/.readthedocs.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# .readthedocs.yaml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -# Set the version of Python and other tools you might need -build: - os: ubuntu-22.04 - tools: - python: "3.10" - # You can also specify other tool versions: - # nodejs: "19" - # rust: "1.64" - # golang: "1.19" - -# Build documentation in the docs/ directory with Sphinx -sphinx: - configuration: astrodata/doc/conf.py - -# If using Sphinx, optionally build your docs in additional formats such as PDF -# formats: -# - pdf - -# Optionally declare the Python requirements required to build your docs -python: - install: - - requirements: requirements.txt -# - requirements: docs/requirements.txt \ No newline at end of file diff --git a/astrodata/__init__.py b/astrodata/__init__.py deleted file mode 100644 index ec914d9b9f..0000000000 --- a/astrodata/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -This package add another abstraction layer to astronomical data by parsing -the information contained in the headers as attributes. To do so, -one must subclass :class:`astrodata.AstroData` and add parse methods -accordingly to the :class:`~astrodata.TagSet` received. -""" - -__all__ = ['AstroData', 'AstroDataError', 'TagSet', 'NDAstroData', - 'AstroDataMixin', 'astro_data_descriptor', 'astro_data_tag', - 'open', 'create', '__version__', 'version', 'add_header_to_table', - 'Section'] - - -from .core import AstroData -from .fits import add_header_to_table -from .factory import AstroDataFactory, AstroDataError -from .nddata import NDAstroData, AstroDataMixin -from .utils import * -from ._version import version - -__version__ = version() - -factory = AstroDataFactory() -# Let's make sure that there's at least one class that matches the data -# (if we're dealing with a FITS file) -factory.addClass(AstroData) - -open = factory.getAstroData -create = factory.createFromScratch diff --git a/astrodata/_version.py b/astrodata/_version.py deleted file mode 100644 index 2fd60394bd..0000000000 --- a/astrodata/_version.py +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env python -""" -Holds the DRAGONS version to be propagated throught all the DRAGONS package -and to be used in the documentation. -""" - -# --- Setup Version Here --- -API = 4 -FEATURE = 0 -BUG = 0 -TAG = 'dev' - - -def version(short=False, tag=TAG): - """ - Returns DRAGONS's version based on the api, - feature and bug numbers. - - Returns - ------- - str : formatted version - """ - - if short: - _version = "{:d}.{:d}".format(API, FEATURE) - - else: - _tag = '_{:s}'.format(tag) if tag else '' - _version = "{:d}.{:d}.{:d}".format(API, FEATURE, BUG) + _tag - - return _version diff --git a/astrodata/core.py b/astrodata/core.py deleted file mode 100644 index b24654056f..0000000000 --- a/astrodata/core.py +++ /dev/null @@ -1,1302 +0,0 @@ -import inspect -import logging -import os -import re -import textwrap -import warnings -from collections import OrderedDict -from contextlib import suppress -from copy import copy, deepcopy -from functools import partial - -import numpy as np - -from astropy.io import fits -from astropy.nddata import NDData -from astropy.table import Table -from astropy.utils import format_doc - -from .fits import (DEFAULT_EXTENSION, FitsHeaderCollection, _process_table, - read_fits, write_fits) -from .nddata import ADVarianceUncertainty -from .nddata import NDAstroData as NDDataObject -from .utils import (assign_only_single_slice, astro_data_descriptor, - deprecated, normalize_indices, returns_list) - -NO_DEFAULT = object() - - -_arit_doc = """ - Performs {name} by evaluating ``self {op} operand``. - - Parameters - ---------- - oper : number or object - The operand to perform the operation ``self {op} operand``. - - Returns - -------- - `AstroData` instance -""" - - -class AstroData: - """ - Base class for the AstroData software package. It provides an interface - to manipulate astronomical data sets. - - Parameters - ---------- - nddata : `astrodata.NDAstroData` or list of `astrodata.NDAstroData` - List of NDAstroData objects. - tables : dict[name, `astropy.table.Table`] - Dict of table objects. - phu : `astropy.io.fits.Header` - Primary header. - indices : list of int - List of indices mapping the `astrodata.NDAstroData` objects that this - object will access to. This is used when slicing an object, then the - sliced AstroData will have the ``.nddata`` list from its parent and - access the sliced NDAstroData through this list of indices. - - """ - - # Derived classes may provide their own __keyword_dict. Being a private - # variable, each class will preserve its own, and there's no risk of - # overriding the whole thing - __keyword_dict = { - 'instrument': 'INSTRUME', - 'object': 'OBJECT', - 'telescope': 'TELESCOP', - 'ut_date': 'DATE-OBS' - } - - def __init__(self, nddata=None, tables=None, phu=None, indices=None, - is_single=False): - if nddata is None: - nddata = [] - elif not isinstance(nddata, (list, tuple)): - nddata = [nddata] - - # _all_nddatas contains all the extensions from the original file or - # object. And _indices is used to map extensions for sliced objects. - self._all_nddatas = nddata - self._indices = indices - - self.is_single = is_single - """ If this data provider represents a single slice out of a whole - dataset, return True. Otherwise, return False. """ - - if tables is not None and not isinstance(tables, dict): - raise ValueError('tables must be a dict') - self._tables = tables or {} - - self._phu = phu or fits.Header() - self._fixed_settable = {'data', 'uncertainty', 'mask', 'variance', - 'wcs', 'path', 'filename'} - self._logger = logging.getLogger(__name__) - self._orig_filename = None - self._path = None - - def __copy__(self): - """ - Returns a shallow copy of this instance. - """ - obj = self.__class__() - - for attr in ('_phu', '_path', '_orig_filename', '_tables'): - obj.__dict__[attr] = self.__dict__[attr] - - # A new list containing references (rather than a reference to the list) - obj.__dict__['_all_nddatas'] = [nd for nd in self._nddata] - - # If we're copying a single-slice AD, then make its NDAstroData a copy - # so attributes can be modified without affecting the original - if self.is_single: - obj.__dict__['_all_nddatas'][self._indices[0]] = copy(self.nddata) - - return obj - - def __deepcopy__(self, memo): - """ - Returns a new instance of this class. - - Parameters - ---------- - memo : dict - See the documentation on `deepcopy` for an explanation on how - this works. - - """ - obj = self.__class__() - - for attr in ('_phu', '_path', '_orig_filename', '_tables'): - obj.__dict__[attr] = deepcopy(self.__dict__[attr]) - - obj.__dict__['_all_nddatas'] = [deepcopy(nd) for nd in self._nddata] - return obj - - def _keyword_for(self, name): - """ - Returns the FITS keyword name associated to ``name``. - - Parameters - ---------- - name : str - The common "key" name for which we want to know the associated - FITS keyword. - - Returns - ------- - str - The desired keyword name. - - Raises - ------ - AttributeError - If there is no keyword for the specified ``name``. - - """ - for cls in self.__class__.mro(): - with suppress(AttributeError, KeyError): - # __keyword_dict is a mangled variable - return getattr(self, f'_{cls.__name__}__keyword_dict')[name] - else: - raise AttributeError(f"No match for '{name}'") - - def _process_tags(self): - """Return the tag set (as a set of str) for the current instance.""" - results = [] - # Calling inspect.getmembers on `self` would trigger all the - # properties (tags, phu, hdr, etc.), and that's undesirable. To - # prevent that, we'll inspect the *class*. - filt = lambda x: hasattr(x, 'tag_method') - for _, method in inspect.getmembers(self.__class__, filt): - ts = method(self) - if ts.add or ts.remove or ts.blocks: - results.append(ts) - - # Sort by the length of substractions... those that substract - # from others go first - results = sorted(results, key=lambda x: len(x.remove) + len(x.blocks), - reverse=True) - - # Sort by length of blocked_by, those that are never disabled go first - results = sorted(results, key=lambda x: len(x.blocked_by)) - - # Sort by length of if_present... those that need other tags to - # be present go last - results = sorted(results, key=lambda x: len(x.if_present)) - - tags = set() - removals = set() - blocked = set() - for plus, minus, blocked_by, blocks, is_present in results: - if is_present: - # If this TagSet requires other tags to be present, make - # sure that all of them are. Otherwise, skip... - if len(tags & is_present) != len(is_present): - continue - allowed = (len(tags & blocked_by) + len(plus & blocked)) == 0 - if allowed: - # This set is not being blocked by others... - removals.update(minus) - tags.update(plus - removals) - blocked.update(blocks) - - return tags - - @staticmethod - def _matches_data(source): - # This one is trivial. Will be more specific for subclasses. - return True - - @property - def path(self): - """Return the file path.""" - return self._path - - @path.setter - def path(self, value): - if self._path is None and value is not None: - self._orig_filename = os.path.basename(value) - self._path = value - - @property - def filename(self): - """Return the file name.""" - if self.path is not None: - return os.path.basename(self.path) - - @filename.setter - def filename(self, value): - if os.path.isabs(value): - raise ValueError("Cannot set the filename to an absolute path!") - elif self.path is None: - self.path = os.path.abspath(value) - else: - dirname = os.path.dirname(self.path) - self.path = os.path.join(dirname, value) - - @property - def orig_filename(self): - """Return the original file name (before it was modified).""" - return self._orig_filename - - @orig_filename.setter - def orig_filename(self, value): - self._orig_filename = value - - @property - def phu(self): - """Return the primary header.""" - return self._phu - - @phu.setter - def phu(self, phu): - self._phu = phu - - @property - def hdr(self): - """Return all headers, as a `astrodata.fits.FitsHeaderCollection`.""" - if not self.nddata: - return None - headers = [nd.meta['header'] for nd in self._nddata] - return headers[0] if self.is_single else FitsHeaderCollection(headers) - - @property - @deprecated("Access to headers through this property is deprecated and " - "will be removed in the future. Use '.hdr' instead.") - def header(self): - return [self.phu] + [ndd.meta['header'] for ndd in self._nddata] - - @property - def tags(self): - """A set of strings that represent the tags defining this instance.""" - return self._process_tags() - - @property - def descriptors(self): - """ - Returns a sequence of names for the methods that have been - decorated as descriptors. - - Returns - -------- - tuple of str - """ - members = inspect.getmembers(self.__class__, - lambda x: hasattr(x, 'descriptor_method')) - return tuple(mname for (mname, method) in members) - - @property - def id(self): - """Returns the extension identifier (1-based extension number) - for sliced objects. - """ - if self.is_single: - return self._indices[0] + 1 - else: - raise ValueError("Cannot return id for an AstroData object " - "that is not a single slice") - - @property - def indices(self): - """Returns the extensions indices for sliced objects.""" - return self._indices if self._indices else list(range(len(self))) - - @property - def is_sliced(self): - """ - If this data provider instance represents the whole dataset, return - False. If it represents a slice out of the whole, return True. - """ - return self._indices is not None - - def is_settable(self, attr): - """Return True if the attribute is meant to be modified.""" - if self.is_sliced and attr in {'path', 'filename'}: - return False - return attr in self._fixed_settable or attr.isupper() - - @property - def _nddata(self): - """Return the list of `astrodata.NDAstroData` objects. Contrary to - ``self.nddata`` this always returns a list. - """ - if self._indices is not None: - return [self._all_nddatas[i] for i in self._indices] - else: - return self._all_nddatas - - @property - def nddata(self): - """Return the list of `astrodata.NDAstroData` objects. - - If the `AstroData` object is sliced, this returns only the NDData - objects of the sliced extensions. And if this is a single extension - object, the NDData object is returned directly (i.e. not a list). - - """ - return self._nddata[0] if self.is_single else self._nddata - - def table(self): - # FIXME: do we need this in addition to .tables ? - return self._tables.copy() - - @property - def tables(self): - """Return the names of the `astropy.table.Table` objects associated to - the top-level object. - """ - return set(self._tables) - - @property - def ext_tables(self): - """Return the names of the `astropy.table.Table` objects associated to - an extension. - """ - if not self.is_single: - raise AttributeError('this is only available for extensions') - return set(key for key, obj in self.nddata.meta['other'].items() - if isinstance(obj, Table)) - - @property - @returns_list - def shape(self): - return [nd.shape for nd in self._nddata] - - @property - @returns_list - def data(self): - """ - A list of the arrays (or single array, if this is a single slice) - corresponding to the science data attached to each extension. - """ - return [nd.data for nd in self._nddata] - - @data.setter - @assign_only_single_slice - def data(self, value): - # Setting the ._data in the NDData is a bit kludgy, but we're all - # grown adults and know what we're doing, isn't it? - if hasattr(value, 'shape'): - self.nddata._data = value - else: - raise AttributeError("Trying to assign data to be something " - "with no shape") - - @property - @returns_list - def uncertainty(self): - """ - A list of the uncertainty objects (or a single object, if this is - a single slice) attached to the science data, for each extension. - - The objects are instances of AstroPy's `astropy.nddata.NDUncertainty`, - or `None` where no information is available. - - See also - -------- - variance : The actual array supporting the uncertainty object. - - """ - return [nd.uncertainty for nd in self._nddata] - - @uncertainty.setter - @assign_only_single_slice - def uncertainty(self, value): - self.nddata.uncertainty = value - - @property - @returns_list - def mask(self): - """ - A list of the mask arrays (or a single array, if this is a single - slice) attached to the science data, for each extension. - - For objects that miss a mask, `None` will be provided instead. - """ - return [nd.mask for nd in self._nddata] - - @mask.setter - @assign_only_single_slice - def mask(self, value): - self.nddata.mask = value - - @property - @returns_list - def variance(self): - """ - A list of the variance arrays (or a single array, if this is a single - slice) attached to the science data, for each extension. - - For objects that miss uncertainty information, `None` will be provided - instead. - - See also - --------- - uncertainty : The uncertainty objects used under the hood. - - """ - return [nd.variance for nd in self._nddata] - - @variance.setter - @assign_only_single_slice - def variance(self, value): - if value is None: - self.nddata.uncertainty = None - else: - self.nddata.uncertainty = ADVarianceUncertainty(value) - - @property - def wcs(self): - """Returns the list of WCS objects for each extension.""" - if self.is_single: - return self.nddata.wcs - else: - raise ValueError("Cannot return WCS for an AstroData object " - "that is not a single slice") - - @wcs.setter - @assign_only_single_slice - def wcs(self, value): - self.nddata.wcs = value - - def __iter__(self): - if self.is_single: - yield self - else: - for n in range(len(self)): - yield self[n] - - def __getitem__(self, idx): - """ - Returns a sliced view of the instance. It supports the standard - Python indexing syntax. - - Parameters - ---------- - slice : int, `slice` - An integer or an instance of a Python standard `slice` object - - Raises - ------- - TypeError - If trying to slice an object when it doesn't make sense (e.g. - slicing a single slice) - ValueError - If `slice` does not belong to one of the recognized types - IndexError - If an index is out of range - - """ - if self.is_single: - raise TypeError("Can't slice a single slice!") - - indices, multiple = normalize_indices(idx, nitems=len(self)) - if self._indices: - indices = [self._indices[i] for i in indices] - - is_single = not isinstance(idx, (tuple, slice)) - obj = self.__class__(self._all_nddatas, - tables=self._tables, - phu=self.phu, - indices=indices, - is_single=is_single) - obj._path = self.path - obj._orig_filename = self.orig_filename - return obj - - def __delitem__(self, idx): - """ - Called to implement deletion of ``self[idx]``. Supports standard - Python syntax (including negative indices). - - Parameters - ---------- - idx : int - This index represents the order of the element that you want - to remove. - - Raises - ------- - IndexError - If `idx` is out of range. - - """ - if self.is_sliced: - raise TypeError("Can't remove items from a sliced object") - del self._all_nddatas[idx] - - def __getattr__(self, attribute): - """ - Called when an attribute lookup has not found the attribute in the - usual places (not an instance attribute, and not in the class tree - for ``self``). - - Parameters - ---------- - attribute : str - The attribute's name. - - Raises - ------- - AttributeError - If the attribute could not be found/computed. - - """ - # I we're working with single slices, let's look some things up - # in the ND object - if self.is_single and attribute.isupper(): - with suppress(KeyError): - return self.nddata.meta['other'][attribute] - - if attribute in self._tables: - return self._tables[attribute] - - raise AttributeError(f"{self.__class__.__name__!r} object has no " - f"attribute {attribute!r}") - - def __setattr__(self, attribute, value): - """ - Called when an attribute assignment is attempted, instead of the - normal mechanism. - - Parameters - ---------- - attribute : str - The attribute's name. - value : object - The value to be assigned to the attribute. - - """ - - def _my_attribute(attr): - return attr in self.__dict__ or attr in self.__class__.__dict__ - - if (attribute.isupper() and self.is_settable(attribute) and - not _my_attribute(attribute)): - # This method is meant to let the user set certain attributes of - # the NDData objects. First we check if the attribute belongs to - # this object's dictionary. Otherwise, see if we can pass it down. - # - if self.is_sliced and not self.is_single: - raise TypeError("This attribute can only be " - "assigned to a single-slice object") - - if attribute == DEFAULT_EXTENSION: - raise AttributeError(f"{attribute} extensions should be " - "appended with .append") - elif attribute in {'DQ', 'VAR'}: - raise AttributeError(f"{attribute} should be set on the " - "nddata object") - - add_to = self.nddata if self.is_single else None - self._append(value, name=attribute, add_to=add_to) - return - - super().__setattr__(attribute, value) - - def __delattr__(self, attribute): - """Implements attribute removal.""" - if not attribute.isupper(): - super().__delattr__(attribute) - return - - if self.is_sliced: - if not self.is_single: - raise TypeError("Can't delete attributes on non-single slices") - - other = self.nddata.meta['other'] - if attribute in other: - del other[attribute] - else: - raise AttributeError(f"{self.__class__.__name__!r} sliced " - "object has no attribute {attribute!r}") - else: - if attribute in self._tables: - del self._tables[attribute] - else: - raise AttributeError(f"'{attribute}' is not a global table " - "for this instance") - - def __contains__(self, attribute): - """ - Implements the ability to use the ``in`` operator with an - `AstroData` object. - - Parameters - ---------- - attribute : str - An attribute name. - - Returns - -------- - bool - """ - return attribute in self.exposed - - def __len__(self): - """Return the number of independent extensions stored by the object. - """ - if self._indices is not None: - return len(self._indices) - else: - return len(self._all_nddatas) - - @property - def exposed(self): - """ - A collection of strings with the names of objects that can be accessed - directly by name as attributes of this instance, and that are not part - of its standard interface (i.e. data objects that have been added - dynamically). - - Examples - --------- - >>> ad[0].exposed # doctest: +SKIP - set(['OBJMASK', 'OBJCAT']) - - """ - exposed = set(self._tables) - if self.is_single: - exposed |= set(self.nddata.meta['other']) - return exposed - - def _pixel_info(self): - for idx, nd in enumerate(self._nddata): - other_objects = [] - uncer = nd.uncertainty - fixed = (('variance', None if uncer is None else uncer), - ('mask', nd.mask)) - for name, other in fixed + tuple(sorted(nd.meta['other'].items())): - if other is None: - continue - if isinstance(other, Table): - other_objects.append(dict( - attr=name, - type='Table', - dim=str((len(other), len(other.columns))), - data_type='n/a' - )) - else: - dim = '' - if hasattr(other, 'dtype'): - dt = other.dtype.name - dim = str(other.shape) - elif hasattr(other, 'data'): - dt = other.data.dtype.name - dim = str(other.data.shape) - elif hasattr(other, 'array'): - dt = other.array.dtype.name - dim = str(other.array.shape) - else: - dt = 'unknown' - other_objects.append(dict( - attr=name, - type=type(other).__name__, - dim=dim, - data_type=dt - )) - - yield dict( - idx='[{:2}]'.format(idx), - main=dict( - content='science', - type=type(nd).__name__, - dim=str(nd.data.shape), - data_type=nd.data.dtype.name - ), - other=other_objects - ) - - def info(self): - """Prints out information about the contents of this instance.""" - - print("Filename: {}".format(self.path if self.path else "Unknown")) - # This is fixed. We don't support opening for update - # print("Mode: readonly") - - text = 'Tags: ' + ' '.join(sorted(self.tags)) - textwrapper = textwrap.TextWrapper(width=80, subsequent_indent=' ') - for line in textwrapper.wrap(text): - print(line) - - if len(self) > 0: - main_fmt = "{:6} {:24} {:17} {:14} {}" - other_fmt = " .{:20} {:17} {:14} {}" - print("\nPixels Extensions") - print(main_fmt.format("Index", "Content", "Type", "Dimensions", - "Format")) - for pi in self._pixel_info(): - main_obj = pi['main'] - print(main_fmt.format( - pi['idx'], main_obj['content'][:24], main_obj['type'][:17], - main_obj['dim'], main_obj['data_type'])) - for other in pi['other']: - print(other_fmt.format( - other['attr'][:20], other['type'][:17], other['dim'], - other['data_type'])) - - # NOTE: This covers tables, only. Study other cases before - # implementing a more general solution - if self._tables: - print("\nOther Extensions") - print(" Type Dimensions") - for name, table in sorted(self._tables.items()): - if type(table) is list: - # This is not a free floating table - continue - print(".{:13} {:11} {}".format( - name[:13], 'Table', (len(table), len(table.columns)))) - - def _oper(self, operator, operand): - ind = self.indices - ndd = self._all_nddatas - if isinstance(operand, AstroData): - if len(operand) != len(self): - raise ValueError("Operands are not the same size") - for n in range(len(self)): - try: - data = (operand.nddata if operand.is_single - else operand.nddata[n]) - ndd[ind[n]] = operator(ndd[ind[n]], data) - except TypeError: - # This may happen if operand is a sliced, single - # AstroData object - ndd[ind[n]] = operator(ndd[ind[n]], operand.nddata) - op_table = operand.table() - ltab, rtab = set(self._tables), set(op_table) - for tab in (rtab - ltab): - self._tables[tab] = op_table[tab] - else: - for n in range(len(self)): - ndd[ind[n]] = operator(ndd[ind[n]], operand) - - def _standard_nddata_op(self, fn, operand): - return self._oper(partial(fn, handle_mask=np.bitwise_or, - handle_meta='first_found'), operand) - - @format_doc(_arit_doc, name='addition', op='+') - def __add__(self, oper): - copy = deepcopy(self) - copy += oper - return copy - - @format_doc(_arit_doc, name='subtraction', op='-') - def __sub__(self, oper): - copy = deepcopy(self) - copy -= oper - return copy - - @format_doc(_arit_doc, name='multiplication', op='*') - def __mul__(self, oper): - copy = deepcopy(self) - copy *= oper - return copy - - @format_doc(_arit_doc, name='division', op='/') - def __truediv__(self, oper): - copy = deepcopy(self) - copy /= oper - return copy - - @format_doc(_arit_doc, name='inplace addition', op='+=') - def __iadd__(self, oper): - self._standard_nddata_op(NDDataObject.add, oper) - return self - - @format_doc(_arit_doc, name='inplace subtraction', op='-=') - def __isub__(self, oper): - self._standard_nddata_op(NDDataObject.subtract, oper) - return self - - @format_doc(_arit_doc, name='inplace multiplication', op='*=') - def __imul__(self, oper): - self._standard_nddata_op(NDDataObject.multiply, oper) - return self - - @format_doc(_arit_doc, name='inplace division', op='/=') - def __itruediv__(self, oper): - self._standard_nddata_op(NDDataObject.divide, oper) - return self - - add = __iadd__ - subtract = __isub__ - multiply = __imul__ - divide = __itruediv__ - - __radd__ = __add__ - __rmul__ = __mul__ - - def __rsub__(self, oper): - copy = (deepcopy(self) - oper) * -1 - return copy - - def _rdiv(self, ndd, operand): - # Divide method works with the operand first - return NDDataObject.divide(operand, ndd) - - def __rtruediv__(self, oper): - obj = deepcopy(self) - obj._oper(obj._rdiv, oper) - return obj - - def _process_pixel_plane(self, pixim, name=None, top_level=False, - custom_header=None): - # Assume that we get an ImageHDU or something that can be - # turned into one - if isinstance(pixim, fits.ImageHDU): - nd = NDDataObject(pixim.data, meta={'header': pixim.header}) - elif isinstance(pixim, NDDataObject): - nd = pixim - else: - nd = NDDataObject(pixim) - - if custom_header is not None: - nd.meta['header'] = custom_header - - header = nd.meta.setdefault('header', fits.Header()) - currname = header.get('EXTNAME') - - if currname is None: - header['EXTNAME'] = name if name is not None else DEFAULT_EXTENSION - - if top_level: - nd.meta.setdefault('other', OrderedDict()) - - return nd - - def _append_array(self, data, name=None, header=None, add_to=None): - if name in {'DQ', 'VAR'}: - raise ValueError(f"'{name}' need to be associated to a " - f"'{DEFAULT_EXTENSION}' one") - - if add_to is None: - # Top level extension - if name is not None: - hname = name - elif header is not None: - hname = header.get('EXTNAME', DEFAULT_EXTENSION) - else: - hname = DEFAULT_EXTENSION - - hdu = fits.ImageHDU(data, header=header) - hdu.header['EXTNAME'] = hname - ret = self._append_imagehdu(hdu, name=hname, header=None, - add_to=None) - else: - ret = add_to.meta['other'][name] = data - - return ret - - def _append_imagehdu(self, hdu, name, header, add_to): - if name in {'DQ', 'VAR'} or add_to is not None: - return self._append_array(hdu.data, name=name, add_to=add_to) - else: - nd = self._process_pixel_plane(hdu, name=name, top_level=True, - custom_header=header) - return self._append_nddata(nd, name, add_to=None) - - def _append_raw_nddata(self, raw_nddata, name, header, add_to): - # We want to make sure that the instance we add is whatever we specify - # as NDDataObject, instead of the random one that the user may pass - top_level = add_to is None - if not isinstance(raw_nddata, NDDataObject): - raw_nddata = NDDataObject(raw_nddata) - processed_nddata = self._process_pixel_plane(raw_nddata, - top_level=top_level, - custom_header=header) - return self._append_nddata(processed_nddata, name=name, add_to=add_to) - - def _append_nddata(self, new_nddata, name, add_to): - # NOTE: This method is only used by others that have constructed NDData - # according to our internal format. We don't accept new headers at this - # point, and that's why it's missing from the signature. 'name' is - # ignored. It's there just to comply with the _append_XXX signature. - if add_to is not None: - raise TypeError("You can only append NDData derived instances " - "at the top level") - - hd = new_nddata.meta['header'] - hname = hd.get('EXTNAME', DEFAULT_EXTENSION) - if hname == DEFAULT_EXTENSION: - self._all_nddatas.append(new_nddata) - else: - raise ValueError("Arbitrary image extensions can only be added " - f"in association to a '{DEFAULT_EXTENSION}'") - - return new_nddata - - def _append_table(self, new_table, name, header, add_to): - tb = _process_table(new_table, name, header) - hname = tb.meta['header'].get('EXTNAME') - - def find_next_num(tables): - table_num = 1 - while f'TABLE{table_num}' in tables: - table_num += 1 - return f'TABLE{table_num}' - - if add_to is None: - # Find table names for all extensions - ext_tables = set() - for nd in self._nddata: - ext_tables |= set(key for key, obj in nd.meta['other'].items() - if isinstance(obj, Table)) - - if hname is None: - hname = find_next_num(set(self._tables) | ext_tables) - elif hname in ext_tables: - raise ValueError(f"Cannot append table '{hname}' because it " - "would hide an extension table") - - self._tables[hname] = tb - else: - if hname in self._tables: - raise ValueError(f"Cannot append table '{hname}' because it " - "would hide a top-level table") - - add_to.meta['other'][hname] = tb - return tb - - def _append_astrodata(self, ad, name, header, add_to): - if not ad.is_single: - raise ValueError("Cannot append AstroData instances that are " - "not single slices") - elif add_to is not None: - raise ValueError("Cannot append an AstroData slice to " - "another slice") - - new_nddata = deepcopy(ad.nddata) - if header is not None: - new_nddata.meta['header'] = deepcopy(header) - - return self._append_nddata(new_nddata, name=None, add_to=None) - - def _append(self, ext, name=None, header=None, add_to=None): - """ - Internal method to dispatch to the type specific methods. This is - called either by ``.append`` to append on top-level objects only or - by ``__setattr__``. In the second case ``name`` cannot be None, so - this is always the case when appending to extensions (add_to != None). - """ - dispatcher = ( - (NDData, self._append_raw_nddata), - ((Table, fits.TableHDU, fits.BinTableHDU), self._append_table), - (fits.ImageHDU, self._append_imagehdu), - (AstroData, self._append_astrodata), - ) - - for bases, method in dispatcher: - if isinstance(ext, bases): - return method(ext, name=name, header=header, add_to=add_to) - - # Assume that this is an array for a pixel plane - return self._append_array(ext, name=name, header=header, add_to=add_to) - - def append(self, ext, name=None, header=None): - """ - Adds a new top-level extension. - - Parameters - ---------- - ext : array, `astropy.nddata.NDData`, `astropy.table.Table`, other - The contents for the new extension. The exact accepted types depend - on the class implementing this interface. Implementations specific - to certain data formats may accept specialized types (eg. a FITS - provider will accept an `astropy.io.fits.ImageHDU` and extract the - array out of it). - name : str, optional - A name that may be used to access the new object, as an attribute - of the provider. The name is typically ignored for top-level - (global) objects, and required for the others. If the name cannot - be derived from the metadata associated to ``ext``, you will - have to provider one. - It can consist in a combination of numbers and letters, with the - restriction that the letters have to be all capital, and the first - character cannot be a number ("[A-Z][A-Z0-9]*"). - - Returns - -------- - The same object, or a new one, if it was necessary to convert it to - a more suitable format for internal use. - - Raises - ------- - TypeError - If adding the object in an invalid situation (eg. ``name`` is - `None` when adding to a single slice). - ValueError - Raised if the extension is of a proper type, but its value is - illegal somehow. - - """ - if self.is_sliced: - raise TypeError("Can't append objects to slices, use " - "'ext.NAME = obj' instead") - - # NOTE: Most probably, if we want to copy the input argument, we - # should do it here... - if isinstance(ext, fits.PrimaryHDU): - raise ValueError("Only one Primary HDU allowed. " - "Use .phu if you really need to set one") - elif isinstance(ext, Table): - raise ValueError("Tables should be set directly as attribute, " - "i.e. 'ad.MYTABLE = table'") - - if name is not None and not name.isupper(): - warnings.warn(f"extension name '{name}' should be uppercase", - UserWarning) - name = name.upper() - - return self._append(ext, name=name, header=header) - - @classmethod - def read(cls, source, extname_parser=None): - """Read from a file, file object, HDUList, etc.""" - return read_fits(cls, source, extname_parser=extname_parser) - - load = read # for backward compatibility - - def write(self, filename=None, overwrite=False): - """ - Write the object to disk. - - Parameters - ---------- - filename : str, optional - If the filename is not given, ``self.path`` is used. - overwrite : bool - If True, overwrites existing file. - - """ - if filename is None: - if self.path is None: - raise ValueError("A filename needs to be specified") - filename = self.path - - write_fits(self, filename, overwrite=overwrite) - - def operate(self, operator, *args, **kwargs): - """ - Applies a function to the main data array on each extension, replacing - the data with the result. The data will be passed as the first argument - to the function. - - It will be applied to the mask and variance of each extension, too, if - they exist. - - This is a convenience method, which is equivalent to:: - - for ext in ad: - ext.data = operator(ext.data, *args, **kwargs) - if ext.mask is not None: - ext.mask = operator(ext.mask, *args, **kwargs) - if ext.variance is not None: - ext.variance = operator(ext.variance, *args, **kwargs) - - with the additional advantage that it will work on single slices, too. - - Parameters - ---------- - operator : callable - A function that takes an array (and, maybe, other arguments) - and returns an array. - args, kwargs : optional - Additional arguments to be passed to the ``operator``. - - Examples - --------- - >>> import numpy as np - >>> ad.operate(np.squeeze) # doctest: +SKIP - - """ - # Ensure we can iterate, even on a single slice - for ext in [self] if self.is_single else self: - ext.data = operator(ext.data, *args, **kwargs) - if ext.mask is not None: - ext.mask = operator(ext.mask, *args, **kwargs) - if ext.variance is not None: - ext.variance = operator(ext.variance, *args, **kwargs) - - def reset(self, data, mask=NO_DEFAULT, variance=NO_DEFAULT, check=True): - """ - Sets the ``.data``, and optionally ``.mask`` and ``.variance`` - attributes of a single-extension AstroData slice. This function will - optionally check whether these attributes have the same shape. - - Parameters - ---------- - data : ndarray - The array to assign to the ``.data`` attribute ("SCI"). - mask : ndarray, optional - The array to assign to the ``.mask`` attribute ("DQ"). - variance: ndarray, optional - The array to assign to the ``.variance`` attribute ("VAR"). - check: bool - If set, then the function will check that the mask and variance - arrays have the same shape as the data array. - - Raises - ------- - TypeError - if an attempt is made to set the .mask or .variance attributes - with something other than an array - ValueError - if the .mask or .variance attributes don't have the same shape as - .data, OR if this is called on an AD instance that isn't a single - extension slice - - """ - if not self.is_single: - raise ValueError("Trying to reset a non-sliced AstroData object") - - # In case data is an NDData object - try: - self.data = data.data - except AttributeError: - self.data = data - # Set mask, with checking if required - try: - if mask.shape != self.data.shape and check: - raise ValueError("Mask shape incompatible with data shape") - except AttributeError: - if mask is None: - self.mask = mask - elif mask == NO_DEFAULT: - if hasattr(data, 'mask'): - self.mask = data.mask - else: - raise TypeError("Attempt to set mask inappropriately") - else: - self.mask = mask - # Set variance, with checking if required - try: - if variance.shape != self.data.shape and check: - raise ValueError("Variance shape incompatible with data shape") - except AttributeError: - if variance is None: - self.uncertainty = None - elif variance == NO_DEFAULT: - if hasattr(data, 'uncertainty'): - self.uncertainty = data.uncertainty - else: - raise TypeError("Attempt to set variance inappropriately") - else: - self.variance = variance - - if hasattr(data, 'wcs'): - self.wcs = data.wcs - - def update_filename(self, prefix=None, suffix=None, strip=False): - """ - Update the "filename" attribute of the AstroData object. - - A prefix and/or suffix can be specified. If ``strip=True``, these will - replace the existing prefix/suffix; if ``strip=False``, they will - simply be prepended/appended. - - The current filename is broken down into its existing prefix, root, - and suffix using the ``ORIGNAME`` phu keyword, if it exists and is - contained within the current filename. Otherwise, the filename is - split at the last underscore and the part before is assigned as the - root and the underscore and part after the suffix. No prefix is - assigned. - - Note that, if ``strip=True``, a prefix or suffix will only be stripped - if '' is specified. - - Parameters - ---------- - prefix: str, optional - New prefix (None => leave alone) - suffix: str, optional - New suffix (None => leave alone) - strip: bool, optional - Strip existing prefixes and suffixes if new ones are given? - - """ - if self.filename is None: - if 'ORIGNAME' in self.phu: - self.filename = self.phu['ORIGNAME'] - else: - raise ValueError("A filename needs to be set before it " - "can be updated") - - # Set the ORIGNAME keyword if it's not there - if 'ORIGNAME' not in self.phu: - self.phu.set('ORIGNAME', self.orig_filename, - 'Original filename prior to processing') - - if strip: - root, filetype = os.path.splitext(self.phu['ORIGNAME']) - filename, filetype = os.path.splitext(self.filename) - m = re.match('(.*){}(.*)'.format(re.escape(root)), filename) - # Do not strip a prefix/suffix unless a new one is provided - if m: - if prefix is None: - prefix = m.groups()[0] - existing_suffix = m.groups()[1] - if '_' in existing_suffix: - last_underscore = existing_suffix.rfind("_") - root += existing_suffix[:last_underscore] - existing_suffix = existing_suffix[last_underscore:] - else: - try: - root, existing_suffix = filename.rsplit("_", 1) - existing_suffix = "_" + existing_suffix - except ValueError: - root, existing_suffix = filename, '' - if suffix is None: - suffix = existing_suffix - else: - root, filetype = os.path.splitext(self.filename) - - # Cope with prefix or suffix as None - self.filename = (prefix or '') + root + (suffix or '') + filetype - - def _crop_nd(self, nd, x1, y1, x2, y2): - nd.data = nd.data[y1:y2+1, x1:x2+1] - if nd.uncertainty is not None: - nd.uncertainty = nd.uncertainty[y1:y2+1, x1:x2+1] - if nd.mask is not None: - nd.mask = nd.mask[y1:y2+1, x1:x2+1] - - def crop(self, x1, y1, x2, y2): - """Crop the NDData objects given indices. - - Parameters - ---------- - x1, y1, x2, y2 : int - Minimum and maximum indices for the x and y axis. - - """ - # TODO: Consider cropping of objects in the meta section - for nd in self._nddata: - orig_shape = nd.data.shape - self._crop_nd(nd, x1, y1, x2, y2) - for o in nd.meta['other'].values(): - try: - if o.shape == orig_shape: - self._crop_nd(o, x1, y1, x2, y2) - except AttributeError: - # No 'shape' attribute in the object. It's probably - # not array-like - pass - - @astro_data_descriptor - def instrument(self): - """Returns the name of the instrument making the observation.""" - return self.phu.get(self._keyword_for('instrument')) - - @astro_data_descriptor - def object(self): - """Returns the name of the object being observed.""" - return self.phu.get(self._keyword_for('object')) - - @astro_data_descriptor - def telescope(self): - """Returns the name of the telescope.""" - return self.phu.get(self._keyword_for('telescope')) diff --git a/astrodata/doc/.readthedocs.yaml b/astrodata/doc/.readthedocs.yaml deleted file mode 100644 index 02f76b790d..0000000000 --- a/astrodata/doc/.readthedocs.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# .readthedocs.yaml -# Read the Docs configuration file -# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details - -# Required -version: 2 - -# Set the version of Python and other tools you might need -build: - os: ubuntu-22.04 - tools: - python: "3.10" - # You can also specify other tool versions: - # nodejs: "19" - # rust: "1.64" - # golang: "1.19" - -# Build documentation in the docs/ directory with Sphinx -sphinx: - configuration: astrodata/doc/conf.py - -# If using Sphinx, optionally build your docs in additional formats such as PDF -# formats: -# - pdf - -# Optionally declare the Python requirements required to build your docs -python: - install: - - requirements: requirements.txt - - requirements: doc/requirements.txt - - method: pip - path: . \ No newline at end of file diff --git a/astrodata/doc/Makefile b/astrodata/doc/Makefile deleted file mode 100644 index 241432216c..0000000000 --- a/astrodata/doc/Makefile +++ /dev/null @@ -1,206 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " dummy to check syntax errors of document sources" - -.PHONY: clean -clean: - rm -rf $(BUILDDIR)/ api/ - -.PHONY: html -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -.PHONY: dirhtml -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -.PHONY: singlehtml -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -.PHONY: pickle -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -.PHONY: json -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -.PHONY: htmlhelp -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -.PHONY: qthelp -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/AstrodataProgrammersManual.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/AstrodataProgrammersManual.qhc" - -.PHONY: devhelp -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/AstrodataProgrammersManual" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/AstrodataProgrammersManual" - @echo "# devhelp" - -.PHONY: epub -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -.PHONY: latex -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -.PHONY: latexpdf -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: latexpdfja -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: text -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -.PHONY: man -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -.PHONY: texinfo -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -.PHONY: info -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -.PHONY: gettext -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -.PHONY: changes -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -.PHONY: linkcheck -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -.PHONY: doctest -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -.PHONY: xml -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -.PHONY: pseudoxml -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." - -.PHONY: dummy -dummy: - $(SPHINXBUILD) -b dummy $(ALLSPHINXOPTS) $(BUILDDIR)/dummy - @echo - @echo "Build finished. Dummy builder generates no files." diff --git a/astrodata/doc/_static/rtd_theme_overrides.css b/astrodata/doc/_static/rtd_theme_overrides.css deleted file mode 100644 index 62910a0bb7..0000000000 --- a/astrodata/doc/_static/rtd_theme_overrides.css +++ /dev/null @@ -1,20 +0,0 @@ -/* override RTD table width restrictions */ - -@media screen and (min-width: 767px) { - - .wy-table-responsive table td, .wy-table-responsive table th { - /* !important prevents the common CSS stylesheets from - overriding this as on RTD the are loaded after this stylesheet */ - white-space: normal !important; - } - - .wy-table-responsive { - /* margin-bottom: 24px; */ - /* max-width: 100%; */ - overflow: visible !important; - } - - .wy-nav-content { - max-width: 1200px !important; - } -} diff --git a/astrodata/doc/_static/rtd_theme_overrides_references.css b/astrodata/doc/_static/rtd_theme_overrides_references.css deleted file mode 100644 index 785c691ebf..0000000000 --- a/astrodata/doc/_static/rtd_theme_overrides_references.css +++ /dev/null @@ -1,16 +0,0 @@ - -/* , a .rst-content tt, a .rst-content code */ - -.rst-content code.xref { - background-color: transparent; - border: solid 1px transparent; - color: #2980B9; - padding: 0px 0px; - font-size: 80%; - } - -.rst-content code.xref:hover { - /* border: solid 1px #e1e4e5; */ - text-decoration: underline; - } - diff --git a/astrodata/doc/_static/todo-styles.css b/astrodata/doc/_static/todo-styles.css deleted file mode 100644 index 09f3659575..0000000000 --- a/astrodata/doc/_static/todo-styles.css +++ /dev/null @@ -1,7 +0,0 @@ -div.admonition-todo { -border-top: 2px solid red; -border-bottom: 2px solid red; -border-left: 2px solid red; -border-right: 2px solid red; -background-color: #ff6347 -} diff --git a/astrodata/doc/api.rst b/astrodata/doc/api.rst deleted file mode 100644 index 66212bf1a8..0000000000 --- a/astrodata/doc/api.rst +++ /dev/null @@ -1,11 +0,0 @@ -.. _api: - -************* -Reference API -************* - -.. toctree:: - :maxdepth: 1 - - api/astrodata - api/gemini_instruments diff --git a/astrodata/doc/appendix_descriptors.rst b/astrodata/doc/appendix_descriptors.rst deleted file mode 100644 index 42917f76ee..0000000000 --- a/astrodata/doc/appendix_descriptors.rst +++ /dev/null @@ -1,248 +0,0 @@ -.. descriptors.rst - -.. _descriptors: - -*********************************** -List of Gemini Standard Descriptors -*********************************** - -To run and re-use Gemini primitives and functions this list of Standard -Descriptors must be defined for input data. This also applies to data -that is to be served by the Gemini Observatory Archive (GOA). - -For any ``AstroData`` objects, to get the list of the descriptors that are -defined use the ``AstroData.descriptors`` attribute:: - - >>> import astrodata - >>> import gemini_instruments - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - - >>> ad.descriptors - ('airmass', 'amp_read_area', 'ao_seeing', ..., 'well_depth_setting') - -To get the values:: - - >>> ad.airmass() - - >>> for descriptor in ad.descriptors: - ... print(descriptor, getattr(ad, descriptor)()) - -Note that not all of the descriptors below are defined for all of the -instruments. For example, ``shuffle_pixels`` is defined only for GMOS data -since only GMOS offers a Nod & Shuffle mode. - - -.. tabularcolumns:: |l|p{3.0in}|l| - - -+--------------------------------+----------------------------------------------------------------+-----------------+ -| **Descriptor** | **Short Definition** | **Python type** | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| | | ad[0].desc() | -| | +-----------------+ -| | | ad.desc() | -+================================+================================================================+=================+ -| airmass | Airmass of the observation. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| amp_read_area | Combination of amplifier name and 1-indexed section relative | str | -| | to the detector. +-----------------+ -| | | list of str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| ao_seeing | Estimate of the natural seeing as calculated from the | float | -| | adaptive optics systems. | | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| array_name | Name assigned to the array generated by a given amplifier, | str | -| | one array per amplifier. +-----------------+ -| | | list of str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| array_section | Section covered by the array(s), in 0-indexed pixels, relative | Section | -| | to the detector frame (e.g. position of multiple amps read +-----------------+ -| | within a CCD). Uses ``namedtuple`` "Section" defined in | list of Section | -| | ``gemini_instruments.common``. | | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| azimuth | Pointing position in azimuth, in degrees. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| calibration_key | Key used in the database that the ``getProcessed*`` primitives | str | -| | use to store previous calibration association information. | | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| camera | Name of the camera. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| cass_rotator_pa | Position angle of the Cassegrain rotator, in degrees. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| central_wavelength | Central wavelength, in meters. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| coadds | Number of co-adds. | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| data_label | Gemini data label. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| data_section | Section where the sky-exposed data falls, in 0-indexed pixels. | Section | -| | Uses ``namedtuple`` "Section" defined in +-----------------+ -| | ``gemini_instruments.common`` | list of Section | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| dec | Declination of the center of the field, in degrees. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| decker | Name of the decker. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| detector_name | Name assigned to the detector. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| detector_roi_setting | Human readable Region of Interest (ROI) setting | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| detector_rois_requested | Section defining the Regions of Interest, in 0-indexed pixels. | list of Section | -| | Uses ``namedtuple`` "Section" defined in | | -| | ``gemini_instruments.common``. | | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| detector_section | Section covered by the detector(s), in 0-indexed pixels, | list | -| | relative to the whole mosaic of detectors. +-----------------+ -| | Uses ``namedtuple`` "Section" defined in | list of Section | -| | ``gemini_instruments.common``. | | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| detector_x_bin | X-axis binning. | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| detector_x_offset | Telescope offset along the detector X-axis, in pixels. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| detector_y_bin | Y-axis binning. | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| detector_y_offset | Telescope offset along the detector Y-axis, in pixels. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| disperser | Name of the disperser. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| dispersion | Value for the dispersion, in meters per pixel. | float | -| | +-----------------+ -| | | list of float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| dispersion_axis | Dispersion axis. | int | -| | +-----------------+ -| | | list of int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| effective_wavelength | Wavelength representing the bandpass or the spectrum coverage. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| elevation | Pointing position in elevation, in degrees. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| exposure_time | Exposure time, in seconds. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| filter_name | Name of the filter combination. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| focal_plane_mask | Name of the mask in the focal plane. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| gain | Gain in electrons per ADU | float | -| | +-----------------+ -| | | list of float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| gain_setting | Human readable gain setting (eg. low, high) | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| gcal_lamp | Returns the name of the GCAL lamp being used, or "Off" if no | str | -| | lamp is in use. | | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| group_id | Gemini observation group ID that identifies compatible data. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| instrument | Name of the instrument | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| is_ao | Whether or not the adaptive optics system was used. | bool | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| is_coadds_summed | Whether co-adds are summed or averaged. | bool | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| local_time | Local time. | datetime | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| lyot_stop | Name of the lyot stop. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| mdf_row_id | Mask Definition File row ID of a cut MOS or XD spectrum. | int ?? | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| nod_count | Number of nods to A and B positions. | tuple of int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| nod_offsets | Nod offsets to A and B positions, in arcseconds | tuple of float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| nominal_atmospheric_extinction | Nomimal atmospheric extinction, from model. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| nominal_photometric_zeropoint | Nominal photometric zeropoint. | float | -| | +-----------------+ -| | | list of float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| non_linear_level | Lower boundary of the non-linear regime. | float | -| | +-----------------+ -| | | list of int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| object | Name of the target (as entered by the user). | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| observation_class | Gemini class name for the observation | str | -| | (eg. 'science', 'acq', 'dayCal'). | | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| observation_epoch | Observation epoch. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| observation_id | Gemini observation ID. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| observation_type | Gemini observation type (eg. 'OBJECT', 'FLAT', 'ARC'). | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| overscan_section | Section where the overscan data falls, in 0-indexed pixels. | Section | -| | Uses namedtuple "Section" defined in +-----------------+ -| | ``gemini_instruments.common``. | list of Section | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| pixel_scale | Pixel scale in arcsec per pixel. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| program_id | Gemini program ID. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| pupil_mask | Name of the pupil mask. | str ?? | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| qa_state | Gemini quality assessment state (eg. pass, usable, fail). | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| ra | Right ascension, in degrees. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| raw_bg | Gemini sky background band. | int ?? | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| raw_cc | Gemini cloud coverage band. | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| raw_iq | Gemini image quality band. | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| raw_wv | Gemini water vapor band. | int ?? | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| read_mode | Gemini name for combination for gain setting and read setting. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| read_noise | Read noise in electrons. | float | -| | +-----------------+ -| | | list of float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| read_speed_setting | human readable read mode setting (eg. slow, fast). | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| requested_bg | PI requested Gemini sky background band. | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| requested_cc | PI requested Gemini cloud coverage band. | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| requested_iq | PI requested Gemini image quality band. | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| requested_wv | PI requested Gemini water vapor band. | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| saturation_level | Saturation level. | int | -| | +-----------------+ -| | | list of int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| shuffle_pixels | Charge shuffle, in pixels. (nod and shuffle mode) | int | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| slit | Name of the slit. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| target_dec | Declination of the target, in degrees. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| target_ra | Right Ascension of the target, in degrees. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| telescope | Name of the telescope. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| telescope_x_offset | Offset along the telescope's x-axis. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| telescope_y_offset | Offset along the telescope's y-axis. | float | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| ut_date | UT date of the observation. | datetime.date | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| ut_datetime | UT date and time of the observation. | datetime | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| ut_time | UT time of the observation. | datetime.time | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| wavefront_sensor | Wavefront sensor used for the observation. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| wavelength_band | Band associated with the filter or the central wavelength. | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| wcs_dec | Declination of the center of field from the WCS keywords. | float | -| | In degrees. | | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| wcs_ra | Right Ascension of the center of field from the WCS keywords. | float | -| | In degrees. | | -+--------------------------------+----------------------------------------------------------------+-----------------+ -| well_depth_setting | Human readable well depth setting (eg. shallow, deep) | str | -+--------------------------------+----------------------------------------------------------------+-----------------+ diff --git a/astrodata/doc/cheatsheet.rst b/astrodata/doc/cheatsheet.rst deleted file mode 100644 index 98c351c159..0000000000 --- a/astrodata/doc/cheatsheet.rst +++ /dev/null @@ -1,463 +0,0 @@ -.. cheatsheet - -.. _cheatsheet: - -*********** -Cheat Sheet -*********** - -.. admonition:: Document ID - - PIPE-USER-105_AstrodataCheatSheet - -A data package is available for download if you wish to run the examples -included in this cheat sheet. Download it at: - - ``_ - -To unpack:: - - $ cd - $ tar xvf ad_usermanual_datapkg-v1.tar - $ bunzip2 ad_usermanual/playdata/*.bz2 - -Then go to the ``ad_usermanual/playground`` directory to run the examples. - -Imports -======= - -Import :mod:`astrodata` and :mod:`gemini_instruments`:: - - >>> import astrodata - >>> import gemini_instruments - -Basic read and write operations -=============================== - -Open a file:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - -Get path and filename:: - - >>> ad.path - '../playdata/N20170609S0154.fits' - >>> ad.filename - 'N20170609S0154.fits' - -Write to a new file:: - - >>> ad.write(filename='new154.fits') - >>> ad.filename - N20170609S0154.fits - -Overwrite the file:: - - >>> adnew = astrodata.open('new154.fits') - >>> adnew.filename - new154.fits - >>> adnew.write(overwrite=True) - -Object structure -================ - -Description ------------ -The |AstroData| object is assigned by "tags" that describe the -type of data it contains. The tags are drawn from rules defined in -|gemini_instruments| and are based on header information. - -When mapping a FITS file, each science pixel extension is loaded as a -|NDAstroData| object. The list is zero-indexed. So FITS -extension 1 becomes element 0 of the |AstroData| object. If a ``VAR`` -extension is present, it is loaded to the variance attribute of the -|NDAstroData|. If a ``DQ`` extension is present, it is loaded to the ``.mask`` -attribute of the |NDAstroData|. ``SCI``, ``VAR`` and ``DQ`` are associated -through the ``EXTVER`` keyword value. - -In the file below, each |AstroData| "extension" contains the pixel data, -then an error plane (``.variance``) and a bad pixel mask plane (``.mask``). -|Table| can be attached to an extension, like OBJCAT, or to the -|AstroData| object globally, like REFCAT. (In this case, OBJCAT is a -catalogue of the sources detected in the image, REFCAT is a reference catalog -for the area covered by the whole file.) If other 2D data needs to be -associated with an extension this can also be done, like here with OBJMASK, -a 2D mask matching the sources in the image. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - >>> ad.info() - Filename: ../playdata/N20170609S0154_varAdded.fits - Tags: ACQUISITION GEMINI GMOS IMAGE NORTH OVERSCAN_SUBTRACTED OVERSCAN_TRIMMED - PREPARED SIDEREAL - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) int16 - .OBJCAT Table (6, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - [ 1] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) int16 - .OBJCAT Table (8, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - [ 2] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) int16 - .OBJCAT Table (7, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - [ 3] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) int16 - .OBJCAT Table (5, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - Other Extensions - Type Dimensions - .REFCAT Table (245, 16) - - - -Modifying the structure ------------------------ - -Let's first get our play data loaded. You are encouraged to do a -:meth:`~astrodata.AstroData.info` before and after each structure-modification -step, to see how things change. - -:: - - >>> from copy import deepcopy - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> adcopy = deepcopy(ad) - >>> advar = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - -Append an extension:: - - >>> adcopy.append(advar[3]) - >>> adcopy.append(advar[3].data) - - -Delete an extension:: - - >>> del adcopy[5] - -Delete and add variance and mask planes:: - - >>> var = adcopy[4].variance - >>> adcopy[4].variance = None - >>> adcopy[4].variance = var - -Attach a table to an extension:: - - >>> adcopy[3].SMAUG = advar[0].OBJCAT.copy() - - -Attach a table to the |AstroData| object:: - - >>> adcopy.DROGON = advar.REFCAT.copy() - -Delete a table:: - - >>> del adcopy[3].SMAUG - >>> del adcopy.DROGON - - - -Astrodata tags -============== - -:: - - >>> ad = astrodata.open('../playdata/N20170521S0925_forStack.fits') - >>> ad.tags - {'GMOS', 'OVERSCAN_SUBTRACTED', 'SIDEREAL', 'NORTH', 'OVERSCAN_TRIMMED', - 'PREPARED', 'IMAGE', 'GEMINI'} - - >>> type(ad.tags) - - - >>> {'IMAGE', 'PREPARED'}.issubset(ad.tags) - True - >>> 'PREPARED' in ad.tags - True - - -Headers -======= - -The use of descriptors is favored over direct header access when retrieving -values already represented by descriptors, and when writing instrument agnostic -routines. - -Descriptors ------------ - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> ad.filter_name() - 'open1-6&g_G0301' - >>> ad.filter_name(pretty=True) - 'g' - >>> ad.gain() # uses a look-up table to get the correct values - [2.03, 1.97, 1.96, 2.01] - >>> ad.hdr['GAIN'] - [1.0, 1.0, 1.0, 1.0] # the wrong values contained in the raw data. - >>> ad[0].gain() - 2.03 - >>> ad.gain()[0] - 2.03 - - >>> ad.descriptors - ('airmass', 'amp_read_area', 'ao_seeing', ... - ...) - - -Direct access to header keywords --------------------------------- - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - -Primary Header Unit -******************* - -To see a print out of the full PHU: - - >>> ad.phu - -Get value from PHU:: - - >>> ad.phu['EXPTIME'] - 1.0 - - >>> default = 5. - >>> ad.phu.get('BOGUSKEY', default) - 5.0 - -Set PHU keyword, with and without comment:: - - >>> ad.phu['NEWKEY'] = 50. - >>> ad.phu['ANOTHER'] = (30., 'Some comment') - -Delete PHU keyword:: - - >>> del ad.phu['NEWKEY'] - - - -Pixel extension header -********************** -To see a print out of the full header for an extension or all the extensions: - - >>> ad[0].hdr - >>> list(ad.hdr) - -Get value from an extension header:: - - >>> ad[0].hdr['OVERSCAN'] - 469.7444308769482 - >>> ad[0].hdr.get('OVERSCAN', default) - -Get keyword value for all extensions:: - - >>> ad.hdr['OVERSCAN'] - [469.7444308769482, 469.656175780001, 464.9815279808291, 467.5701178951787] - >>> ad.hdr.get('BOGUSKEY', 5.) - [5.0, 5.0, 5.0, 5.0] - -Set extension header keyword, with and without comment:: - - >>> ad[0].hdr['NEWKEY'] = 50. - >>> ad[0].hdr['ANOTHER'] = (30., 'Some comment') - -Delete an extension keyword:: - - >>> del ad[0].hdr['NEWKEY'] - -Table header -************ -See the :ref:`cheatsheet_tables` section. - - -Pixel data -========== - -Arithmetics ------------ -Arithmetics with variance and mask propagation is offered for -``+``, ``-``, ``*``, ``/``, and ``**``. - -:: - - >>> ad_hcont = astrodata.open('../playdata/N20170521S0925_forStack.fits') - >>> ad_halpha = astrodata.open('../playdata/N20170521S0926_forStack.fits') - - >>> adsub = ad_halpha - ad_hcont - - >>> ad_halpha[0].data.mean() - 646.11896 - >>> ad_hcont[0].data.mean() - 581.81342 - >>> adsub[0].data.mean() - 64.305862 - - >>> ad_halpha[0].variance.mean() - 669.80664 - >>> ad_hcont[0].variance.mean() - 598.46667 - >>> adsub[0].variance.mean() - 1268.274 - - - # In place multiplication - >>> ad_mult = deepcopy(ad) - >>> ad_mult.multiply(ad) - >>> ad_mult.multiply(5.) - - - # Using descriptors to operate in-place on extensions. - >>> from copy import deepcopy - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - >>> ad_gain = deepcopy(ad) - >>> for (ext, gain) in zip(ad_gain, ad_gain.gain()): - ... ext.multiply(gain) - >>> ad_gain[0].data.mean() - 366.39545 - >>> ad[0].data.mean() - 180.4904 - >>> ad[0].gain() - 2.03 - - -Other pixel data operations ---------------------------- - -:: - - >>> import numpy as np - >>> ad_halpha[0].mask[300:350,300:350] = 1 - >>> np.mean(ad_halpha[0].data[ad_halpha[0].mask==0]) - 657.1994 - >>> np.mean(ad_halpha[0].data) - 646.11896 - - -.. _cheatsheet_tables: - -Tables -====== - -Tables are stored as :class:`astropy.table.Table` class. FITS tables are -represented in :mod:`astrodata` as |Table| and FITS headers are stored in the -|NDAstroData| ``meta`` attribute. Most table -access should be done through the |Table| interface. The best reference is the -|astropy| documentation itself. Below are just a few examples. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - -Get column names:: - - >>> ad.REFCAT.colnames - -Get column content:: - - >>> ad.REFCAT['zmag'] - >>> ad.REFCAT['zmag', 'zmag_err'] - -Get content of row:: - - >>> ad.REFCAT[4] # 5th row - >>> ad.REFCAT[4:6] # 5th and 6th rows - - -Get content from specific row and column:: - - >>> ad.REFCAT['zmag'][4] - -Add a column:: - - >>> new_column = [0] * len(ad.REFCAT) - >>> ad.REFCAT['new_column'] = new_column - -Add a row:: - - >>> new_row = [0] * len(ad.REFCAT.colnames) - >>> new_row[1] = '' # Cat_Id column is of "str" type. - >>> ad.REFCAT.add_row(new_row) - -Selecting value from criterion:: - - >>> ad.REFCAT['zmag'][ad.REFCAT['Cat_Id'] == '1237662500002005475'] - >>> ad.REFCAT['zmag'][ad.REFCAT['zmag'] < 18.] - -Rejecting :class:`numpy.nan` before doing something with the values:: - - >>> t = ad.REFCAT # to save typing. - >>> t['zmag'][np.where(np.isnan(t['zmag']), 99, t['zmag']) < 18.] - - >>> t['zmag'].mean() - nan - >>> t['zmag'][np.where(~np.isnan(t['zmag']))].mean() - 20.377306 - -If for some reason you need to access the FITS table headers, here is how to do it. - -To see the FITS headers:: - - >>> ad.REFCAT.meta - >>> ad[0].OBJCAT.meta - -To retrieve a specific FITS table header:: - - >>> ad.REFCAT.meta['header']['TTYPE3'] - 'RAJ2000' - >>> ad[0].OBJCAT.meta['header']['TTYPE3'] - 'Y_IMAGE' - -To retrieve all the keyword names matching a selection:: - - >>> keynames = [key for key in ad.REFCAT.meta['header'] if key.startswith('TTYPE')] - - -Create new AstroData object -=========================== - -Basic header and data array set to zeros:: - - >>> from astropy.io import fits - - >>> phu = fits.PrimaryHDU() - >>> pixel_data = np.zeros((100,100)) - - >>> hdu = fits.ImageHDU() - >>> hdu.data = pixel_data - >>> ad = astrodata.create(phu) - >>> ad.append(hdu, name='SCI') - -or another way:: - - >>> hdu = fits.ImageHDU(data=pixel_data, name='SCI') - >>> ad = astrodata.create(phu, [hdu]) - -A |Table| as an |AstroData| object:: - - >>> from astropy.table import Table - - >>> my_astropy_table = Table(list(np.random.rand(2,100)), names=['col1', 'col2']) - >>> phu = fits.PrimaryHDU() - - >>> ad = astrodata.create(phu) - >>> ad.SMAUG = my_astropy_table - - >>> phu = fits.PrimaryHDU() - >>> ad = astrodata.create(phu) - >>> ad.SMAUG = my_fits_table - -WARNING: This last line will not run like the others as we have not defined -``my_fits_table``. This is nonetheless how it is done if you had a FITS table. diff --git a/astrodata/doc/conf.py b/astrodata/doc/conf.py deleted file mode 100644 index cb3999fee2..0000000000 --- a/astrodata/doc/conf.py +++ /dev/null @@ -1,465 +0,0 @@ -# Astrodata build configuration file - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -import os -import sys -# sys.path.insert(0, os.path.abspath('.')) -sys.path.insert(0, os.path.abspath('../')) - -# -- General configuration ------------------------------------------------ - -# If your documentation needs a minimal Sphinx version, state it here. -# -# needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.coverage', - 'sphinx.ext.doctest', - 'sphinx.ext.graphviz', - 'sphinx.ext.ifconfig', - 'sphinx.ext.imgmath', - 'sphinx.ext.intersphinx', - 'sphinx.ext.napoleon', - 'sphinx.ext.todo', - 'sphinx.ext.viewcode', -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix(es) of source filenames. -# You can specify multiple suffix as a list of string: -# -# source_suffix = ['.rst', '.md'] -source_suffix = '.rst' - -# The encoding of source files. -# -# source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = 'Astrodata Manual' -copyright = '2025, Association of Universities for Research in Astronomy' -author = 'DRAGONS Team' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = '4.0-dev' -# The full version, including alpha/beta/rc tags. -#release = '3.2.x' -#rtdurl = 'release-'+release -#release = '3.2.2' -#rtdurl = 'v'+release -rtdurl = 'latest' - - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -# -# This is also used if you do content translation via gettext catalogs. -# Usually you set "language" from the command line for these cases. -language = "en" - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# -today = 'January 2025' -# -# Else, today_fmt is used as the format for a strftime call. -# -# today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This patterns also effect to html_static_path and html_extra_path -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - -# The reST default role (used for this markup: `text`) to use for all -# documents. -default_role = 'obj' - -# If true, '()' will be appended to :func: etc. cross-reference text. -# -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -# modindex_common_prefix = [] - -# If true, keep warnings as "system message" paragraphs in the built documents. -# keep_warnings = False - -# If true, `todo` and `todoList` produce output, else they produce nothing. -todo_include_todos = True - -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = { - 'astropy': ('https://docs.astropy.org/en/stable/', None), - #'gemini_instruments': ('https://dragons-recipe-system-programmers-manual.readthedocs.io/en/latest/', None), - #'geminidr': ('https://dragons-recipe-system-programmers-manual.readthedocs.io/en/latest/', None), - 'matplotlib': ('https://matplotlib.org/', None), - 'numpy': ('https://numpy.org/doc/stable/', None), - 'python': ('https://docs.python.org/3', None), -} - -# -- Options for HTML output ---------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_rtd_theme' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# html_theme_path = [] - -# The name for this set of Sphinx documents. -# " v documentation" by default. -# -# html_title = '' - -# A shorter title for the navigation bar. Default is the same as html_title. -# -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# -# html_logo = None - -# The name of an image file (relative to this directory) to use as a favicon of -# the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# Add any extra paths that contain custom files (such as robots.txt or -# .htaccess) here, relative to this directory. These files are copied -# directly to the root of the documentation. -# -# html_extra_path = [] - -# If not None, a 'Last updated on:' timestamp is inserted at every page -# bottom, using the given strftime format. -# The empty string is equivalent to '%b %d, %Y'. -# -# html_last_updated_fmt = None - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# -# html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -# -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# -# html_additional_pages = {} - -# If false, no module index is generated. -# -# html_domain_indices = True - -# If false, no index is generated. -# -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# -# html_split_index = False - -# If true, links to the reST sources are added to the pages. -# -# html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -# -# html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -# -# html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# -# html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = None - -# Language to be used for generating the HTML full-text search index. -# Sphinx supports the following languages: -# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' -# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr', 'zh' -# -# html_search_language = 'en' - -# A dictionary with options for the search language support, empty by default. -# 'ja' uses this config value. -# 'zh' user can custom change `jieba` dictionary path. -# -# html_search_options = {'type': 'default'} - -# The name of a javascript file (relative to the configuration directory) that -# implements a search results scorer. If empty, the default will be used. -# -# html_search_scorer = 'scorer.js' - -# Output file base name for HTML help builder. -htmlhelp_basename = 'AstrodataManual' - -# -- Options for LaTeX output --------------------------------------------- - -latex_elements = { - # This will remove blank pages. - 'classoptions': ',openany,oneside', - 'babel': '\\usepackage[english]{babel}', - - # The paper size ('letterpaper' or 'a4paper'). - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # 'preamble': '', - 'preamble': '\\usepackage{appendix} \\setcounter{tocdepth}{0}', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - ('index', 'AstrodataManual.tex', 'Astrodata Manual', - 'DRAGONS Team', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# -latex_logo = 'images/GeminiLogo_new_2014.jpg' - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# -# latex_use_parts = False - -# If true, show page references after internal links. -# -# latex_show_pagerefs = False - -# If true, show URL addresses after external links. -# -# latex_show_urls = False - -# Documents to append as an appendix to all manuals. -# -# latex_appendices = [] - -# It false, will not define \strong, \code, itleref, \crossref ... but only -# \sphinxstrong, ..., \sphinxtitleref, ... To help avoid clash with user added -# packages. -# -# latex_keep_old_macro_names = True - -# If false, no module index is generated. -# -# latex_domain_indices = True - - -# -- Options for manual page output --------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'astrodatamanual', 'Astrodata Manual', - ['DRAGONS Team'], 1) -] - -# If true, show URL addresses after external links. -# -# man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------- - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'AstrodataManual', 'Astrodata Manual', - 'DRAGONS Team', 'AstrodataManual', - 'Manual for the astrodata package', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -# -# texinfo_appendices = [] - -# If false, no module index is generated. -# -# texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -# -# texinfo_show_urls = 'footnote' - -# If true, do not generate a @detailmenu in the "Top" node's menu. -# -# texinfo_no_detailmenu = False - - -# -- Automatically generate API documentation -------------------------------- -# -- Enable autoapi ---------------------------------------------------------- - - -def run_api_doc(_): - """ - Automatic API generator - - This method is used to generate API automatically by importing all the - modules and sub-modules inside a package. - - It is equivalent to run: - >>> sphinx-apidoc --force --no-toc --separate --module --output-dir api/ ../../ ../../cal_service - - It is useful because it creates .rst files on the file. - - NOTE - ---- - This does not work with PyCharm default build. If you want to trigger - this function, use the standard `$ make html` in the command line. - The .rst files will be generated. After that, you can use PyCharm's - build helper. - """ - build_packages = [ - 'astrodata', - 'gemini_instruments' - ] - - current_path = os.path.abspath(os.path.dirname(__file__)) - root_path = os.path.abspath(os.path.join(current_path, '..', '..')) - - print(("Current Path:", current_path)) - - for p in build_packages: - - build_path = os.path.join(root_path, p) - - ignore_paths = ['doc', 'test*', '**/test*'] - ignore_paths = [os.path.join(build_path, i, '*') for i in ignore_paths] - - argv = [ - # "--force", - "--no-toc", - # "--separate", - "--module", - "--output-dir", "api/", - build_path - ] + ignore_paths - - sys.path.insert(0, root_path) - - try: - # Sphinx 1.7+ - from sphinx.ext import apidoc - apidoc.main(argv) - - except ImportError: - # Sphinx 1.6 (and earlier) - from sphinx import apidoc - argv.insert(0, apidoc.__file__) - apidoc.main(argv) - - -# -- Finishing with a setup that will run always ----------------------------- -def setup(app): - - # Adding style in order to have the todos show up in a red box. - app.add_css_file('todo-styles.css') - app.add_css_file('rtd_theme_overrides.css') - app.add_css_file('rtd_theme_overrides_references.css') - - # Automatic API generation - app.connect('builder-inited', run_api_doc) - -# This is added to the end of RST files - a good place to put substitutions to -# be used globally. -rst_epilog = """ -.. _`Anaconda`: https://www.anaconda.com/ -.. _`Astropy`: http://docs.astropy.org/en/stable/ -.. _`Conda`: https://conda.io/docs/ -.. _`Numpy`: https://numpy.org/doc/stable/ -.. |numpy| replace:: `Numpy`_ -.. |astropy| replace:: `Astropy`_ - -.. |AstroData| replace:: :class:`~astrodata.AstroData` -.. |astrodata| replace:: :mod:`~astrodata` -.. |geminidr| replace:: :mod:`~geminidr` -.. |gemini_instruments| replace:: :mod:`gemini_instruments` -.. |gemini| replace:: ``gemini`` -.. |Mapper| replace:: :class:`~recipe_system.mappers.baseMapper.Mapper` -.. |mappers| replace:: :mod:`recipe_system.mappers` -.. |NDAstroData| replace:: :class:`~astrodata.nddata.NDAstroData` -.. |NDData| replace:: :class:`~astropy.nddata.NDData` -.. |PrimitiveMapper| replace:: :class:`~recipe_system.mappers.primitiveMapper.PrimitiveMapper` -.. |RecipeMapper| replace:: :class:`~recipe_system.mappers.recipeMapper.RecipeMapper` -.. |recipe_system| replace:: :mod:`recipe_system` -.. |Reduce| replace:: :class:`~recipe_system.reduction.coreReduce.Reduce` -.. |reduce| replace:: ``reduce`` -.. |Table| replace:: :class:`~astropy.table.Table` -.. |TagSet| replace:: :class:`~astrodata.TagSet` - -.. role:: raw-html(raw) - :format: html - -.. |DRAGONS| replace:: :raw-html:`DRAGONS` -.. |RSProgManual| replace:: :raw-html:`Recipe System Programmer Manual` -.. |RSUserManual| replace:: :raw-html:`Recipe System User Manual` - -.. |RSUserInstall| replace:: :raw-html:`Installation Guide` - - -""".format(v = rtdurl) - diff --git a/astrodata/doc/images/GeminiLogo_new_2014.jpg b/astrodata/doc/images/GeminiLogo_new_2014.jpg deleted file mode 100644 index 295c4e1a33..0000000000 Binary files a/astrodata/doc/images/GeminiLogo_new_2014.jpg and /dev/null differ diff --git a/astrodata/doc/index.rst b/astrodata/doc/index.rst deleted file mode 100644 index 13968e7959..0000000000 --- a/astrodata/doc/index.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. Astrodata Master Manual - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - Manually edited by SC December 2020 - -################ -Astrodata Manual -################ - -.. admonition:: Document ID - - PIPE-USER-120_AstrodataMasterManual - -.. toctree:: - :hidden: - :numbered: - - cheatsheet - usermanual/index - progmanual/index - -This documentation provides different levels of information: - -- :doc:`cheatsheet` - A refresher on common astrodata operations -- :doc:`usermanual/index` - How to code with astrodata -- :doc:`progmanual/index` - How to code for astrodata - - -.. raw:: latex - - % Set up the appendix mode and modify the LaTeX toc behavior - \appendix - \noappendicestocpagenum - \addappheadtotoc - - -.. rubric:: Appendix - -.. toctree:: - :maxdepth: 1 - - appendix_descriptors - api - -.. ****************** -.. Indices and tables -.. ****************** - -.. * :ref:`genindex` -.. * :ref:`modindex` -.. * :ref:`search` - -.. todolist:: diff --git a/astrodata/doc/make.bat b/astrodata/doc/make.bat deleted file mode 100644 index 53564a8019..0000000000 --- a/astrodata/doc/make.bat +++ /dev/null @@ -1,281 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=_build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. epub3 to make an epub3 - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - echo. coverage to run coverage check of the documentation if enabled - echo. dummy to check syntax errors of document sources - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -REM Check if sphinx-build is available and fallback to Python version if any -%SPHINXBUILD% 1>NUL 2>NUL -if errorlevel 9009 goto sphinx_python -goto sphinx_ok - -:sphinx_python - -set SPHINXBUILD=python -m sphinx.__init__ -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -:sphinx_ok - - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\AstrodataProgrammersManual.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\AstrodataProgrammersManual.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "epub3" ( - %SPHINXBUILD% -b epub3 %ALLSPHINXOPTS% %BUILDDIR%/epub3 - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub3 file is in %BUILDDIR%/epub3. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %~dp0 - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %~dp0 - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "coverage" ( - %SPHINXBUILD% -b coverage %ALLSPHINXOPTS% %BUILDDIR%/coverage - if errorlevel 1 exit /b 1 - echo. - echo.Testing of coverage in the sources finished, look at the ^ -results in %BUILDDIR%/coverage/python.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -if "%1" == "dummy" ( - %SPHINXBUILD% -b dummy %ALLSPHINXOPTS% %BUILDDIR%/dummy - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. Dummy builder generates no files. - goto end -) - -:end diff --git a/astrodata/doc/progmanual/adclass.rst b/astrodata/doc/progmanual/adclass.rst deleted file mode 100644 index 5fb36f653b..0000000000 --- a/astrodata/doc/progmanual/adclass.rst +++ /dev/null @@ -1,397 +0,0 @@ -.. astrodata.rst - -.. _astrodata: - -************************* -AstroData and Derivatives -************************* - -The |AstroData| class is the main interface to the package. When opening files -or creating new objects, a derivative of this class is returned, as the -|AstroData| class is not intended to be used directly. It provides the logic to -calculate the :ref:`tag set ` for an image, which is common to all -data products. Aside from that, it lacks any kind of specialized knowledge -about the different instruments that produce the FITS files. More importantly, -it defines two methods (``info`` and ``load``) as abstract, meaning that the -class cannot be instantiated directly: a derivative must implement those -methods in order to be useful. Such derivatives can also implement descriptors, -which provide processed metadata in a way that abstracts the user from the raw -information (e.g., the keywords in FITS headers). - -|AstroData| does define a common interface, though. Much of it consists on -implementing semantic behavior (access to components through indices, like a -list; arithmetic using standard operators; etc), mostly by implementing -standard Python methods: - -* Defines a common ``__init__`` function. - -* Implements ``__deepcopy__``. - -* Implements ``__iter__`` to allow sequential iteration over the main set of - components (e.g., FITS science HDUs). - -* Implements ``__getitem__`` to allow data slicing (e.g., ``ad[2:4]`` returns - a new |AstroData| instance that contains only the third and fourth main - components). - -* Implements ``__delitem__`` to allow for data removal based on index. It does - not define ``__setitem__``, though. The basic AstroData series of classes - only allows to append new data blocks, not to replace them in one sweeping - move. - -* Implements ``__iadd__``, ``__isub__``, ``__imul__``, ``__itruediv__``, and - their not-in-place versions, based on them. - -There are a few other methods. For a detailed discussion, please refer to the -:ref:`api`. - -.. _tags_prop_entry: - -The ``tags`` Property -===================== - -Additionally, and crucial to the package, AstroData offers a ``tags`` property, -that under the hood calculates textual tags that describe the object -represented by an instance, and returns a set of strings. Returning a set (as -opposed to a list, or other similar structure) is intentional, because it is -fast to compare sets, e.g., testing for membership; or calculating intersection, -etc., to figure out if a certain dataset belongs to an arbitrary category. - -The implementation for the tags property is just a call to -``AstroData._process_tags()``. This function implements the actual logic behind -calculating the tag set (described :ref:`below `). A derivative class -could redefine the algorithm, or build upon it. - - -Writing an ``AstroData`` Derivative -=================================== - -The first step when creating new |AstroData| derivative hierarchy would be to -create a new class that knows how to deal with some kind of specific data in a -broad sense. - -|AstroData| implements both ``.info()`` and ``.load()`` in ways that are -specific to FITS files. It also introduces a number of FITS-specific methods -and properties, e.g.: - -* The properties ``phu`` and ``hdr``, which return the primary header and - a list of headers for the science HDUs, respectively. - -* A ``write`` method, which will write the data back to a FITS file. - -* A ``_matches_data`` **static** method, which is very important, involved in - guiding for the automatic class choice algorithm during data loading. We'll - talk more about this when dealing with :ref:`registering our classes - `. - -It also defines the first few descriptors, which are common to all Gemini data: -``instrument``, ``object``, and ``telescope``, which are good examples of simple -descriptors that just map a PHU keyword without applying any conversion. - -A typical AstroData programmer will extend this class (|AstroData|). Any of -the classes under the ``gemini_instruments`` package can be used as examples, -but we'll describe the important bits here. - - -Create a package for it ------------------------ - -This is not strictly necessary, but simplifies many things, as we'll see when -talking about *registration*. The package layout is up to the designer, so you -can decide how to do it. For DRAGONS we've settled on the following -recommendation for our internal process (just to keep things familiar):: - - gemini_instruments - __init__.py - instrument_name - __init__.py - adclass.py - lookup.py - -Where ``instrument_name`` would be the package name (for Gemini we group all -our derivative packages under ``gemini_instruments``, and we would import -``gemini_instruments.gmos``, for example). ``__init__.py`` and ``adclass.py`` -would be the only required modules under our recommended layout, with -``lookup.py`` being there just to hold hard-coded values in a module separate -from the main logic. - -``adclass.py`` would contain the declaration of the derivative class, and -``__init__.py`` will contain any code needed to register our class with the -|AstroData| system upon import. - - -Create your derivative class ----------------------------- - -This is an excerpt of a typical derivative module:: - - from astrodata import astro_data_tag, astro_data_descriptor, TagSet - from astrodata import AstroData - - from . import lookup - - class AstroDataInstrument(AstroData): - __keyword_dict = dict( - array_name = 'AMPNAME', - array_section = 'CCDSECT' - ) - - @staticmethod - def _matches_data(source): - return source[0].header.get('INSTRUME', '').upper() == 'MYINSTRUMENT' - - @astro_data_tag - def _tag_instrument(self): - return TagSet(['MYINSTRUMENT']) - - @astro_data_tag - def _tag_image(self): - if self.phu.get('GRATING') == 'MIRROR': - return TagSet(['IMAGE']) - - @astro_data_tag - def _tag_dark(self): - if self.phu.get('OBSTYPE') == 'DARK': - return TagSet(['DARK'], blocks=['IMAGE', 'SPECT']) - - @astro_data_descriptor - def array_name(self): - return self.phu.get(self._keyword_for('array_name')) - - @astro_data_descriptor - def amp_read_area(self): - ampname = self.array_name() - detector_section = self.detector_section() - return "'{}':{}".format(ampname, detector_section) - -.. note:: - An actual Gemini Facility Instrument class will derive from - ``gemini_instruments.AstroDataGemini``, but this is irrelevant - for the example. - -The class typically relies on functionality declared elsewhere, in some -ancestor, e.g., the tag set computation and the ``_keyword_for`` method are -defined at |AstroData|. - -Some highlights: - -* ``__keyword_dict``\ [#keywdict]_ defines one-to-one mappings, assigning a more - readable moniker for an HDU header keyword. The idea here is to prevent - hard-coding the names of the keywords, in the actual code. While these are - typically quite stable and not prone to change, it's better to be safe than - sorry, and this can come in useful during instrument development, which is - the more likely source of instability. The actual value can be extracted by - calling ``self._keyword_for('moniker')``. - -* ``_matches_data`` is a static method. It does not have any knowledge about - the class itself, and it does not work on an *instance* of the class: it's - a member of the class just to make it easier for the AstroData registry to - find it. This method is passed some object containing cues of the internal - structure and contents of the data. This could be, for example, an instance - of ``HDUList``. Using these data, ``_matches_data`` must return a boolean, - with ``True`` meaning "I know how to handle this data". - - Note that ``True`` **does not mean "I have full knowledge of the data"**. It - is acceptable for more than one class to claim compatibility. For a GMOS FITS - file, the classes that will return ``True`` are: |AstroData| (because it is - a FITS file that comply with certain minimum requirements), - `~gemini_instruments.gemini.AstroDataGemini` (the data contains Gemini - Facility common metadata), and `~gemini_instruments.gmos.AstroDataGmos` (the - actual handler!). - - But this does not mean that multiple classes can be valid "final" candidates. - If AstroData's automatic class discovery finds more than one class claiming - matching with the data, it will start discarding them on the basis of - inheritance: any class that appears in the inheritance tree of another one is - dropped, because the more specialized one is preferred. If at some point the - algorithm cannot find more classes to drop, and there is more than one left - in the list, an exception will occur, as AstroData will have no way to choose - one over the other. - -* A number of "tag methods" have been declared. Their naming is a convention, - at the end of the day (the "``_tag_``" prefix, and the related "``_status_``" - one, are *just hints* for the programmer): each team should establish - a convention that works for them. What is important here is to **decorate** - them using `~astrodata.astro_data_tag`, which earmarks the method so that it - can be discovered later, and ensures that it returns an appropriate value. - - A tag method will return either a `~astrodata.TagSet` instance (which can be - empty), or ``None``, which is the same as returning an empty - `~astrodata.TagSet`\ [#tagset1]_. - - **All** these methods will be executed when looking up for tags, and it's up - to the tag set construction algorithm (see :ref:`ad_tags`) to figure out the final - result. In theory, one **could** provide *just one* big method, but this is - feasible only when the logic behind deciding the tag set is simple. The - moment that there are a few competing alternatives, with some conditions - precluding other branches, one may end up with a rather complicated dozens of - lines of logic. Let the algorithm do the heavy work for you: split the tags - as needed to keep things simple, with an easy to understand logic. - - Also, keeping the individual (or related) tags in separate methods lets you - exploit the inheritance, keeping common ones at a higher level, and - redefining them as needed later on, at derived classes. - - Please, refer to `~gemini_instruments.gemini.AstroDataGemini`, - `~gemini_instruments.gmos.AstroDataGmos`, and - `~gemini_instruments.gnirs.AstroDataGnirs` for examples using most of the - features. - -* The `astrodata.AstroData.read` method calls the `astrodata.fits.read_fits` - function, which uses metadata in the FITS headers to determine how the data - should be stored in the |AstroData| object. In particular, the ``EXTNAME`` - and ``EXTVER`` keywords are used to assign individual FITS HDUs, using the - same names (``SCI``, ``DQ``, and ``VAR``) as Gemini-IRAF for the ``data``, - ``mask``, and ``variance`` planes. A ``SCI`` HDU *must* exist if there is - another HDU with the same ``EXTVER``, or else an error will occur. - - If the raw data do not conform to this format, the `astrodata.AstroData.read` - method can be overridden by your class, by having it call the - `astrodata.fits.read_fits` function with an additional parameter, - ``extname_parser``, that provides a function to modify the header. This - function will be called on each HDU before further processing. As an example, - the SOAR Adaptive Module Imager (SAMI) instrument writes raw data as - a 4-extension MEF file, with the extensions having ``EXTNAME`` values - ``im1``, ``im2``, etc. These need to be modified to ``SCI``, and an - appropriate ``EXTVER`` keyword added` [#extver]_\. This can be done by - writing a suitable ``read`` method for the ``AstroDataSami`` class:: - - @classmethod - def read(cls, source, extname_parser=None): - def sami_parser(hdu): - m = re.match('im(\d)', hdu.header.get('EXTNAME', '')) - if m: - hdu.header['EXTNAME'] = ('SCI', 'Added by AstroData') - hdu.header['EXTVER'] = (int(m.group(1)), 'Added by AstroData') - - return super().read(source, extname_parser=extname_parser) - - -* *Descriptors* will make the bulk of the class: again, the name is arbitrary, - and it should be descriptive. What *may* be important here is to use - `~astrodata.astro_data_descriptor` to decorate them. This is *not required*, - because unlike tag methods, descriptors are meant to be called explicitly by - the programmer, but they can still be marked (using this decorator) to be - listed when calling the ``descriptors`` property. The decorator does not - alter the descriptor input or output in any way, so it is always safe to use - it, and you probably should, unless there's a good reason against it (e.g., - if a descriptor is deprecated and you don't want it to show up in lookups). - - More detailed information can be found in :ref:`ad_descriptors`. - - -.. _class_registration: - -Register your class -------------------- - -Finally, you need to include your class in the **AstroData Registry**. This is -an internal structure with a list of all the |AstroData|\-derived classes that -we want to make available for our programs. Including the classes in this -registry is an important step, because a file should be opened using -`astrodata.open` or `astrodata.create`, which uses the registry to identify -the appropriate class (via the ``_matches_data`` methods), instead of having -the user specify it explicitly. - -The version of AstroData prior to DRAGONS had an auto-discovery mechanism, that -explored the source tree looking for the relevant classes and other related -information. This forced a fixed directory structure (because the code needed -to know where to look for files), and gave the names of files and classes -semantic meaning (to know *which* files to look into, for example). Aside from -the rigidness of the scheme, this introduced all sort of inefficiencies, -including an unacceptably high overhead when importing the AstroData package -for the first time during execution. - -In this new version of AstroData we've introduced a more manageable scheme, -that places the discovery responsibility on the programmer. A typical -``__init__.py`` file on an instrument package will look like this:: - - __all__ = ['AstroDataMyInstrument'] - - from astrodata import factory - from .adclass import AstroDataMyInstrument - - factory.addClass(AstroDataMyInstrument) - -The call to ``factory.addClass`` is the one registering the class. This step -**needs** to be done **before** the class can be used effectively in the -AstroData system. Placing the registration step in the ``__init__.py`` file is -convenient, because importing the package will be enough! - -Thus, a script making use of DRAGONS' AstroData to manipulate GMOS data -could start like this:: - - import astrodata - from gemini_instruments import gmos - - ... - - ad = astrodata.open(some_file) - -The first import line is not needed, technically, because the ``gmos`` package -will import it too, anyway, but we'll probably need the ``astrodata`` package -in the namespace anyway, and it's always better to be explicit. Our -typical DRAGONS scripts and modules start like this, instead:: - - import astrodata - import gemini_instruments - -``gemini_instruments`` imports all the packages under it, making knowledge -about all Gemini instruments available for the script, which is perfect for a -multi-instrument pipeline, for example. Loading all the instrument classes is -not typically a burden on memory, though, so it's easier for everyone to take -the more general approach. It also makes things easier on the end user, because -they won't need to know internal details of our packages (like their naming -scheme). We suggest this "*cascade import*" scheme for all new source trees, -letting the user decide which level of detail they need. - -As an additional step, the ``__init__.py`` file in a package may do extra -initialization. For example, for the Gemini modules, one piece of functionality -that is shared across instruments is a descriptor that translates a filter's -name (say "u" or "FeII") to its central wavelength (e.g., -0.35µm, 1.644µm). As it is a rather common function for us, it is implemented -by `~gemini_instruments.gemini.AstroDataGemini`. This class **does not know** -about its daughter classes, though, meaning that it **cannot know** about the -filters offered by their instruments. Instead, we offer a function that can -be used to update the filter → wavelength mapping in -`gemini_instruments.gemini.lookup` so that it is accessible by the -`~gemini_instruments.gemini.AstroDataGemini`\-level descriptor. So our -``gmos/__init__.py`` looks like this:: - - __all__ = ['AstroDataGmos'] - - from astrodata import factory - from ..gemini import addInstrumentFilterWavelengths - from .adclass import AstroDataGmos - from .lookup import filter_wavelengths - - factory.addClass(AstroDataGmos) - # Use the generic GMOS name for both GMOS-N and GMOS-S - addInstrumentFilterWavelengths('GMOS', filter_wavelengths) - -where `~gemini_instruments.gemini.addInstrumentFilterWavelengths` is provided -by the ``gemini`` package to perform the update in a controlled way. - -We encourage package maintainers and creators to follow such explicit -initialization methods, driven by the modules that add functionality -themselves, as opposed to active discovery methods on the core code. This -favors decoupling between modules, which is generally a good idea. - -.. rubric:: Footnotes - -.. [#keywdict] Note that the keyword dictionary is a "private" property of the - class (due to the double-underscore prefix). Each class can define its own - set, which will not be replaced by derivative classes. ``_keyword_for`` is - aware of this and will look up each class up the inheritance chain, in turn, - when looking up for keywords. - -.. [#tagset1] Notice that the example functions will return only - a `~astrodata.TagSet`, if appropriate. This is OK, remember that *every - function* in Python returns a value, which will be ``None``, implicitly, if - you don't specify otherwise. - -.. [#extver] An ``EXTVER`` keyword is not formally required as the - `astrodata.fits.read_fits` method will assign the lowest available integer - to a ``SCI`` header with no ``EXTVER`` keyword (or if its value is -1). But - we wish to be able to identify the original ``im1`` header by assigning it - an ``EXTVER`` of 1, etc. diff --git a/astrodata/doc/progmanual/containers.rst b/astrodata/doc/progmanual/containers.rst deleted file mode 100644 index fe2a1a26a7..0000000000 --- a/astrodata/doc/progmanual/containers.rst +++ /dev/null @@ -1,96 +0,0 @@ -.. containers.rst - -.. _containers: - -*************** -Data Containers -*************** - -A third, and very important part of the AstroData core package is the data -container. We have chosen to extend Astropy's |NDData| with our own -requirements, particularly lazy-loading of data using by opening the FITS files -in read-only, memory-mapping mode, and exploiting the windowing capability of -`astropy.io.fits` (using ``section``) to reduce our memory requirements, which -becomes important when reducing data (e.g., stacking). - -We'll describe here how we depart from |NDData|, and how do we integrate the -data containers with the rest of the package. Please refer to |NDData| for the -full interface. - -Our main data container is `astrodata.NDAstroData`. Fundamentally, it is -a derivative of `astropy.nddata.NDData`, plus a number of mixins to add -functionality:: - - class NDAstroData(AstroDataMixin, NDArithmeticMixin, NDSlicingMixin, NDData): - ... - -This allows us out of the box to have proper arithmetic with error -propagation, and slicing the data with the array syntax. - -Our first customization is ``NDAstroData.__init__``. It relies mostly on the -upstream initialization, but customizes it because our class is initialized -with lazy-loaded data wrapped around a custom class -(`astrodata.fits.FitsLazyLoadable`) that mimics a `astropy.io.fits` HDU -instance just enough to play along with |NDData|'s initialization code. - -``FitsLazyLoadable`` is an integral part of our memory-mapping scheme, and -among other things it will scale data on the fly, as memory-mapped FITS data -can only be read unscaled. Our NDAstroData redefines the properties ``data``, -``uncertainty``, and ``mask``, in two ways: - -* To deal with the fact that our class is storing ``FitsLazyLoadable`` - instances, not arrays, as |NDData| would expect. This is to keep data out - of memory as long as possible. - -* To replace lazy-loaded data with a real in-memory array, under certain - conditions (e.g., if the data is modified, as we won't apply the changes to the - original file!) - -Our obsession with lazy-loading and discarding data is directed to reduce -memory fragmentation as much as possible. This is a real problem that can hit -applications dealing with large arrays, particularly when using Python. Given -the choice to optimize for speed or for memory consumption, we've chosen the -latter, which is the more pressing issue. - -We've added another new property, ``window``, that can be used to -explicitly exploit the `astropy.io.fits`'s ``section`` property, to (again) -avoid loading unneeded data to memory. This property returns an instance of -``NDWindowing`` which, when sliced, in turn produces an instance of -``NDWindowingAstroData``, itself a proxy of ``NDAstroData``. This scheme may -seem complex, but it was deemed the easiest and cleanest way to achieve the -result that we were looking for. - -The base ``NDAstroData`` class provides the memory-mapping functionality, -with other important behaviors added by the ``AstroDataMixin``, which can -be used with other |NDData|-like classes (such as ``Spectrum1D``) to add -additional convenience. - -One addition is the ``variance`` property, which allows direct access and -setting of the data's uncertainty, without the user needing to explicitly wrap -it as an ``NDUncertainty`` object. Internally, the variance is stored as an -``ADVarianceUncertainty`` object, which is subclassed from Astropy's standard -``VarianceUncertainty`` class with the addition of a check for negative values -whenever the array is accessed. - -``NDAstroDataMixin`` also changes the default method of combining the ``mask`` -attributes during arithmetic operations from ``logical_or`` to ``bitwise_or``, -since the individual bits in the mask have separate meanings. - -The way slicing affects the ``wcs`` is also changed since DRAGONS regularly -uses the callable nature of ``gWCS`` objects and this is broken by the standard -slicing method. - -Finally, the additional image planes and tables stored in the ``meta`` dict -are exposed as attributes of the ``NDAstroData`` object, and any image planes -that have the same shape as the parent ``NDAstroData`` object will be handled -by ``NDWindowingAstroData``. Sections will be ignored when accessing image -planes with a different shape, as well as tables. - - -.. note:: - - We expect to make changes to ``NDAstroData`` in future releases. In particular, - we plan to make use of the ``unit`` attribute provided by the - |NDData| class and increase the use of memory-mapping by default. These - changes mostly represent increased functionality and we anticipate a high - (and possibly full) degree of backward compatibility. diff --git a/astrodata/doc/progmanual/descriptors.rst b/astrodata/doc/progmanual/descriptors.rst deleted file mode 100644 index e9ae03a28b..0000000000 --- a/astrodata/doc/progmanual/descriptors.rst +++ /dev/null @@ -1,65 +0,0 @@ -.. descriptors.rst - -.. _ad_descriptors: - -*********** -Descriptors -*********** - -Descriptors are just regular methods that translate metadata from the raw -storage (e.g., cards from FITS headers) to values useful for the user, -potentially doing some processing in between. They exist to: - -* Abstract the actual organization of the metadata; e.g. - `~gemini_instruments.gemini.AstroDataGemini` takes the detector gain from - a keyword in the FITS PHU, where `~gemini_instruments.niri.AstroDataNiri` - overrides this to provide a hard-coded value. - - More complex implementations also exist. In order to determine the gain of - a GMOS observation, `~gemini_instruments.gmos.AstroDataGmos` uses the - observation date (provided by a descriptor) to select a particular lookup - table, and then uses the values of other descriptors to select the correct - entry in the table. - -* Provide a common interface to a set of instruments. This simplifies user - training (no need to learn a different API for each instrument), and - facilitates the reuse of code for pipelines, etc. - -* Also, since FITS header keywords are limited to 8 characters, for simple - keyword → value mappings, they provide a more meaningful and readable name. - -Descriptors **should** be decorated using `~astrodata.astro_data_descriptor`. -The only function of this decorator is to ensure that the descriptor is marked -as such: it does not alter its input or output in any way. This lets the user -explore the API of an |AstroData| object via the -`~astrodata.AstroData.descriptors` property. - -Descriptors **can** be decorated with `~astrodata.core.returns_list` to -eliminate the need to code some logic. Some descriptors return single values, -while some return lists, one per extension. Typically, the former are -descriptors that refer to the entire observation (and, for MEF files, are -usually extracted from metadata in the PHU, such as ``airmass``), while the -latter are descriptors where different extensions might return different values -(and typically come from metadata in the individual HDUs, such as ``gain``). -A list is returned even if there is only one extension in the |AstroData| -object, as this allows code to be written generically to iterate over the -|AstroData| object and the descriptor return, without needing to know how many -extensions there are. The `~astrodata.core.returns_list` decorator ensures that -the descriptor returns an appropriate object (value or list), using the -following rules: - -* If the |AstroData| object is not a single slice: - - * If the undecorated descriptor returns a list, an exception is raised - if the list is not the same length as the number of extensions. - * If the undecorated descriptor returns a single value, the decorator - will turn it into a list of the correct length by copying this value. - -* If the |AstroData| object is a single slice and the undecorated - descriptor returns a list, only the first element is returned. - -An example of the use of this decorator is the NIRI -`~gemini_instruments.niri.AstroDataNiri.gain` descriptor, which reads the -value from a lookup table and simply returns it. A single value is only -appropriate if the |AstroData| object is singly-sliced and the decorator ensures -that a list is returned otherwise. diff --git a/astrodata/doc/progmanual/design.rst b/astrodata/doc/progmanual/design.rst deleted file mode 100644 index bab79e65f6..0000000000 --- a/astrodata/doc/progmanual/design.rst +++ /dev/null @@ -1,82 +0,0 @@ -.. design.rst - -.. _design: - -************** -General Design -************** - -As astronomical instruments have become more complex, there -has been an increasing need for bespoke reduction packages and pipelines to -deal with the specific needs of each instrument. Despite this -complexity, many of the reduction steps can be very similar and the overall -effort could be reduced significantly by sharing code. In practice, however, -there are often issues regarding the manner in which the data are stored -internally. The purpose of AstroData is to provide a uniform interface to the data -and metadata, in a manner that is independent both of the specific instrument -and the way the data are stored on disk, thereby facilitating this code-sharing. -It is *not* a new astronomical data format. - -One of the main features of AstroData is the use of *descriptors*, which -provide a level of abstraction between the metadata and the code accessing it. -Somebody using the AstroData interface who wishes to know the exposure time -of a particular astronomical observation represented by the ``AstroData`` object -``ad`` can simply write ``ad.exposure_time()`` without needing to concern -themselves about how that value is stored internally, for example, the name -of the FITS header keyword. These are discussed further in :ref:`ad_descriptors`. - -AstroData also provides a clearer representation of the relationships -between different parts of the data produced from a single astronomical -observation. Modern astronomical instruments often contain multiple -detectors that are read out separately and the multi-extension FITS (MEF) -format used by many institutions, including Gemini Observatory, handles -the raw data well. In this format, each detector's data and metadata is -assigned to its own extension, -while there is also a separate extension (the Primary Header Unit, -or PHU) containing additional metadata that applies to the entire -observation. However, as the data are processed, more data and/or -metadata may be added whose relationship is obscured by the limitations -of the MEF format. One example is the creation and propagation of information -describing the quality and uncertainty of the scientific data: while -this was a feature of -Gemini IRAF\ [#iraf]_, the coding required to implement it was cumbersome -and AstroData uses the `astropy.nddata.NDData` class, -as discussed in :ref:`containers`. This makes the relationship between these -data much clearer, and AstroData creates a syntax that makes readily apparent the -roles of other data and metadata that may be created during the reduction -process. - -An ``AstroData`` object therefore consists of one or more self-contained -"extensions" (data and metadata) plus additional data and metadata that is -relevant to all the extensions. In many data reduction processes, the same -operation will be performed on each extension (e.g., subtracting an overscan -region from a CCD frame) and an axiom of AstroData is that iterating over -the extensions produces AstroData "slices" which retain knowledge of the -top-level data and metadata. Since a slice has one (or more) extensions -plus this top-level (meta)data, it too is an ``AstroData`` object and, -specifically, an instance of the same subclass as its parent. - - -A final feature of AstroData is the implementation of very high-level metadata. -These data, called ``tags``, facilitate a key part of the Gemini data reduction -system, DRAGONS, by linking the astronomical data to the recipes -required to process them. They are explained in detail in :ref:`ad_tags` and the -Recipe System Programmers Manual\ [#rsprogman]_. - -.. note:: - - AstroData and DRAGONS have been developed for the reduction of data from - Gemini Observatory, which produces data in the FITS format that is still the - most widely-used format for astronomical data. In light of this, and the - limited resources in the Science User Support Department, we have only - *developed* support for FITS, even though the AstroData format is designed - to be independent of the file format. In some cases, this has led to - uncertainty and internal disagreement over where precisely to engage in - abstraction and, should AstroData support a different file format, we - may find alternative solutions that result in small, but possibly - significant, changes to the API. - - -.. [#iraf] ``_ - -.. [#rsprogman] |RSProgManual| diff --git a/astrodata/doc/progmanual/index.rst b/astrodata/doc/progmanual/index.rst deleted file mode 100644 index 751abd29a1..0000000000 --- a/astrodata/doc/progmanual/index.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. Astrodata Programmer's Manual documentation master file, created by - sphinx-quickstart on Fri Jun 1 11:08:23 2018. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -=================== -Programmer's Manual -=================== - -.. admonition:: Document ID - - PIPE-USER-104_AstrodataProgManual - -.. toctree:: - :maxdepth: 2 - - intro - design - adclass - containers - tags - descriptors diff --git a/astrodata/doc/progmanual/intro.rst b/astrodata/doc/progmanual/intro.rst deleted file mode 100644 index d2b9a93975..0000000000 --- a/astrodata/doc/progmanual/intro.rst +++ /dev/null @@ -1,45 +0,0 @@ -.. intro.rst - -.. _intro_progmanual: - -************************* -Precedents and Motivation -************************* - - -The Gemini Observatory has produced a number of tools for data processing. -Historically this has translated into a number of IRAF\ [#IRAF]_ packages but -the lack of long-term support for IRAF, coupled with the well-known -difficulty in creating robust reduction pipelines within the IRAF -environment, led to a decision -to adopt Python as a programming tool and a new -package was born: Gemini Python. Gemini Python provided tools to load and -manipulate Gemini-produced multi-extension FITS\ [#FITS]_ (MEF) files, -along with a pipeline that -allowed the construction of reduction recipes. At the center of this package -was the AstroData subpackage, which supported the abstraction of the FITS -files. - -Gemini Python reached version 1.0.1, released during November 2014. In 2015 -the Science User Support Department (SUSD) was created at Gemini, which took on the -responsibility of maintaining the software reduction tools, and started -planning future steps. With improved oversight and time and thought, it became -evident that the design of Gemini Python and, specially, of AstroData, made -further development a daunting task. - -In 2016 a decision was reached to overhaul Gemini Python. While the -principles behind AstroData were sound, the coding involved unnecessary -layers of abstraction and eschewed features of the Python language in favor -of its own implementation. Thus, -|DRAGONS| was born, with a new, simplified (and backward *incompatible*) -AstroData v2.0 (which we will refer to simply as AstroData) - -This manual documents both the high level design and some implementation -details of AstroData, together with an explanation of how to extend the -package to work for new environments. - -.. rubric:: Footnotes - -.. [#IRAF] http://iraf.net -.. [#FITS] The `Flexible Image Transport System `_ -.. [#DRAGONS] The `Data Reduction for Astronomy from Gemini Observatory North and South `_ package diff --git a/astrodata/doc/progmanual/tags.rst b/astrodata/doc/progmanual/tags.rst deleted file mode 100644 index 01a2c82b5d..0000000000 --- a/astrodata/doc/progmanual/tags.rst +++ /dev/null @@ -1,160 +0,0 @@ -.. tags.rst - -.. _ad_tags: - -**** -Tags -**** - -We described :ref:`in previous section ` how to generate tags for an -AstroData derivative. In this section we'll describe the algorithm that -generates the complete tag set out of the individual ``TagSet`` instances. The -algorithm collects all the tags in a list and then decides whether to apply -them or not following certain rules, but let's talk about ``TagSet`` first. - -``TagSet`` is actually a standard named tuple customized to generate default -values (``None``) for its missing members. Its signature is:: - - TagSet(add=None, remove=None, blocked_by=None, blocks=None, - if_present=None) - -The most common ``TagSet`` is an **additive** one: ``TagSet(['FOO', 'BAR'])``. -If all you need is to add tags, then you're done here. But the real power of -our tag generating system is that you can specify some conditions to apply a -certain ``TagSet``, or put restrictions on others. The different arguments to -``TagSet`` all expect a list (or some others work in the following way): - -* ``add``: if this ``TagSet`` is selected, then add all these members to the tag - set. -* ``remove``: if this ``TagSet`` is selected, then prevent all these members - from joining the tag set. -* ``blocked_by``: if any of the tags listed in here exist in the tag set, then - discard this ``TagSet`` altogether. -* ``blocks``: discard from the list of unprocessed ones any ``TagSet`` that - would add any of the tags listed here. -* ``if_present``: process this tag only if all the tags listed in here exist in - the tag set at this point. - -Note that ``blocked_by`` and ``blocks`` look like two sides of the same coin. -This is intentional: which one to use is up to the programmer, depending on -what will reduce the amount of typing and/or make the logic easier (sometimes one -wants to block a bunch of other tags from a single one; sometimes one wants a -tag to be blocked by a bunch of others). Furthermore, while ``blocks`` and -``blocked_by`` prevent the entire ``TagSet`` from being added if it contains a -tag affected by these, ``remove`` only affects the specific tag. - -Now, the algorithm works like this: - -#. Collect all the ``TagSet`` generated by methods in the instance that are - decorated using ``astro_data_tag``. -#. Then we sort them out: - - #. Those that subtract tags from the tag set go first (the ones with - non-empty ``remove`` or ``blocks``), allowing them to act early on - #. Those with non-empty ``blocked_by`` are moved to the end of the list, to - ensure that other tags can be generated before them. - #. Those with non-empty ``if_present`` are moved behind those with - ``blocked_by``. - -#. Now that we've sorted the tags, process them sequentially and for each one: - - #. If they require other tags to be present, make sure that this is the case. - If the requirements are not met, drop the tagset. If not... - #. Figure out if any other tag is blocking the tagset. This will be the - case if *any* of the tags to be added is in the "blocked" list, or if - any of the tags added by previous tag sets are in the ``blocked_by`` - list of the one being processed. Then... - #. If all the previous hurdles have been passed, apply the changes declared - by this tag (add, remove, and/or block others). - -Note that Python's sort algorithm is stable. This means, that if two elements -are indistinguishable from the point of view of the sorting algorithm, they are -guaranteed to stay in the same relative position. To better understand how this -affects our tags, and the algorithm itself, let's follow up with an example taken -from real code (the Gemini-generic and GMOS modules):: - - # Simple tagset, with only a constant, additive content - @astro_data_tag - def _tag_instrument(self): - return TagSet(['GMOS']) - - # Simple tagset, also with additive content. This one will - # check if the frame fits the requirements to be classified - # as "GMOS imaging". It returns a value conditionally: - # if this is not imaging, then it will return None, which - # means the algorithm will ignore the value - @astro_data_tag - def _tag_image(self): - if self.phu.get('GRATING') == 'MIRROR': - return TagSet(['IMAGE']) - - # This is a slightly more complex TagSet (but fairly simple, anyway), - # inherited by all Gemini instruments. - @astro_data_tag - def _type_gcal_lamp(self): - if self.phu.get('GCALLAMP') == 'IRhigh': - shut = self.phu.get('GCALSHUT') - if shut == 'OPEN': - return TagSet(['GCAL_IR_ON', 'LAMPON'], - blocked_by=['PROCESSED']) - elif shut == 'CLOSED': - return TagSet(['GCAL_IR_OFF', 'LAMPOFF'], - blocked_by=['PROCESSED']) - - # This tagset is only active when we detect that the frame is - # a bias. In that case we want to prevent the frame from being - # classified as "imaging" or "spectroscopy", which depend on the - # configuration of the instrument - @astro_data_tag - def _tag_bias(self): - if self.phu.get('OBSTYPE') == 'BIAS': - return TagSet(['BIAS', 'CAL'], blocks=['IMAGE', 'SPECT']) - -These four simple tag methods will serve to illustrate the algorithm. Let's pretend -that the requirements for all four of them are somehow met, meaning that we get four -``TagSet`` instances in our list, in some random order. After step 1 in the algorithm, -then, we may have collected the following list:: - - [ TagSet(['GMOS']), - TagSet(['GCAL_IR_OFF', 'LAMPOFF'], blocked_by=['PROCESSED']), - TagSet(['BIAS', 'CAL'], blocks=['IMAGE', 'SPECT']), - TagSet(['IMAGE']) ] - -The algorithm then proceeds to sort them. First, it will promote the ``TagSet`` -with non-empty ``blocks`` or ``remove``:: - - [ TagSet(['BIAS', 'CAL'], blocks=['IMAGE', 'SPECT']), - TagSet(['GMOS']), - TagSet(['GCAL_IR_OFF', 'LAMPOFF'], blocked_by=['PROCESSED']), - TagSet(['IMAGE']) ] - -Note that the other three ``TagSet`` stay in exactly the same order. Now the -algorithm will sort the list again, moving the ones with non-empty -``blocked_by`` to the end:: - - [ TagSet(['BIAS', 'CAL'], blocks=['IMAGE', 'SPECT']), - TagSet(['GMOS']), TagSet(['IMAGE']), - TagSet(['GCAL_IR_OFF', 'LAMPOFF'], blocked_by=['PROCESSED']) ] - -Note that at each step, all the instances (except the ones "being moved") have -kept the same position relative to each other -here's where the "stability" of -the sorting comes into play,- ensuring that each step does not affect the previous -one. Finally, there are no ``if_present`` in our example, so no more instances are -moved around. - -Now the algorithm prepares three empty sets (``tags``, ``removals``, and ``blocked``), -and starts iterating over the ``TagSet`` list. - - 1. For the first ``TagSet`` there are no blocks or removals, so we just add its - contents to the current sets: ``tags = {'BIAS', 'CAL'}``, - ``blocked = {'IMAGE', 'SPECT'}``. - 2. Then comes ``TagSet(['GMOS'])``. Again, there are no removals in place, and - ``GMOS`` is not in the list of blocked tags. Thus, we just add it to the current - tag set: ``tags = {'BIAS', 'CAL', 'GMOS'}``. - 3. When processing ``TagSet(['IMAGE'])``, the algorithm observes that this ``IMAGE`` - is in the ``blocked`` set, and stops processing this tag set. - 4. Finally, neither ``GCAL_IR_OFF`` nor ``LAMPOFF`` are in ``blocked``, and - ``PROCESSED`` is not in ``tags``, meaning that we can add this tag set to - the final one. - -Our result will look something like: ``{'BIAS', 'CAL', 'GMOS', 'GCAL_IR_OFF', 'LAMPOFF'}`` diff --git a/astrodata/doc/usermanual/data.rst b/astrodata/doc/usermanual/data.rst deleted file mode 100644 index b7d54a700d..0000000000 --- a/astrodata/doc/usermanual/data.rst +++ /dev/null @@ -1,903 +0,0 @@ -.. data.rst - -.. _pixel-data: - -********** -Pixel Data -********** - -**Try it yourself** - -Download the data package (:ref:`datapkg`) if you wish to follow along and run the -examples. Then :: - - $ cd /ad_usermanual/playground - $ python - -Then import core astrodata and the Gemini astrodata configurations. :: - - >>> import astrodata - >>> import gemini_instruments - - -Operate on Pixel Data -===================== -The pixel data are stored in the ``AstroData`` object as a list of -``NDAstroData`` objects. The ``NDAstroData`` is a subclass of Astropy's -``NDData`` class which combines in one "package" the pixel values, the -variance, and the data quality plane or mask (as well as associated meta-data). -The data can be retrieved as a standard NumPy ``ndarray``. - -In the sections below, we will present several typical examples of data -manipulation. But first let's start with a quick example on how to access -the pixel data. :: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - - >>> the_data = ad[1].data - >>> type(the_data) - - - >>> # Loop through the extensions - >>> for ext in ad: - ... the_data = ext.data - ... print(the_data.sum()) - 333071030 - 335104458 - 333170484 - 333055206 - -In this example, we first access the pixels for the second extensions. -Remember that in Python, list are zero-indexed, hence we access the second -extension as ``ad[1]``. The ``.data`` attribute contains a NumPy ``ndarray``. -In the for-loop, for each extension, we get the data and use the NumPy -``.sum()`` method to sum the pixel values. Anything that can be done -with a ``ndarray`` can be done on ``AstroData`` pixel data. - - -Arithmetic on AstroData Objects -=============================== -``AstroData`` objects support basic in-place arithmetics with these methods: - -+----------------+-------------+ -| addition | .add() | -+----------------+-------------+ -| subtraction | .subtract() | -+----------------+-------------+ -| multiplication | .multiply() | -+----------------+-------------+ -| division | .divide() | -+----------------+-------------+ - -Normal, not in-place, arithmetics is also possible using the standard -operators, ``+``, ``-``, ``*``, and ``/``. - -The big advantage of using ``AstroData`` to do arithmetics is that the -variance and mask, if present, will be propagated through to the output -``AstroData`` object. We will explore the variance propagation in the next -section and mask usage later in this chapter. - -Simple operations ------------------ -Here are a few examples of arithmetics on ``AstroData`` objects.:: - - >>> ad = astrodata.open('../playdata/N20170521S0925_forStack.fits') - - >>> # Addition - >>> ad.add(50.) - >>> ad = ad + 50. - >>> ad += 50. - - >>> # Subtraction - >>> ad.subtract(50.) - >>> ad = ad - 50. - >>> ad -= 50. - - >>> # Multiplication (Using a descriptor) - >>> ad.multiply(ad.exposure_time()) - >>> ad = ad * ad.exposure_time() - >>> ad *= ad.exposure_time() - - >>> # Division (Using a descriptor) - >>> ad.divide(ad.exposure_time()) - >>> ad = ad / ad.exposure_time() - >>> ad /= ad.exposure_time() - -When the syntax ``adout = adin + 1`` is used, the output variable is a copy -of the original. In the examples above we reassign the result back onto the -original. The two other forms, ``ad.add()`` and ``ad +=`` are in-place -operations. - -When a descriptor returns a list because the value changes for each -extension, a for-loop is needed:: - - >>> for (ext, gain) in zip(ad, ad.gain()): - ... ext.multiply(gain) - -If you want to do the above but on a new object, leaving the original unchanged, -use ``deepcopy`` first. :: - - >>> from copy import deepcopy - >>> adcopy = deepcopy(ad) - >>> for (ext, gain) in zip(adcopy, adcopy.gain()): - ... ext.multiply(gain) - - -Operator Precedence -------------------- -The ``AstroData`` arithmetics methods can be stringed together but beware that -there is no operator precedence when that is done. For arithmetics that -involve more than one operation, it is probably safer to use the normal -Python operator syntax. Here is a little example to illustrate the difference. - -:: - - >>> ad.add(5).multiply(10).subtract(5) - - >>> # means: ad = ((ad + 5) * 10) - 5 - >>> # NOT: ad = ad + (5 * 10) - 5 - -This is because the methods modify the object in-place, one operation after -the other from left to right. This also means that the original is modified. - -This example applies the expected operator precedence:: - - >>> ad = ad + ad * 3 - 40. - >>> # means: ad = ad + (ad * 3) - 40. - -If you need a copy, leaving the original untouched, which is sometimes useful -you can use ``deepcopy`` or just use the normal operator and assign to a new -variable.:: - - >>> adnew = ad + ad * 3 - 40. - - -Variance -======== -When doing arithmetic on an ``AstroData`` object, if a variance is present -it will be propagated appropriately to the output no matter which syntax -you use (the methods or the Python operators). - -Adding a Variance Plane ------------------------ -In this example, we will add the poisson noise to an ``AstroData`` dataset. -The data is still in ADU, therefore the poisson noise as variance is -``signal / gain``. We want to set the variance for each of the pixel -extensions. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - - >>> for (extension, gain) in zip(ad, ad.gain()): - ... extension.variance = extension.data / gain - -Check ``ad.info()``, you will see a variance plane for each of the four -extensions. - -Automatic Variance Propagation ------------------------------- -As mentioned before, if present, the variance plane will be propagated to the -resulting ``AstroData`` object when doing arithmetics. The variance -calculation assumes that the data are not correlated. - -Let's look into an example. - -:: - - >>> # output = x * x - >>> # var_output = var * x^2 + var * x^2 - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - - >>> ad[1].data[50,50] - 56.160931 - >>> ad[1].variance[50,50] - 96.356529 - >>> adout = ad * ad - >>> adout[1].data[50,50] - 3154.05 - >>> adout[1].variance[50,50] - 607826.62 - -Data Quality Plane -================== -The NDData ``mask`` stores the data quality plane. The simplest form is a -True/False array of the same size at the pixel array. In Astrodata we favor -a bit array that allows for additional information about why the pixel is being -masked. For example at Gemini here is our bit mapping for bad pixels. - -+---------------+-------+ -| Meaning | Value | -+===============+=======+ -| Bad pixel | 1 | -+---------------+-------+ -| Non Linear | 2 | -+---------------+-------+ -| Saturated | 4 | -+---------------+-------+ -| Cosmic Ray | 8 | -+---------------+-------+ -| No Data | 16 | -+---------------+-------+ -| Overlap | 32 | -+---------------+-------+ -| Unilluminated | 64 | -+---------------+-------+ - -(These definitions are located in ``geminidr.gemini.lookups.DQ_definitions``.) - -So a pixel marked 10 in the mask, would be a "non-linear" "cosmic ray". The -``AstroData`` masks are propagated with bitwise-OR operation. For example, -let's say that we are stacking frames. A pixel is set as bad (value 1) -in one frame, saturated in another (value 4), and fine in all the other -the frames (value 0). The mask of the resulting stack will be assigned -a value of 5 for that pixel. - -These bitmasks will work like any other NumPy True/False mask. There is a -usage example below using the mask. - -The mask can be accessed as follow:: - - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - >>> ad.info() - - >>> ad[2].mask - - -Display -======= -Since the data is stored in the ``AstroData`` object as a NumPy ``ndarray`` -any tool that works on ``ndarray`` can be used. To display to DS9 there -is the ``imexam`` package. The ``numdisplay`` package is still available for -now but it is no longer supported by STScI. We will show -how to use ``imexam`` to display and read the cursor position. Read the -documentation on that tool to learn more about what else it has -to offer. - -Displaying with imexam ----------------------- - -Here is an example how to display pixel data to DS9 with ``imexam``. You must -start ``ds9`` before running this example. - -:: - - >>> import imexam - >>> ad = astrodata.open('../playdata/N20170521S0925_forStack.fits') - - # Connect to the DS9 window (should already be opened.) - >>> ds9 = imexam.connect(list(imexam.list_active_ds9())[0]) - - >>> ds9.view(ad[0].data) - - # To scale "a la IRAF" - >>> ds9.view(ad[0].data) - >>> ds9.scale('zscale') - - # To set the mininum and maximum scale values - >>> ds9.view(ad[0].data) - >>> ds9.scale('limits 0 2000') - - -Retrieving cursor position with imexam --------------------------------------- - -The function ``readcursor()`` can be used to retrieve cursor -position in pixel coordinates. Note that it will **not** respond to -mouse clicks, **only** keyboard entries are acknowledged. - -When invoked, ``readcursor()`` will stop the flow of the program and wait -for the user to put the cursor on top of the image and type a key. A -tuple with three values will be returned: the x and -y coordinates **in 0-based system**, and the value of the key the user -hit. - -:: - - >>> import imexam - >>> ad = astrodata.open('../playdata/N20170521S0925_forStack.fits') - - # Connect to the DS9 window (should already be opened.) - # and display - >>> ds9 = imexam.connect(list(imexam.list_active_ds9())[0]) - >>> ds9.view(ad[0].data) - >>> ds9.scale('zscale') - - - >>> cursor_coo = ds9.readcursor() - >>> print(cursor_coo) - - # To extract only the x,y coordinates - >>> (xcoo, ycoo) = cursor_coo[:2] - >>> print(xcoo, ycoo) - - # If you are also interested in the keystroke - >>> keystroke = cursor_coo[2] - >>> print('You pressed this key: %s' % keystroke) - - -Useful tools from the NumPy, SciPy, and Astropy Packages -======================================================== -Like for the Display section, this section is not really specific to -Astrodata but is rather a quick show-and-tell of a few things that can -be done on the pixels with the big scientific packages NumPy, SciPy, -and Astropy. - -Those three packages are very large and rich. They have their own -extensive documentation and it is highly recommend for the users to learn about what -they have to offer. It might save you from re-inventing the wheel. - -The pixels, the variance, and the mask are stored as NumPy ``ndarray``'s. -Let us go through some basic examples, just to get a feel for how the -data in an ``AstroData`` object can be manipulated. - -ndarray -------- -The data are contained in NumPy ``ndarray`` objects. Any tools that works -on an ``ndarray`` can be used with Astrodata. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - - >>> data = ad[0].data - - >>> # Shape of the array. (equivalent to NAXIS2, NAXIS1) - >>> data.shape - (2112, 288) - - >>> # Value of a pixel at "IRAF" or DS9 coordinates (100, 50) - >>> data[49,99] - 455 - - >>> # Data type - >>> data.dtype - dtype('uint16') - -The two most important thing to remember for users coming from the IRAF -world or the Fortran world are that the array has the y-axis in the first -index, the x-axis in the second, and that the array indices are zero-indexed, -not one-indexed. The examples above illustrate those two critical -differences. - -It is sometimes useful to know the data type of the values stored in the -array. Here, the file is a raw dataset, fresh off the telescope. No -operations has been done on the pixels yet. The data type of Gemini raw -datasets is always "Unsigned integer (0 to 65535)", ``uint16``. - -.. warning:: - Beware that doing arithmetic on ``uint16`` can lead to unexpected - results. This is a NumPy behavior. If the result of an operation - is higher than the range allowed by ``uint16``, the output value will - be "wrong". The data type will not be modified to accommodate the large - value. A workaround, and a safety net, is to multiply the array by - ``1.0`` to force the conversion to a ``float64``. :: - - >>> a = np.array([65535], dtype='uint16') - >>> a + a - array([65534], dtype=uint16) - >>> 1.0*a + a - array([ 131070.]) - - - -Simple Numpy Statistics ------------------------ -A lot of functions and methods are available in NumPy to probe the array, -too many to cover here, but here are a couple examples. - -:: - - >>> import numpy as np - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> data = ad[0].data - - >>> data.mean() - >>> np.average(data) - >>> np.median(data) - -Note how ``mean()`` is called differently from the other two. ``mean()`` -is a ``ndarray`` method, the others are NumPy functions. The implementation -details are clearly well beyond the scope of this manual, but when looking -for the tool you need, keep in mind that there are two sets of functions to -look into. Duplications like ``.mean()`` and ``np.average()`` can happen, -but they are not the norm. The readers are strongly encouraged to refer to -the NumPy documentation to find the tool they need. - - -Clipped Statistics ------------------- -It is common in astronomy to apply clipping to the statistics, a clipped -average, for example. The NumPy ``ma`` module can be used to create masks -of the values to reject. In the examples below, we calculated the clipped -average of the first pixel extension with a rejection threshold set to -+/- 3 times the standard deviation. - -Before Astropy, it was possible to do something like that with only -NumPy tools, like in this example:: - - >>> import numpy as np - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> data = ad[0].data - - >>> stddev = data.std() - >>> mean = data.mean() - - >>> clipped_mean = np.ma.masked_outside(data, mean-3*stddev, mean+3*stddev).mean() - -There is no iteration in that example. It is a straight one-time clipping. - -For something more robust, there is an Astropy function that can help, in -particular by adding an iterative process to the calculation. Here is -how it is done:: - - >>> import numpy as np - >>> from astropy.stats import sigma_clip - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> data = ad[0].data - - >>> clipped_mean = np.ma.mean(sigma_clip(data, sigma=3)) - - -Filters with SciPy ------------------- -Another common operation is the filtering of an image, for example convolving -with a gaussian filter. The SciPy module ``ndimage.filters`` offers -several functions for image processing. See the SciPy documentation for -more information. - -The example below applies a gaussian filter to the pixel array. - -:: - - >>> from scipy.ndimage import filters - >>> import imexam - - >>> ad = astrodata.open('../playdata/N20170521S0925_forStack.fits') - >>> data = ad[0].data - - >>> # We need to prepare an array of the same size and shape as - >>> # the data array. The result will be put in there. - >>> convolved_data = np.zeros(data.size).reshape(data.shape) - - >>> # We now apply the convolution filter. - >>> sigma = 10. - >>> filters.gaussian_filter(data, sigma, output=convolved_data) - - >>> # Let's visually compare the convolved image with the original - >>> ds9 = imexam.connect(list(imexam.list_active_ds9())[0]) - >>> ds9.view(data) - >>> ds9.scale('zscale') - >>> ds9.frame(2) - >>> ds9.view(convolved_data) - >>> ds9.scale('zscale') - >>> ds9.blink() - >>> # When you are convinced it's been convolved, stop the blinking. - >>> ds9.blink(blink=False) - -Note that there is an Astropy way to do this convolution, with tools in -``astropy.convolution`` package. Beware that for this particular kernel -we have found that the Astropy ``convolve`` function is extremely slow -compared to the SciPy solution. -This is because the SciPy function is optimized for a Gaussian convolution -while the generic ``convolve`` function in Astropy can take in any kernel. -Being able to take in any kernel is a very powerful feature, but the cost -is time. The lesson here is do your research, and find the best tool for -your needs. - - -Many other tools ----------------- -There are many, many other tools available out there. Here are the links to -the three big projects we have featured in this section. - -* NumPy: `www.numpy.org `_ -* SciPy: `www.scipy.org `_ -* Astropy: `www.astropy.org `_ - -Using the Astrodata Data Quality Plane -====================================== -Let us look at an example where the use of the Astrodata mask is -necessary to get correct statistics. A GMOS imaging frame has large sections -of unilluminated pixels; the edges are not illuminated and there are two -bands between the three CCDs that represent the physical gap between the -CCDs. Let us have a look at the pixels to have a better sense of the -data:: - - >>> ad = astrodata.open('../playdata/N20170521S0925_forStack.fits') - >>> import imexam - >>> ds9 = imexam.connect(list(imexam.list_active_ds9())[0]) - - >>> ds9.view(ad[0].data) - >>> ds9.scale('zscale') - -See how the right and left portions of the frame are not exposed to the sky, -and the 45 degree angle cuts of the four corners. The chip gaps too. -If we wanted to do statistics on the whole frames, we certainly would not want -to include those unilluminated areas. We would want to mask them out. - -Let us have a look at the mask associated with that image:: - - >>> ds9.view(ad[0].mask) - >>> ds9.scale('zscale') - -The bad sections are all white (pixel value > 0). There are even some -illuminated pixels that have been marked as bad for a reason or another. - -Let us use that mask to reject the pixels with no or bad information and -do calculations only on the good pixels. For the sake of simplicity we will -just do an average. This is just illustrative. We show various ways to -accomplish the task; choose the one that best suits your need or that you -find most readable. - -:: - - >>> import numpy as np - - >>> # For clarity... - >>> data = ad[0].data - >>> mask = ad[0].mask - - >>> # Reject all flagged pixels and calculate the mean - >>> np.mean(data[mask == 0]) - >>> np.ma.masked_array(data, mask).mean() - - >>> # Reject only the pixels flagged "no_data" (bit 16) - >>> np.mean(data[(mask & 16) == 0]) - >>> np.ma.masked_array(data, mask & 16).mean() - >>> np.ma.masked_where(mask & 16, data).mean() - -The "long" form with ``np.ma.masked_*`` is useful if you are planning to do -more than one operation on the masked array. For example:: - - >>> clean_data = np.ma.masked_array(data, mask) - >>> clean_data.mean() - >>> np.ma.median(clean_data) - >>> clean_data.max() - - -Manipulate Data Sections -======================== -So far we have shown examples using the entire data array. It is possible -to work on sections of that array. If you are already familiar with -Python, you probably already know how to do most if not all of what is in -this section. For readers new to Python, and especially those coming -from IRAF, there are a few things that are worth explaining. - -When indexing a NumPy ``ndarray``, the left most number refers to the -highest dimension's axis. For example, in a 2D array, the IRAF section -are in (x-axis, y-axis) format, while in Python they are in -(y-axis, x-axis) format. Also important to remember is that the ``ndarray`` -is 0-indexed, rather than 1-indexed like in Fortran or IRAF. - -Putting it all together, a pixel position (x,y) = (50,75) in IRAF or from -the cursor on a DS9 frame, is accessed in Python as ``data[74,49]``. -Similarly, the IRAF section [10:20, 30:40] translate in Python to -[9:20, 29:40]. Also remember that when slicing in Python, the upper limit -of the slice is not included in the slice. This is why here we request -20 and 40 rather 19 and 39. - -Let's put it in action. - -Basic Statistics on Section ---------------------------- -In this example, we do simple statistics on a section of the image. - -:: - - >>> import numpy as np - - >>> ad = astrodata.open('../playdata/N20170521S0925_forStack.fits') - >>> data = ad[0].data - - >>> # Get statistics for a 25x25 pixel-wide box centered on pixel - >>> # (50,75) (DS9 frame coordinate) - >>> xc = 49 - >>> yc = 74 - >>> buffer = 25 - >>> (xlow, xhigh) = (xc - buffer//2, xc + buffer//2 + 1) - >>> (ylow, yhigh) = (yc - buffer//2, yc + buffer//2 + 1) - >>> # The section is [62:87, 37:62] - >>> stamp = data[ylow:yhigh, xlow:xhigh] - >>> mean = stamp.mean() - >>> median = np.median(stamp) - >>> stddev = stamp.std() - >>> minimum = stamp.min() - >>> maximum = stamp.max() - - >>> print(' Mean Median Stddev Min Max\n \ - ... %.2f %.2f %.2f %.2f %.2f' % \ - ... (mean, median, stddev, minimum, maximum)) - -Have you noticed that the median is calculated with a function rather -than a method? This is simply because the ``ndarray`` object does not -have a method to calculate the median. - -Example - Overscan Subtraction with Trimming --------------------------------------------- -Several concepts from previous sections and chapters are used in this -example. The Descriptors are used to retrieve the overscan section and -the data section information from the headers. Statistics are done on the -NumPy ``ndarray`` representing the pixel data. Astrodata arithmetics is -used to subtract the overscan level. Finally, the overscan section is -trimmed off and the modified ``AstroData`` object is written to a new file -on disk. - -To make the example more complete, and to show that when the pixel data -array is trimmed, the variance (and mask) arrays are also trimmed, let us -add a variance plane to our raw data frame. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - - >>> for (extension, gain) in zip(ad, ad.gain()): - ... extension.variance = extension.data / gain - ... - - >>> # Here is how the data structure looks like before the trimming. - >>> ad.info() - Filename: ../playdata/N20170609S0154.fits - Tags: ACQUISITION GEMINI GMOS IMAGE NORTH RAW SIDEREAL UNPREPARED - - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2112, 288) uint16 - .variance ndarray (2112, 288) float64 - [ 1] science NDAstroData (2112, 288) uint16 - .variance ndarray (2112, 288) float64 - [ 2] science NDAstroData (2112, 288) uint16 - .variance ndarray (2112, 288) float64 - [ 3] science NDAstroData (2112, 288) uint16 - .variance ndarray (2112, 288) float64 - - >>> # Let's operate on the first extension. - >>> # - >>> # The section descriptors return the section in a Python format - >>> # ready to use, 0-indexed. - >>> oversec = ad[0].overscan_section() - >>> datasec = ad[0].data_section() - - >>> # Measure the overscan level - >>> mean_overscan = ad[0].data[oversec.y1: oversec.y2, oversec.x1: oversec.x2].mean() - - >>> # Subtract the overscan level. The variance will be propagated. - >>> ad[0].subtract(mean_overscan) - - >>> # Trim the data to remove the overscan section and keep only - >>> # the data section. Note that the WCS will be automatically - >>> # adjusted when the trimming is done. - >>> # - >>> # Here we work on the NDAstroData object to have the variance - >>> # trimmed automatically to the same size as the science array. - >>> # To reassign the cropped NDAstroData, we use the reset() method. - >>> ad[0].reset(ad[0].nddata[datasec.y1:datasec.y2, datasec.x1:datasec.x2]) - - >>> # Now look at the dimensions of the first extension, science - >>> # and variance. That extension is smaller than the others. - >>> ad.info() - Filename: ../playdata/N20170609S0154.fits - Tags: ACQUISITION GEMINI GMOS IMAGE NORTH RAW SIDEREAL UNPREPARED - - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2112, 256) float64 - .variance ndarray (2112, 256) float64 - [ 1] science NDAstroData (2112, 288) uint16 - .variance ndarray (2112, 288) float64 - [ 2] science NDAstroData (2112, 288) uint16 - .variance ndarray (2112, 288) float64 - [ 3] science NDAstroData (2112, 288) uint16 - .variance ndarray (2112, 288) float64 - - >>> # We can write this to a new file - >>> ad.write('partly_overscan_corrected.fits') - -A new feature presented in this example is the ability to work on the -``NDAstroData`` object directly. This is particularly useful when cropping -the science pixel array as one will want the variance and the mask arrays -cropped exactly the same way. Taking a section of the ``NDAstroData`` -object (ad[0].nddata[y1:y2, x1:x2]), instead of just the ``.data`` array, -does all that for us. - -To reassign the cropped ``NDAstroData`` to the extension one uses the -``.reset()`` method as shown in the example. - -Of course to do the overscan correction correctly and completely, one would -loop over all four extensions. But that's the only difference. - -Data Cubes -========== -Reduced Integral Field Unit (IFU) data is commonly represented as a cube, -a three-dimensional array. The ``data`` component of an ``AstroData`` -object extension can be such a cube, and it can be manipulated and explored -with NumPy, AstroPy, SciPy, imexam, like we did already in this section -with 2D arrays. We can use matplotlib to plot the 1D spectra represented -in the third dimension. - -In Gemini IFU cubes, the first axis is the X-axis, the second, the Y-axis, -and the wavelength is in the third axis. Remember that in a ``ndarray`` -that order is reversed (wlen, y, x). - -In the example below we "collapse" the cube along the wavelenth axis to -create a "white light" image and display it. Then we plot a 1D spectrum -from a given (x,y) position. - -:: - - >>> import imexam - >>> import matplotlib.pyplot as plt - - >>> ds9 = imexam.connect(list(imexam.list_active_ds9())[0]) - - >>> adcube = astrodata.open('../playdata/gmosifu_cube.fits') - >>> adcube.info() - - >>> # Sum along the wavelength axis to create a "white light" image - >>> summed_image = adcube[0].data.sum(axis=0) - >>> ds9.view(summed_image) - >>> ds9.scale('minmax') - - >>> # Plot a 1-D spectrum from the spatial position (14,25). - >>> plt.plot(adcube[0].data[:,24,13]) - >>> plt.show() # might be needed, depends on matplotlibrc interactive setting - - -Now that is nice but it would be nicer if we could plot the x-axis in units -of Angstroms instead of pixels. We use the AstroData's WCS handler, which is -based on ``gwcs.wcs.WCS`` to get the necessary information. A particularity -of ``gwcs.wcs.WCS`` is that it refers to the axes in the "natural" way, -(x, y, wlen) contrary to Python's (wlen, y, x). It truly requires you to pay -attention. - -:: - - >>> import matplotlib.pyplot as plt - - >>> adcube = astrodata.open('../playdata/gmosifu_cube.fits') - - # We get the wavelength axis in Angstroms at the position we want to - # extract, x=13, y=24. - # The wcs call returns a 3-element list, the third element ([2]) contains - # the wavelength values for each pixel along the wavelength axis. - - >>> length_wlen_axis = adcube[0].shape[0] # (wlen, y, x) - >>> wavelengths = adcube[0].wcs(13, 24, range(length_wlen_axis))[2] # (x, y, wlen) - - # We get the intensity along that axis - >>> intensity = adcube[0].data[:, 24, 13] # (wlen, y, x) - - # We plot - >>> plt.clf() - >>> plt.plot(wavelengths, intensity) - >>> plt.show() - - -Plot Data -========= -The main plotting package in Python is ``matplotlib``. We have used it in the -previous section on data cubes to plot a spectrum. There is also the project -called ``imexam`` which provides astronomy-specific tools for the -exploration and measurement of data. We have also used that package above to -display images to DS9. - -In this section we absolutely do not aim at covering all the features of -either package but rather to give a few examples that can get the readers -started in their exploration of the data and of the visualization packages. - -Refer to the projects web pages for full documentation. - -* Matplotlib: `https://matplotlib.org `_ -* imexam: `https://github.com/spacetelescope/imexam `_ - -Matplotlib ----------- -With Matplotlib you have full control on your plot. You do have to do a bit -for work to get it perfect though. However it can produce publication -quality plots. Here we just scratch the surface of Matplotlib. - -:: - - >>> import numpy as np - >>> import matplotlib.pyplot as plt - >>> from astropy import wcs - - >>> ad_image = astrodata.open('../playdata/N20170521S0925_forStack.fits') - >>> ad_spectrum = astrodata.open('../playdata/estgsS20080220S0078.fits') - - >>> # Line plot from image. Row #1044 (y-coordinate) - >>> line_index = 1043 - >>> line = ad_image[0].data[line_index, :] - >>> plt.clf() - >>> plt.plot(line) - >>> plt.show() - - >>> # Column plot from image, averaging across 11 pixels around colum #327 - >>> col_index = 326 - >>> width = 5 - >>> xlow = col_index - width - >>> xhigh = col_index + width + 1 - >>> thick_column = ad_image[0].data[:, xlow:xhigh] - >>> plt.clf() - >>> plt.plot(thick_column.mean(axis=1)) # mean along the width. - >>> plt.show() - >>> plt.ylim(0, 50) # Set the y-axis range - >>> plt.plot(thick_column.mean(axis=1)) - >>> plt.show() - - >>> # Contour plot for a section of an image. - >>> center = (1646, 2355) - >>> width = 15 - >>> xrange = (center[1]-width//2, center[1] + width//2 + 1) - >>> yrange = (center[0]-width//2, center[0] + width//2 + 1) - >>> blob = ad_image[0].data[yrange[0]:yrange[1], xrange[0]:xrange[1]] - >>> plt.clf() - >>> plt.imshow(blob, cmap='gray', origin='lower') - >>> plt.contour(blob) - >>> plt.show() - - >>> # Spectrum in pixels - >>> plt.clf() - >>> plt.plot(ad_spectrum[0].data) - >>> plt.show() - - >>> # Spectrum in Angstroms - >>> spec_wcs = wcs.WCS(ad_spectrum[0].hdr) - >>> pixcoords = np.array(range(ad_spectrum[0].data.shape[0])) - >>> wlen = spec_wcs.wcs_pix2world(pixcoords, 0)[0] - >>> plt.clf() - >>> plt.plot(wlen, ad_spectrum[0].data) - >>> plt.show() - - -imexam ------- -For those who have used IRAF, ``imexam`` is a well-known tool. The Python -``imexam`` reproduces many of of the features of its IRAF predecesor, the interactive mode of -course, but it also offers programmatic tools. One can even control DS9 -from Python. As for Matplotlib, here we really just scratch the surface of -what ``imexam`` has to offer. - -:: - - >>> import imexam - >>> from imexam.imexamine import Imexamine - - >>> ad_image = astrodata.open('../playdata/N20170521S0925_forStack.fits') - - # Display the image - >>> ds9 = imexam.connect(list(imexam.list_active_ds9())[0]) - >>> ds9.view(ad_image[0].data) - >>> ds9.scale('zscale') - - # Run in interactive mode. Try the various commands. - >>> ds9.imexam() - - # Use the programmatic interface - # First initialize an Imexamine object. - >>> plot = Imexamine() - - # Line plot from image. Row #1044 (y-coordinate) - >>> line_index = 1043 - >>> plot.plot_line(0, line_index, ad_image[0].data) - - # Column plot from image, averaging across 11 pixels around colum #327 - # There is no setting for this, so we have to do something similar - # to what we did with matplotlib. - >>> col_index = 326 - >>> width = 5 - >>> xlow = col_index - width - >>> xhigh = col_index + width + 1 - >>> thick_column = ad_image[0].data[:, xlow:xhigh] - >>> mean_column = thick_column.mean(axis=1) - >>> plot.plot_column(0, 0, np.expand_dims(mean_column, 1)) - - >>> # Contour plot for a section of an image. - >>> center = (1646, 2355) # in python coordinates - >>> width = 15 - >>> plot.contour_pars['ncolumns'][0] = width - >>> plot.contour_pars['nlines'][0] = width - >>> plot.contour(center[1], center[0], ad_image[0].data) diff --git a/astrodata/doc/usermanual/headers.rst b/astrodata/doc/usermanual/headers.rst deleted file mode 100644 index fec6cdee9c..0000000000 --- a/astrodata/doc/usermanual/headers.rst +++ /dev/null @@ -1,302 +0,0 @@ -.. headers.rst - -.. _headers: - -******************** -Metadata and Headers -******************** - -**Try it yourself** - -Download the data package (:ref:`datapkg`) if you wish to follow along and run the -examples. Then :: - - $ cd /ad_usermanual/playground - $ python - -You need to import Astrodata and the Gemini instrument configuration package. - -:: - - >>> import astrodata - >>> import gemini_instruments - -Astrodata Descriptors -===================== - -We show in this chapter how to use the Astrodata Descriptors. But first -let's explain what they are. - -Astrodata Descriptors provide a "header-to-concept" mapping that allows the -user to access header information from a unique interface, regardless of -which instrument the dataset is from. Like for the Astrodata Tags, the -mapping is coded in a configuration package separate from core Astrodata. -For Gemini instruments, that package is named ``gemini_instruments``. - -For example, if the user is interested to know the effective filter used -for an observation, normally one needs to know which specific keyword or -set of keywords to look at for that instrument. However, once the concept -of "filter" is coded as a Descriptor, the user only needs to call the -``filter_name()`` descriptor to retrieve the information. - -The Descriptors are closely associated with the Astrodata Tags. In fact, -they are implemented in the same ``AstroData`` class as the tags. Once -the specific ``AstroData`` class is selected (upon opening the file), all -the tags and descriptors for that class are defined. For example, all the -descriptor functions of GMOS data, ie. the functions that map a descriptor -concept to the actual header content, are defined in the ``AstroDataGmos`` -class. - -This is all completely transparent to the user. One simply opens the data -file and all the descriptors are ready to be used. - -.. note:: - Of course if the Descriptors have not been implemented for that specific - data, they will not work. They should all be defined for Gemini data. - For other sources, the headers can be accessed directly, one keyword at - a time. This type of access is discussed below. This is also useful - when the information needed is not associated with one of the standard - descriptors. - -To get the list of descriptors available for an ``AstroData`` object:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> ad.descriptors - ('airmass', 'amp_read_area', 'ao_seeing', ... - ...) - -Most Descriptor names are readily understood, but one can get a short -description of what the Descriptor refers to by calling the Python help -function. For example:: - - >>> help(ad.airmass) - >>> help(ad.filter_name) - -The full list of standard descriptors is available in the Appendix -:ref:`descriptors`. - -Accessing Metadata -================== - -Accessing Metadata with Descriptors ------------------------------------ -Whenever possible the Descriptors should be used to get information from -headers. This allows for maximum re-usability of the code as it will then -work on any datasets with an ``AstroData`` class. - -Here are a few examples using Descriptors:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - - >>> #--- print a value - >>> print('The airmass is : ', ad.airmass()) - The airmass is : 1.089 - - >>> #--- use a value to control the flow - >>> if ad.exposure_time() < 240.: - ... print('This is a short exposure.') - ... else: - ... print('This is a long exposure.') - This is a short exposure. - - >>> #--- multiply all extensions by their respective gain - >>> for ext, gain in zip(ad, ad.gain()): - ... ext *= gain - - >>> #--- do arithmetics - >>> fwhm_pixel = 3.5 - >>> fwhm_arcsec = fwhm_pixel * ad.pixel_scale() - -The return values for Descriptors depend on the nature of the information -being requested and the number of extensions in the ``AstroData`` object. -When the value has words, it will be string, if it is a number -it will be a float or an integer. -The dataset used in this section has 4 extensions. When the descriptor -value can be different for each extension, the descriptor will return a -Python list. - -:: - - >>> ad.airmass() - 1.089 - >>> ad.gain() - [2.03, 1.97, 1.96, 2.01] - >>> ad.filter_name() - 'open1-6&g_G0301' - -Some descriptors accept arguments. For example:: - - >>> ad.filter_name(pretty=True) - 'g' - -A full list of standard descriptors is available in the Appendix -:ref:`descriptors`. - - -Accessing Metadata Directly ---------------------------- -Not all header content is mapped to Descriptors, nor should it. Direct access -is available for header content falling outside the scope of the descriptors. - -One important thing to keep in mind is that the PHU (Primary Header Unit) and -the extension headers are accessed slightly differently. The attribute -``phu`` needs to be used for the PHU, and ``hdr`` for the extension headers. - -Here are some examples of direct header access:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - - >>> #--- Get keyword value from the PHU - >>> ad.phu['AOFOLD'] - 'park-pos.' - - >>> #--- Get keyword value from a specific extension - >>> ad[0].hdr['CRPIX1'] - 511.862999160781 - - >>> #--- Get keyword value from all the extensions in one call. - >>> ad.hdr['CRPIX1'] - [511.862999160781, 287.862999160781, -0.137000839218696, -224.137000839219] - - - -Whole Headers -------------- -Entire headers can be retrieved as ``fits`` ``Header`` objects:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> type(ad.phu) - - >>> type(ad[0].hdr) - - -In interactive mode, it is possible to print the headers on the screen as -follows:: - - >>> ad.phu - SIMPLE = T / file does conform to FITS standard - BITPIX = 16 / number of bits per data pixel - NAXIS = 0 / number of data axes - .... - - >>> ad[0].hdr - XTENSION= 'IMAGE ' / IMAGE extension - BITPIX = 16 / number of bits per data pixel - NAXIS = 2 / number of data axes - .... - - - -Updating, Adding and Deleting Metadata -====================================== -Header cards can be updated, added to, or deleted from the headers. The PHU -and the extensions headers are again accessed in a mostly identical way -with ``phu`` and ``hdr``, respectively. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - -Add and update a keyword, without and with comment:: - - >>> ad.phu['NEWKEY'] = 50. - >>> ad.phu['NEWKEY'] = (30., 'Updated PHU keyword') - - >>> ad[0].hdr['NEWKEY'] = 50. - >>> ad[0].hdr['NEWKEY'] = (30., 'Updated extension keyword') - -Delete a keyword:: - - >>> del ad.phu['NEWKEY'] - >>> del ad[0].hdr['NEWKEY'] - - -World Co-ordinate System attribute -================================== - -The ``wcs`` of an extension's ``nddata`` attribute (eg. ``ad[0].nddata.wcs``; -see :ref:`pixel-data`) is stored as an instance of ``astropy.wcs.WCS`` (a -standard FITS WCS object) or ``gwcs.WCS`` (a `"Generalized WCS" or gWCS -`_ object). This defines a transformation -between array indices and some other co-ordinate system such as "World" -co-ordinates (see `APE 14 -`_). GWCS allows -multiple, almost arbitrary co-ordinate mappings from different calibration -steps (eg. CCD mosaicking, distortion correction & wavelength calibration) to -be combined in a single, reversible transformation chain --- but this -information cannot always be represented as a FITS standard WCS. If a gWCS -object is too complex to be defined by the basic FITS keywords, it gets stored -as a table extension named 'WCS' when the ``AstroData`` instance is saved to a -file (with the same EXTVER as the corresponding 'SCI' array) and the FITS -header keywords are updated to provide an approximation to the true WCS and an -additional keyword ``FITS-WCS`` is added with the value 'APPROXIMATE'. -The representation in the table is produced using -`ASDF `_, with one line of text per row. Likewise, -when the file is re-opened, the gWCS object gets recreated in ``wcs`` from the -table. If the transformation defined by the gWCS object can be accurately -described by standard FITS keywords, then no WCS extension is created as the -gWCS object can be created from these keywords when the file is re-opened. - -In future, it is intended to improve the quality of the FITS approximation -using the Simple Imaging Polynomial convention -(`SIP `_) or -a discrete sampling of the World co-ordinate -values will be stored as part of the FITS WCS, following `Greisen et al. (2006) -`_, S6 (in addition to the -definitive 'WCS' table), allowing standard FITS readers to report accurate -World co-ordinates for each pixel. - - -Adding Descriptors [Advanced Topic] -=================================== -For proper and complete instructions on how to create Astrodata Descriptors, -the reader is invited to refer to the Astrodata Programmer Manual. Here we -provide a simple introduction that might help some readers better understand -Astrodata Descriptors, or serve as a quick reference for those who have -written Astrodata Descriptors in the past but need a little refresher. - -The Astrodata Descriptors are defined in an ``AstroData`` class. The -``AstroData`` class specific to an instrument is located in a separate -package, not in ``astrodata``. For example, for Gemini instruments, all the -various ``AstroData`` classes are contained in the ``gemini_instruments`` -package. - -An Astrodata Descriptor is a function within the instrument's ``AstroData`` -class. The descriptor function is distinguished from normal functions by -applying the ``@astro_data_descriptor`` decorator to it. The descriptor -function returns the value(s) using a Python type, ``int``, ``float``, -``string``, ``list``; it depends on the value being returned. There is no -special "descriptor" type. - -Here is an example of code defining a descriptor:: - - class AstroDataGmos(AstroDataGemini): - ... - @astro_data_descriptor - def detector_x_bin(self): - def _get_xbin(b): - try: - return int(b.split()[0]) - except (AttributeError, ValueError): - return None - - binning = self.hdr.get('CCDSUM') - if self.is_single: - return _get_xbin(binning) - else: - xbin_list = [_get_xbin(b) for b in binning] - # Check list is single-valued - return xbin_list[0] if xbin_list == xbin_list[::-1] else None - -This descriptor returns the X-axis binning as a integer when called on a -single extension, or an object with only one extension, for example after the -GMOS CCDs have been mosaiced. If there are more than one extensions, it -will return a Python list or an integer if the binning is the same for all -the extensions. - -Gemini has defined a standard list of descriptors that should be defined -one way or another for each instrument to ensure the re-usability of our -algorithms. That list is provided in the Appendix :ref:`descriptors`. - -For more information on adding to Astrodata, see the Astrodata Programmer -Manual. diff --git a/astrodata/doc/usermanual/index.rst b/astrodata/doc/usermanual/index.rst deleted file mode 100644 index 9c5dff7a8e..0000000000 --- a/astrodata/doc/usermanual/index.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. Astrodata User Manual master file, created from team template - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - Manually edited by KL Wed Jan 18 2017 - -=========== -User Manual -=========== - -.. admonition:: Document ID - - PIPE-USER-106_AstrodataUserManual - -.. toctree:: - :maxdepth: 2 - - intro - structure - iomef - tags - headers - data - tables diff --git a/astrodata/doc/usermanual/intro.rst b/astrodata/doc/usermanual/intro.rst deleted file mode 100644 index ebd615e9e8..0000000000 --- a/astrodata/doc/usermanual/intro.rst +++ /dev/null @@ -1,138 +0,0 @@ -.. intro.rst - -.. _intro_usermanual: - -************ -Introduction -************ - -This is the AstroData User's Manual. AstroData is a DRAGONS package. -The current chapter covers basic concepts -like what is the |astrodata| package and how to install it (together with the -other DRAGONS' packages). :ref:`Chapter 2 ` -explains with more details what is |AstroData| and how the data is represented -using it. :ref:`Chapter 3 ` describes input and output operations and -how multi-extension (MEF) FITS files are represented. :ref:`Chapter 4 ` -provides information regarding the |TagSet| class, its usage and a few advanced -topics. In :ref:`Chapter 5 ` you will find information about the FITS -headers and how to access/modify the metadata. The last two chapters, -:ref:`Chapter 6 ` and :ref:`Chapter 7 ` cover more details -about how to read, manipulate and write pixel data and tables, respectively. - - -If you are looking for a quick reference, please, have a look on the -:doc:`../cheatsheet`. - -Reference Documents -=================== - - - |DRAGONS| - - :doc:`../cheatsheet` - - |RSUserManual| - - |RSProgManual| - -What is |astrodata|? -==================== - -|astrodata| is a package that wraps together tools to represent internally -astronomical datasets stored on disks and to properly parse their metadata -using the |AstroData| and the |TagSet| classes. |astrodata| provides uniform -interfaces for working on datasets from different -instruments. Once a dataset has been opened with |astrodata|, the object -"knows about itself". Information like instrument, observation mode, and how -to access headers, is readily available through the uniform interface. All -the details are coded inside the class associated with the instrument, that -class then provides the interface. The appropriate class is selected -automatically when the file is opened and inspected by |astrodata|. - -Currently |astrodata| implements a representation for Multi-Extension FITS -(MEF) files. (Other representations can be implemented.) - - -.. _install: - -Installing Astrodata -==================== - -The |astrodata| package has a few dependencies, |astropy|, |numpy| and others. -The best way to get everything you need is to install Miniconda, and the -|dragons| stack from conda-forge and Gemini's public conda channel. - -|astrodata| itself is part of |DRAGONS|. It is available from the -repository, as a tar file, or as a conda package. The bare |astrodata| package -does not do much by itself, it needs a companion instrument definitions -package. For Gemini, this is ``gemini_instruments``, also included in -|DRAGONS|. - -.. note:: We are in the process of making ``astrodata`` an Astropy affiliated - package. For now, |DRAGONS| uses the ``astrodata`` integrated with - DRAGONS not the affiliated package. - -Installing Miniforge and the DRAGONS stack ------------------------------------------- -This is required whether you are installing |DRAGONS| from the -repository, the tar file or the conda package. - -To avoid duplication, please follow the installation guide provided in the -Recipe System User Manual: - - |RSUserInstall| - - -Smoke test the Astrodata installation -------------------------------------- -From the configured bash shell:: - - $ type python - python is hashed (/anaconda3/envs/dragons/python) - - Make sure that python is indeed pointing to the Anaconda environment you - have just set up. - -:: - - $ python - >>> import astrodata - >>> import gemini_instruments - - Expected result: Just a python prompt and no error messages. - -Source code availability ------------------------- -The source code is available on Github: - - ``_ - -.. _datapkg: - -Try it yourself -=============== - -**Try it yourself** - -Download the data package if you wish to follow along and run the -examples presented in this manual. It is available at: - - ``_ - -Unpack it:: - - $ cd - $ tar xvf ad_usermanual_datapkg-v1.tar - $ bunzip2 ad_usermanual/playdata/*.bz2 - -Then :: - - $ cd ad_usermanual/playground - $ python - - -Astrodata Support -================= - -Astrodata is developed and supported by staff at the Gemini Observatory. -Questions about the reduction of Gemini data should be directed to the -Gemini Helpdesk system at -``_ -The github issue tracker can be used to report software bugs in DRAGONS -(``_). diff --git a/astrodata/doc/usermanual/iomef.rst b/astrodata/doc/usermanual/iomef.rst deleted file mode 100644 index 8bb228da24..0000000000 --- a/astrodata/doc/usermanual/iomef.rst +++ /dev/null @@ -1,546 +0,0 @@ -.. iomef.rst - -.. _iomef: - -************************************************************ -Input and Output Operations and Extension Manipulation - MEF -************************************************************ - -|AstroData| is not intended to be Multi-Extension FITS (MEF) centric. The core -is independent of the file format. At Gemini, our data model uses MEF. -Therefore we have implemented a FITS handler that maps a MEF to the -internal |AstroData| representation. A different handler can be implemented -for a different file format. - -In this chapter, we present examples that will help the reader understand how -to access the information stored in a MEF with the |AstroData| object and -understand that mapping. - -**Try it yourself** - -Download the data package (:ref:`datapkg`) if you wish to follow along and run the -examples. Then :: - - $ cd /ad_usermanual/playground - $ python - - -Imports -======= - -Before doing anything, you need to import |AstroData| and the Gemini instrument -configuration package |gemini_instruments|. - -:: - - >>> import astrodata - >>> import gemini_instruments - - -Open and access existing dataset -================================ - -Read in the dataset -------------------- - -The file on disk is loaded into the |AstroData| class associated with the -instrument the data is from. This association is done automatically based on -header content. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> type(ad) - - -From now on, ``ad`` knows it is GMOS data. It knows how to access its headers -and when using the Recipe System (|recipe_system|), it will trigger the -selection of the GMOS primitives and recipes. - -The original path and filename are stored in the object. If you were to write -the |AstroData| object to disk without specifying anything, those path and -filename would be used. :: - - >>> ad.path - '../playdata/N20170609S0154.fits' - >>> ad.filename - 'N20170609S0154.fits' - - -Accessing the content of a MEF file ------------------------------------ - -Accessing pixel data, headers, and tables will be covered in detail in the -following chapters. Here we just introduce the basic content interface. - -For details on the |AstroData| structure, please refer to the -:ref:`previous chapter `. - -|AstroData| uses |NDData| as the core of its structure. Each FITS extension -becomes a |NDAstroData| object, subclassed from |NDData|, and is added to -a list. - -Pixel data -^^^^^^^^^^ - -To access pixel data, the list index and the ``.data`` attribute are used. That -returns a :class:`numpy.ndarray`. The list of |NDAstroData| is zero-indexed. -*Extension number 1 in a MEF is index 0 in an |AstroData| object*. :: - - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - >>> data = ad[0].data - >>> type(data) - - >>> data.shape - (2112, 256) - -Remember that in a :class:`~numpy.ndarray` the y-axis is the first number. - -The variance and data quality planes, the VAR and DQ planes in Gemini MEF -files, are represented by the ``.variance`` and ``.mask`` attributes, -respectively. They are not their own "extension", they don't have their -own index in the list, unlike in a MEF. They are attached to the pixel data, -packaged together by the |NDAstroData| object. They are represented as -:class:`numpy.ndarray` just like the pixel data :: - - >>> var = ad[0].variance - >>> dq = ad[0].mask - -Tables -^^^^^^ -Tables in the MEF file will also be loaded into the |AstroData| object. If a table -is associated with a specific science extension through the EXTVER header keyword, that -table will be packaged within the same AstroData extension as the pixel data. -The |AstroData| "extension" is the |NDAstroData| object plus any table or other pixel -array. If the table is not associated with a specific extension and applies -globally, it will be added to the AstroData object as a global addition. No -indexing will be required to access it. In the example below, one ``OBJCAT`` is -associated with each extension, while the ``REFCAT`` has a global scope :: - - >>> ad.info() - Filename: ../playdata/N20170609S0154_varAdded.fits - Tags: ACQUISITION GEMINI GMOS IMAGE NORTH OVERSCAN_SUBTRACTED OVERSCAN_TRIMMED - PREPARED SIDEREAL - - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) uint16 - .OBJCAT Table (6, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - [ 1] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) uint16 - .OBJCAT Table (8, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - [ 2] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) uint16 - .OBJCAT Table (7, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - [ 3] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) uint16 - .OBJCAT Table (5, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - - Other Extensions - Type Dimensions - .REFCAT Table (245, 16) - - -The tables are stored internally as :class:`astropy.table.Table` objects. :: - - >>> ad[0].OBJCAT - - NUMBER X_IMAGE Y_IMAGE ... REF_MAG_ERR PROFILE_FWHM PROFILE_EE50 - int32 float32 float32 ... float32 float32 float32 - ------ ------- ------- ... ----------- ------------ ------------ - 1 283.461 55.4393 ... 0.16895 -999.0 -999.0 - ... - >>> type(ad[0].OBJCAT) - - - >>> refcat = ad.REFCAT - >>> type(refcat) - - - -Headers -^^^^^^^ -Headers are stored in the |NDAstroData| ``.meta`` attribute as :class:`astropy.io.fits.Header` objects, -which is a form of Python ordered dictionaries. Headers associated with extensions -are stored with the corresponding |NDAstroData| object. The MEF Primary Header -Unit (PHU) is stored "globally" in the |AstroData| object. Note that when slicing an |AstroData| object, -for example copying over just the first extension, the PHU will follow. The -slice of an |AstroData| object is an |AstroData| object. -Headers can be accessed directly, or for some predefined concepts, the use of -Descriptors is preferred. See the chapters on headers for details. - -Using Descriptors:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> ad.filter_name() - 'open1-6&g_G0301' - >>> ad.filter_name(pretty=True) - 'g' - -Using direct header access:: - - >>> ad.phu['FILTER1'] - 'open1-6' - >>> ad.phu['FILTER2'] - 'g_G0301' - -Accessing the extension headers:: - - >>> ad.hdr['CCDSEC'] - ['[1:512,1:4224]', '[513:1024,1:4224]', '[1025:1536,1:4224]', '[1537:2048,1:4224]'] - >>> ad[0].hdr['CCDSEC'] - '[1:512,1:4224]' - - With descriptors: - >>> ad.array_section(pretty=True) - ['[1:512,1:4224]', '[513:1024,1:4224]', '[1025:1536,1:4224]', '[1537:2048,1:4224]'] - - -Modify Existing MEF Files -========================= -Before you start modify the structure of an |AstroData| object, you should be -familiar with it. Please make sure that you have read the previous chapter -on :ref:`the structure of the AstroData object `. - -Appending an extension ----------------------- -In this section, we take an extension from one |AstroData| object and append it -to another. - -Here is an example appending a whole AstroData extension, with pixel data, -variance, mask and tables. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> advar = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - - >>> ad.info() - Filename: ../playdata/N20170609S0154.fits - Tags: ACQUISITION GEMINI GMOS IMAGE NORTH RAW SIDEREAL UNPREPARED - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2112, 288) uint16 - [ 1] science NDAstroData (2112, 288) uint16 - [ 2] science NDAstroData (2112, 288) uint16 - [ 3] science NDAstroData (2112, 288) uint16 - - >>> ad.append(advar[3]) - >>> ad.info() - Filename: ../playdata/N20170609S0154.fits - Tags: ACQUISITION GEMINI GMOS IMAGE NORTH RAW SIDEREAL UNPREPARED - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2112, 288) uint16 - [ 1] science NDAstroData (2112, 288) uint16 - [ 2] science NDAstroData (2112, 288) uint16 - [ 3] science NDAstroData (2112, 288) uint16 - [ 4] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) int16 - .OBJCAT Table (5, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - - >>> ad[4].hdr['EXTVER'] - 4 - >>> advar[3].hdr['EXTVER'] - 4 - -As you can see above, the fourth extension of ``advar``, along with everything -it contains was appended at the end of the first |AstroData| object. However, -note that, because the EXTVER of the extension in ``advar`` was 4, there are -now two extensions in ``ad`` with this EXTVER. This is not a problem because -EXTVER is not used by |AstroData| (it uses the index instead) and it is handled -only when the file is written to disk. - -In this next example, we are appending only the pixel data, leaving behind the other -associated data. One can attach the headers too, like we do here. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> advar = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - - >>> ad.append(advar[3].data, header=advar[3].hdr) - >>> ad.info() - Filename: ../playdata/N20170609S0154.fits - Tags: ACQUISITION GEMINI GMOS IMAGE NORTH RAW SIDEREAL UNPREPARED - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2112, 288) uint16 - [ 1] science NDAstroData (2112, 288) uint16 - [ 2] science NDAstroData (2112, 288) uint16 - [ 3] science NDAstroData (2112, 288) uint16 - [ 4] science NDAstroData (2112, 256) float32 - -Notice how a new extension was created but ``variance``, ``mask``, the OBJCAT -table and OBJMASK image were not copied over. Only the science pixel data was -copied over. - -Please note, there is no implementation for the "insertion" of an extension. - -Removing an extension or part of one ------------------------------------- -Removing an extension or a part of an extension is straightforward. The -Python command :func:`del` is used on the item to remove. Below are a few -examples, but first let us load a file :: - - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - >>> ad.info() - -As you go through these examples, check the new structure with :func:`ad.info()` -after every removal to see how the structure has changed. - -Deleting a whole |AstroData| extension, the fourth one :: - - >>> del ad[3] - -Deleting only the variance array from the second extension :: - - >>> ad[1].variance = None - -Deleting a table associated with the first extension :: - - >>> del ad[0].OBJCAT - -Deleting a global table, not attached to a specific extension :: - - >>> del ad.REFCAT - - - -Writing back to disk -==================== -The :class:`~astrodata.AstroData` layer takes care of converting -the |AstroData| object back to a MEF file on disk. When writing to disk, -one should be aware of the path and filename information associated -with the |AstroData| object. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> ad.path - '../playdata/N20170609S0154.fits' - >>> ad.filename - 'N20170609S0154.fits' - -Writing to a new file ---------------------- -There are various ways to define the destination for the new FITS file. -The most common and natural way is :: - - >>> ad.write('new154.fits') - - >>> ad.write('new154.fits', overwrite=True) - -This will write a FITS file named 'new154.fits' in the current directory. -With ``overwrite=True``, it will overwrite the file if it already exists. -A path can be prepended to the filename if the current directory is not -the destination. -Note that ``ad.filename`` and ``ad.path`` have not changed, we have just -written to the new file, the |AstroData| object is in no way associated -with that new file. :: - - >>> ad.path - '../playdata/N20170609S0154.fits' - >>> ad.filename - 'N20170609S0154.fits' - -If you want to create that association, the ``ad.filename`` and ``ad.path`` -needs to be modified first. For example:: - - >>> ad.filename = 'new154.fits' - >>> ad.write(overwrite=True) - - >>> ad.path - '../playdata/new154.fits' - >>> ad.filename - 'new154.fits' - -Changing ``ad.filename`` also changes the filename in the ``ad.path``. The -sequence above will write 'new154.fits' not in the current directory but -rather to the directory that is specified in ``ad.path``. - -WARNING: :func:`ad.write` has an argument named ``filename``. Setting ``filename`` -in the call to :func:`ad.write`, as in ``ad.write(filename='new154.fits')`` will NOT -modify ``ad.filename`` or ``ad.path``. The two "filenames", one a method argument -the other a class attribute have no association to each other. - - -Updating an existing file on disk ----------------------------------- -Updating an existing file on disk requires explicitly allowing overwrite. - -If you have not written 'new154.fits' to disk yet (from previous section) :: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> ad.write('new154.fits', overwrite=True) - -Now let's open 'new154.fits', and write to it :: - - >>> adnew = astrodata.open('new154.fits') - >>> adnew.write(overwrite=True) - - -A note on FITS header keywords ------------------------------- - -.. _fitskeys: - -When writing an |AstroData| object to disk as a FITS file, it is necessary to add or -update header keywords to represent some of the internally-stored information. Any -extensions that did not originally belong to this |AstroData| will be assigned new -EXTVER keywords to avoid conflicts with existing extensions, and the internal WCS is -converted to the appropriate FITS keywords. Note that in some cases it may not be -possible for standard FITS keywords to accurately represent the true WCS. In such -cases, the FITS keywords are written as an approximation to the true WCS, together -with an additional keyword :: - - FITS-WCS= 'APPROXIMATE' / FITS WCS is approximate - -to indicate this. The accurate WCS is written as an additional FITS extension with -``EXTNAME='WCS'`` that AstroData will recognize when the file is read back in. The -``WCS`` extension will not be written to disk if there is an accurate FITS -representation of the WCS (e.g., for a simple image). - - -Create New MEF Files -==================== - -A new MEF file can be created from an existing, maybe modified, file or it -can be created from scratch. We discuss both cases here. - -Create New Copy of MEF Files ----------------------------- -To create a new copy of a MEF file, modified or not, the user has already -been given most of the tools in the sections above. Yet, let's throw a -couple examples for completeness. - -Basic example -^^^^^^^^^^^^^ -As seen above, a MEF file can be opened with |astrodata|, the |AstroData| -object can be modified (or not), and then written back to disk under a -new name. :: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - ... optional modifications here ... - >>> ad.write('newcopy.fits') - - -Needing true copies in memory -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Sometimes it is a true copy in memory that is needed. This is not specific -to MEF. In Python, doing something like ``adnew = ad`` does not create a -new copy of the AstrodData object; it just gives it a new name. If you -modify ``adnew`` you will be modifying ``ad`` too. They point to the same -block of memory. - -To create a true independent copy, the ``deepcopy`` utility needs to be used. :: - - >>> from copy import deepcopy - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> adcopy = deepcopy(ad) - -Be careful using ``deepcopy``, your memory could balloon really fast. Use it -only when truly needed. - - -Create New MEF Files from Scratch ---------------------------------- -Before one creates a new MEF file on disk, one has to create the AstroData -object that will be eventually written to disk. The |AstroData| object -created also needs to know that it will have to be written using the MEF -format. This is fortunately handled fairly transparently by |astrodata|. - -The key to associating the FITS data to the |AstroData| object is simply to -create the |AstroData| object from :mod:`astropy.io.fits` header objects. Those -will be recognized by |astrodata| as FITS and the constructor for FITS will be -used. The user does not need to do anything else special. Here is how it is -done. - -Create a MEF with basic header and data array set to zeros -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - >>> import numpy as np - >>> from astropy.io import fits - - >>> phu = fits.PrimaryHDU() - - >>> pixel_data = np.zeros((100,100)) - - >>> hdu = fits.ImageHDU() - >>> hdu.data = pixel_data - - >>> ad = astrodata.create(phu) - >>> ad.append(hdu, name='SCI') - - or another way to do the last two blocs: - >>> hdu = fits.ImageHDU(data=pixel_data, name='SCI') - >>> ad = astrodata.create(phu, [hdu]) - -Then it is just a matter of calling ``ad.write('somename.fits')`` on that -new ``Astrodata`` object. - -Associate a pixel array with a science pixel array -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Only main science ("SCI") pixel arrays are added as slices to an astrodata -object. It not uncommon to have pixels information associated with those -main science pixels, for example an object mask where marked pixels in the mask -are directly associated with sources in the science array. - -Such pixel arrays are added to specific slice of the astrodata object they are -associated with. - -Building on the astrodata object we created in the previous subsection, one -would add a pixel array to the first slice of the astrodata object as -follows: - - >>> extra_data = np.ones((100, 100)) - >>> ad[0].EXTRADATA = extra_data - -When the file is written to disk as a MEF, an extension will be created with -``EXTNAME = EXTRADATA`` and an ``EXTVER`` that matches the slice's ``EXTVER``, -in this case is would be ``1``. - -Represent a table as a FITS binary table in an ``AstroData`` object -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -One first needs to create a table, either an :class:`astropy.table.Table` -or a :class:`~astropy.io.fits.BinTableHDU`. See the |astropy| documentation -on tables and this manual's :ref:`section ` dedicated to tables for -more information. - -In the first example, we assume that ``my_astropy_table`` is -a :class:`~astropy.table.Table` ready to be attached to an |AstroData| -object. (Warning: we have not created ``my_astropy_table`` therefore the -example below will not run, though this is how it would be done.) - -:: - - >>> phu = fits.PrimaryHDU() - >>> ad = astrodata.create(phu) - - >>> astrodata.add_header_to_table(my_astropy_table) - >>> ad.append(my_astropy_table, name='SMAUG') - - -In the second example, we start with a FITS :class:`~astropy.io.fits.BinTableHDU` -and attach it to a new |AstroData| object. (Again, we have not created -``my_fits_table`` so the example will not run.) :: - - >>> phu = fits.PrimaryHDU() - >>> ad = astrodata.create(phu) - >>> ad.append(my_fits_table, name='DROGON') - -As before, once the |AstroData| object is constructed, the ``ad.write()`` -method can be used to write it to disk as a MEF file. diff --git a/astrodata/doc/usermanual/structure.rst b/astrodata/doc/usermanual/structure.rst deleted file mode 100644 index 591edd0369..0000000000 --- a/astrodata/doc/usermanual/structure.rst +++ /dev/null @@ -1,190 +0,0 @@ -.. structure.rst - -.. _structure: - -******************** -The AstroData Object -******************** - -The |AstroData| object is an internal representation of a file on disk. -As of this version, only a FITS layer has been written, but |AstroData| itself -is not limited to FITS. - -The internal structure of the |AstroData| object makes uses of -:class:`astropy.nddata.NDData`, :mod:`astropy.table`, and -:class:`astropy.io.fits.Header`, the latter simply because it is a -convenient ordered dictionary. - -**Try it yourself** - -Download the data package (:ref:`datapkg`) if you wish to follow along and run the -examples. Then :: - - $ cd /ad_usermanual/playground - $ python - - -Global vs Extension-specific -============================ -At the very top level, the structure is divided in two types of information. -In the first category, there is the information that applies to the data -globally, for example the information that would be stored in a FITS Primary -Header Unit, a table from a catalog that matches the RA and DEC of the field, -etc. In the second category, there is the information specific to individual -science pixel extensions, for example the gain of the amplifier, the data -themselves, the error on those data, etc. - -Let us look at an example. The :meth:`~astrodata.AstroData.info` method shows -the content of the |AstroData| object and its organization, from the user's -perspective.:: - - >>> import astrodata - >>> import gemini_instruments - - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - >>> ad.info() - Filename: N20170609S0154_varAdded.fits - Tags: ACQUISITION GEMINI GMOS IMAGE NORTH OVERSCAN_SUBTRACTED OVERSCAN_TRIMMED - PREPARED SIDEREAL - - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) uint16 - .OBJCAT Table (6, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - [ 1] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) uint16 - .OBJCAT Table (8, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - [ 2] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) uint16 - .OBJCAT Table (7, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - [ 3] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) uint16 - .OBJCAT Table (5, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - - Other Extensions - Type Dimensions - .REFCAT Table (245, 16) - - -The "Pixel Extensions" contain the pixel data. Each extension is represented -individually in a list (0-indexed like all Python lists). The science pixel -data, its associated metadata (extension header), and any other pixel or table -extensions directly associated with that science pixel data are stored in -a |NDAstroData| object which is a subclass of astropy |NDData|. We will -return to this structure later. An |AstroData| extension is accessed like -any list: ``ad[0]``. To access the science pixels, one uses ``ad[0].data``; for -the object mask of the first extension, ``ad[0].OBJMASK``. - -In the example above, the "Other Extensions" at the bottom of the -:meth:`~astrodata.AstroData.info` display contains a ``REFCAT`` table which in -this case is a list of stars from a catalog that overlaps the field of view -covered by the pixel data. The "Other Extensions" are global extensions. They -are not attached to any pixel extension in particular. To access a global -extension one simply uses the name of that extension: ``ad.REFCAT``. - - -Organization of the Global Information -====================================== -All the global information is stored in attributes of the |AstroData| object. -The global headers, or Primary Header Unit (PHU), is stored in the ``phu`` -attribute as an :class:`astropy.io.fits.Header`. - -Any global tables, like ``REFCAT`` above, are stored in the private attribute -``_tables`` as a Python dictionary with the name (eg. "REFCAT") as the key. -All tables are stored as :class:`astropy.table.Table`. Access to those table -is done using the key directly as if it were a normal attribute, eg. -``ad.REFCAT``. Header information for the table, if read in from a FITS table, -is stored in the ``meta`` attribute of the :class:`astropy.table.Table`, eg. -``ad.REFCAT.meta['header']``. It is for information only, it is not used. - - -Organization of the Extension-specific Information -================================================== -The pixel data are stored in the |AstroData| attribute ``nddata`` as a list -of |NDAstroData| object. The |NDAstroData| object is a subclass of astropy -|NDData| and it is fully compatible with any function expecting an |NDData| as -input. The pixel extensions are accessible through slicing, eg. ``ad[0]`` or -even ``ad[0:2]``. A slice of an AstroData object is an AstroData object, and -all the global attributes are kept. For example:: - - >>> ad[0].info() - Filename: N20170609S0154_varAdded.fits - Tags: ACQUISITION GEMINI GMOS IMAGE NORTH OVERSCAN_SUBTRACTED OVERSCAN_TRIMMED - PREPARED SIDEREAL - - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2112, 256) float32 - .variance ndarray (2112, 256) float32 - .mask ndarray (2112, 256) uint16 - .OBJCAT Table (6, 43) n/a - .OBJMASK ndarray (2112, 256) uint8 - - Other Extensions - Type Dimensions - .REFCAT Table (245, 16) - -Note how ``REFCAT`` is still present. - -The science data is accessed as ``ad[0].data``, the variance as ``ad[0].variance``, -and the data quality plane as ``ad[0].mask``. Those familiar with astropy -|NDData| will recognize the structure "data, error, mask", and will notice -some differences. First |AstroData| uses the variance for the error plane, not -the standard deviation. Another difference will be evident only when one looks -at the content of the mask. |NDData| masks contain booleans, |AstroData| masks -are ``uint16`` bit mask that contains information about the type of bad pixels -rather than just flagging them a bad or not. Since ``0`` is equivalent to -``False`` (good pixel), the |AstroData| mask is fully compatible with the -|NDData| mask. - -Header information for the extension is stored in the |NDAstroData| ``meta`` -attribute. All table and pixel extensions directly associated with the -science extension are also stored in the ``meta`` attribute. - -Technically, an extension header is located in ``ad.nddata[0].meta['header']``. -However, for obviously needed convenience, the normal way to access that header -is ``ad[0].hdr``. - -Tables and pixel arrays associated with a science extension are -stored in ``ad.nddata[0].meta['other']`` as a dictionary keyed on the array -name, eg. ``OBJCAT``, ``OBJMASK``. As it is for global tables, astropy tables -are used for extension tables. The extension tables and extra pixel arrays are -accessed, like the global tables, by using the table name rather than the long -format, for example ``ad[0].OBJCAT`` and ``ad[0].OBJMASK``. - -When reading a FITS Table, the header information is stored in the -``meta['header']`` of the table, eg. ``ad[0].OBJCAT.meta['header']``. That -information is not used, it is simply a place to store what was read from disk. - -The header of a pixel extension directly associated with the science extension -should match that of the science extension. Therefore such headers are not -stored in |AstroData|. For example, the header of ``ad[0].OBJMASK`` is the -same as that of the science, ``ad[0].hdr``. - -The world coordinate system (WCS) is stored internally in the ``wcs`` attribute -of the |NDAstroData| object. It is constructed from the header keywords when -the FITS file is read from disk, or directly from the ``WCS`` extension if -present (see :ref:`the next chapter `). If the WCS is modified (for -example, by refining the pointing or attaching a more accurate wavelength -calibration), the FITS header keywords are not updated and therefore they should -never be used to determine the world coordinates of any pixel. These keywords are -only updated when the object is written to disk as a FITS file. The WCS is -retrieved as follows: ``ad[0].wcs``. - - -A Note on Memory Usage -====================== -When an file is opened, the headers are loaded into memory, but the pixels -are not. The pixel data are loaded into memory only when they are first -needed. This is not real "memory mapping", more of a delayed loading. This -is useful when someone is only interested in the metadata, especially when -the files are very large. diff --git a/astrodata/doc/usermanual/tables.rst b/astrodata/doc/usermanual/tables.rst deleted file mode 100644 index 20316d3874..0000000000 --- a/astrodata/doc/usermanual/tables.rst +++ /dev/null @@ -1,228 +0,0 @@ -.. tables.rst - -.. _tables: - -********** -Table Data -********** -**Try it yourself** - -Download the data package (:ref:`datapkg`) if you wish to follow along and run the -examples. Then :: - - $ cd /ad_usermanual/playground - $ python - -Then import core astrodata and the Gemini astrodata configurations. :: - - >>> import astrodata - >>> import gemini_instruments - -Tables and Astrodata -==================== -Tables are stored as ``astropy.table`` ``Table`` class. FITS tables too -are represented in Astrodata as ``Table`` and FITS headers are stored in -the NDAstroData `.meta` attribute. Most table access should be done -through the ``Table`` interface. The best reference on ``Table`` is the -Astropy documentation itself. In this chapter we covers some common -examples to get the reader started. - -The ``astropy.table`` documentation can be found at: ``_ - - -Operate on a Table -================== - -Let us open a file with tables. Some tables are associated with specific -extensions, and there is one table that is global to the `AstroData` object. - -:: - - >>> ad = astrodata.open('../playdata/N20170609S0154_varAdded.fits') - >>> ad.info() - -To access the global table named ``REFCAT``:: - - >>> ad.REFCAT - -To access the ``OBJCAT`` table in the first extension :: - - >>> ad[0].OBJCAT - - -Column and Row Operations -------------------------- -Columns are named. Those names are used to access the data as columns. -Rows are not names and are simply represented as a sequential list. - -Read columns and rows -+++++++++++++++++++++ -To get the names of the columns present in the table:: - - >>> ad.REFCAT.colnames - ['Id', 'Cat_Id', 'RAJ2000', 'DEJ2000', 'umag', 'umag_err', 'gmag', - 'gmag_err', 'rmag', 'rmag_err', 'imag', 'imag_err', 'zmag', 'zmag_err', - 'filtermag', 'filtermag_err'] - -Then it is easy to request the values for specific columns:: - - >>> ad.REFCAT['zmag'] - >>> ad.REFCAT['zmag', 'zmag_err'] - -To get the content of a specific row, row 10 in this case:: - - >>> ad.REFCAT[9] - -To get the content of a specific row(s) from a specific column(s):: - - >>> ad.REFCAT['zmag'][4] - >>> ad.REFCAT['zmag'][4:10] - >>> ad.REFCAT['zmag', 'zmag_err'][4:10] - -Change values -+++++++++++++ -Assigning new values works in a similar way. When working on multiple elements -it is important to feed a list that matches in size with the number of elements -to replace. - -:: - - >>> ad.REFCAT['imag'][4] = 20.999 - >>> ad.REFCAT['imag'][4:10] = [5, 6, 7, 8, 9, 10] - - >>> overwrite_col = [0] * len(ad.REFCAT) # a list of zeros, size = nb of rows - >>> ad.REFCAT['imag_err'] = overwrite_col - -Add a row -+++++++++ -To append a row, there is the ``add_row()`` method. The length of the row -should match the number of columns:: - - >>> new_row = [0] * len(ad.REFCAT.colnames) - >>> new_row[1] = '' # Cat_Id column is of "str" type. - >>> ad.REFCAT.add_row(new_row) - -Add a column -++++++++++++ -Adding a new column can be more involved. If you need full control, please -see the AstroPy Table documentation. For a quick addition, which might be -sufficient for your use case, we simply use the "dictionary" technique. Please -note that when adding a column, it is important to ensure that all the -elements are of the same type. Also, if you are planning to use that table -in IRAF/PyRAF, we recommend not using 64-bit types. - -:: - - >>> import numpy as np - - >>> new_column = [0] * len(ad.REFCAT) - >>> # Ensure that the type is int32, otherwise it will default to int64 - >>> # which generally not necessary. Also, IRAF 32-bit does not like it. - >>> new_column = np.array(new_column).astype(np.int32) - >>> ad.REFCAT['my_column'] = new_column - -If you are going to write that table back to disk as a FITS Bintable, then -some additional headers need to be set. Astrodata will take care of that -under the hood when the `write` method is invoked. - -:: - - >>> ad.write('myfile_with_modified_table.fits') - - -Selection and Rejection Operations ----------------------------------- -Normally, one does not know exactly where the information needed is located -in a table. Rather some sort of selection needs to be done. This can also -be combined with various calculations. We show two such examples here. - -Select a table element from criterion -+++++++++++++++++++++++++++++++++++++ - -:: - - >>> # Get the magnitude of a star selected by ID number - >>> ad.REFCAT['zmag'][ad.REFCAT['Cat_Id'] == '1237662500002005475'] - - >>> # Get the ID and magnitude of all the stars brighter than zmag 18. - >>> ad.REFCAT['Cat_Id', 'zmag'][ad.REFCAT['zmag'] < 18.] - - -Rejection and selection before statistics -+++++++++++++++++++++++++++++++++++++++++ - -:: - - >>> t = ad.REFCAT # to save typing - - >>> # The table has "NaN" values. ("Not a number") We need to ignore them. - >>> t['zmag'].mean() - nan - >>> # applying rejection of NaN values: - >>> t['zmag'][np.where(~np.isnan(t['zmag']))].mean() - 20.377306 - - - -Accessing FITS table headers directly -------------------------------------- -If for some reason you need to access the FITS table headers directly, here -is how to do it. It is very unlikely that you will need this. - -To see the FITS headers:: - - >>> ad.REFCAT.meta['header'] - >>> ad[0].OBJCAT.meta['header'] - -To retrieve a specific FITS table header:: - - >>> ad.REFCAT.meta['header']['TTYPE3'] - 'RAJ2000' - >>> ad[0].OBJCAT.meta['header']['TTYPE3'] - 'Y_IMAGE' - -To retrieve all the keyword names matching a selection:: - - >>> keynames = [key for key in ad.REFCAT.meta['header'] if key.startswith('TTYPE')] - - - -Create a Table -============== - -To create a table that can be added to an ``AstroData`` object and eventually -written to disk as a FITS file, the first step is to create an Astropy -``Table``. - -Let us first add our data to NumPy arrays, one array per column:: - - >>> import numpy as np - - >>> snr_id = np.array(['S001', 'S002', 'S003']) - >>> feii = np.array([780., 78., 179.]) - >>> pabeta = np.array([740., 307., 220.]) - >>> ratio = pabeta / feii - -Then build the table from that data:: - - >>> from astropy.table import Table - - >>> my_astropy_table = Table([snr_id, feii, pabeta, ratio], - ... names=('SNR_ID', 'FeII', 'PaBeta', 'ratio')) - - -Now we append this Astropy ``Table`` to a new ``AstroData`` object. - -:: - - >>> # Since we are going to write a FITS, we build the AstroData object - >>> # from FITS objects. - >>> from astropy.io import fits - - >>> phu = fits.PrimaryHDU() - >>> ad = astrodata.create(phu) - >>> ad.MYTABLE = my_astropy_table - >>> ad.info() - >>> ad.MYTABLE - - >>> ad.write('new_table.fits') diff --git a/astrodata/doc/usermanual/tags.rst b/astrodata/doc/usermanual/tags.rst deleted file mode 100644 index 6caaf387bd..0000000000 --- a/astrodata/doc/usermanual/tags.rst +++ /dev/null @@ -1,172 +0,0 @@ -.. tags.rst - -.. _tags: - -************** -Astrodata Tags -************** - -What are the Astrodata Tags? -============================ -The Astrodata Tags identify the data represented in the |AstroData| object. -When a file on disk is opened with |astrodata|, the headers are inspected to -identify which specific |AstroData| class needs to be loaded, -:class:`~gemini_instruments.gmos.AstroDataGmos`, -:class:`~gemini_instruments.niri.AstroDataNiri`, etc. Based on the class the data is -associated with, a list of "tags" will be defined. The tags will tell whether the -file is a flatfield or a dark, if it is a raw dataset, or if it has been processed by the -recipe system, if it is imaging or spectroscopy. The tags will tell the -users and the system what that data is and also give some information about -the processing status. - -As a side note, the tags are used by DRAGONS Recipe System to match recipes -and primitives to the data. - -Using the Astrodata Tags -======================== -**Try it yourself** - -Download the data package (:ref:`datapkg`) if you wish to follow along and run the -examples. Then :: - - $ cd /ad_usermanual/playground - $ python - -Before doing anything, you need to import |astrodata| and the Gemini instrument -configuration package (|gemini_instruments|). - -:: - - >>> import astrodata - >>> import gemini_instruments - -Let us open a Gemini dataset and see what tags we get:: - - >>> ad = astrodata.open('../playdata/N20170609S0154.fits') - >>> ad.tags - {'RAW', 'GMOS', 'GEMINI', 'NORTH', 'SIDEREAL', 'UNPREPARED', 'IMAGE', 'ACQUISITION'} - -The file we loaded is raw, GMOS North data. It is a 2D image and it is an -acquisition image, not a science observation. The "UNPREPARED" tag indicates -that the file has never been touched by the Recipe System which runs a -"prepare" primitive as the first step of each recipe. - -Let's try another :: - - >>> ad = astrodata.open('../playdata/N20170521S0925_forStack.fits') - >>> ad.tags - {'GMOS', 'GEMINI', 'NORTH', 'SIDEREAL', 'OVERSCAN_TRIMMED', 'IMAGE', - 'OVERSCAN_SUBTRACTED', 'PREPARED'} - -This file is a science GMOS North image. It has been processed by the -Recipe System. The overscan level has been subtracted and the overscan section -has been trimmed away. The tags do NOT include all the processing steps. Rather, -at least from the time being, it focuses on steps that matter when associating -calibrations. - -The tags can be used when coding. For example:: - - >>> if 'GMOS' in ad.tags: - ... print('I am GMOS') - ... else: - ... print('I am these instead:', ad.tags) - ... - -And:: - - >>> if {'IMAGE', 'GMOS'}.issubset(ad.tags): - ... print('I am a GMOS Image.') - ... - -Using typewalk -============== -In DRAGONS, there is a convenience tool that will list the Astrodata tags -for all the FITS file in a directory. - -To try it, from the shell, not Python, go to the "playdata" directory and -run typewalk:: - - % cd /ad_usermanual/playdata - % typewalk - - directory: /data/workspace/ad_usermanual/playdata - N20170521S0925_forStack.fits ...... (GEMINI) (GMOS) (IMAGE) (NORTH) (OVERSCAN_SUBTRACTED) (OVERSCAN_TRIMMED) (PREPARED) (SIDEREAL) - N20170521S0926_forStack.fits ...... (GEMINI) (GMOS) (IMAGE) (NORTH) (OVERSCAN_SUBTRACTED) (OVERSCAN_TRIMMED) (PREPARED) (PROCESSED) (PROCESSED_SCIENCE) (SIDEREAL) - N20170609S0154.fits ............... (ACQUISITION) (GEMINI) (GMOS) (IMAGE) (NORTH) (RAW) (SIDEREAL) (UNPREPARED) - N20170609S0154_varAdded.fits ...... (ACQUISITION) (GEMINI) (GMOS) (IMAGE) (NORTH) (OVERSCAN_SUBTRACTED) (OVERSCAN_TRIMMED) (PREPARED) (SIDEREAL) - estgsS20080220S0078.fits .......... (GEMINI) (GMOS) (LONGSLIT) (LS) (PREPARED) (PROCESSED) (PROCESSED_SCIENCE) (SIDEREAL) (SOUTH) (SPECT) - gmosifu_cube.fits ................. (GEMINI) (GMOS) (IFU) (NORTH) (ONESLIT_RED) (PREPARED) (PROCESSED) (PROCESSED_SCIENCE) (SIDEREAL) (SPECT) - new154.fits ....................... (ACQUISITION) (GEMINI) (GMOS) (IMAGE) (NORTH) (RAW) (SIDEREAL) (UNPREPARED) - Done DataSpider.typewalk(..) - -``typewalk`` can be used to select specific data based on tags, and even create -lists:: - - % typewalk --tags RAW - directory: /data/workspace/ad_usermanual/playdata - N20170609S0154.fits ............... (ACQUISITION) (GEMINI) (GMOS) (IMAGE) (NORTH) (RAW) (SIDEREAL) (UNPREPARED) - new154.fits ....................... (ACQUISITION) (GEMINI) (GMOS) (IMAGE) (NORTH) (RAW) (SIDEREAL) (UNPREPARED) - Done DataSpider.typewalk(..) - -:: - - % typewalk --tags RAW -o rawfiles.lis - % cat rawfiles.lis - # Auto-generated by typewalk, vv2.0 (beta) - # Written: Tue Mar 6 13:06:06 2018 - # Qualifying types: RAW - # Qualifying logic: AND - # ----------------------- - /Users/klabrie/data/tutorials/ad_usermanual/playdata/N20170609S0154.fits - /Users/klabrie/data/tutorials/ad_usermanual/playdata/new154.fits - - - -Creating New Astrodata Tags [Advanced Topic] -============================================ -For proper and complete instructions on how to create Astrodata Tags and -the |AstroData| class that hosts the tags, the reader is invited to refer to the -Astrodata Programmer Manual. Here we provide a simple introduction that -might help some readers better understand Astrodata Tags, or serve as a -quick reference for those who have written Astrodata Tags in the past but need -a little refresher. - -The Astrodata Tags are defined in an |AstroData| class. The |AstroData| -class specific to an instrument is located in a separate package, not in -|astrodata|. For example, for Gemini instruments, all the various |AstroData| -classes are contained in the |gemini_instruments| package. - -An Astrodata Tag is a function within the instrument's |AstroData| class. -The tag function is distinguished from normal functions by applying the -:func:`~astrodata.astro_data_tag` decorator to it. -The tag function returns a :class:`astrodata.TagSet`. - -For example:: - - class AstroDataGmos(AstroDataGemini): - ... - @astro_data_tag - def _tag_arc(self): - if self.phu.get('OBSTYPE) == 'ARC': - return TagSet(['ARC', 'CAL']) - -The tag function looks at the headers and if the keyword "OBSTYPE" is set -to "ARC", the tags "ARC" and "CAL" (for calibration) will be assigned to the -|AstroData| object. - -A whole suite of such tag functions is needed to fully characterize all -types of data an instrument can produce. - -Tags are about what the dataset is, not it's flavor. The Astrodata -"descriptors" (see the section on :ref:`headers`) will describe the flavor. -For example, tags will say that the data is an image, but the descriptor -will say whether it is B-band or R-band. Tags are used for recipe and -primitive selection. A way to understand the difference between a tag and -a descriptor is in terms of the recipe that will be selected: A GMOS image -will use the same recipe whether it's a B-band or R-band image. However, -a GMOS longslit spectrum will need a very different recipe. A bias is -reduced differently from a science image, there should be a tag differentiating -a bias from a science image. (There is for GMOS.) - -For more information on adding to Astrodata, see the Astrodata Programmer -Manual. diff --git a/astrodata/factory.py b/astrodata/factory.py deleted file mode 100644 index 8f1b5a62c9..0000000000 --- a/astrodata/factory.py +++ /dev/null @@ -1,144 +0,0 @@ -import logging -import os -from contextlib import contextmanager -from copy import deepcopy - -from astropy.io import fits - -LOGGER = logging.getLogger(__name__) - - -class AstroDataError(Exception): - pass - - -class AstroDataFactory: - - _file_openers = ( - fits.open, - ) - - def __init__(self): - self._registry = set() - - @staticmethod - @contextmanager - def _openFile(source): - """ - Internal static method that takes a ``source``, assuming that it is a - string pointing to a file to be opened. - - If this is the case, it will try to open the file and return an - instance of the appropriate native class to be able to manipulate it - (eg. ``HDUList``). - - If ``source`` is not a string, it will be returned verbatim, assuming - that it represents an already opened file. - - """ - if isinstance(source, (str, os.PathLike)): - stats = os.stat(source) - if stats.st_size == 0: - LOGGER.warning(f"File {source} is zero size") - - # try vs all handlers - for func in AstroDataFactory._file_openers: - try: - fp = func(source) - yield fp - except Exception: - # Just ignore the error. Assume that it is a not supported - # format and go for the next opener - pass - else: - if hasattr(fp, 'close'): - fp.close() - return - raise AstroDataError("No access, or not supported format for: {}" - .format(source)) - else: - yield source - - def addClass(self, cls): - """ - Add a new class to the AstroDataFactory registry. It will be used when - instantiating an AstroData class for a FITS file. - """ - if not hasattr(cls, '_matches_data'): - raise AttributeError("Class '{}' has no '_matches_data' method" - .format(cls.__name__)) - self._registry.add(cls) - - def getAstroData(self, source): - """ - Takes either a string (with the path to a file) or an HDUList as input, - and tries to return an AstroData instance. - - It will raise exceptions if the file is not found, or if there is no - match for the HDUList, among the registered AstroData classes. - - Returns an instantiated object, or raises AstroDataError if it was - not possible to find a match - - Parameters - ---------- - source : `str` or `pathlib.Path` or `fits.HDUList` - The file path or HDUList to read. - - """ - candidates = [] - with self._openFile(source) as opened: - for adclass in self._registry: - try: - if adclass._matches_data(opened): - candidates.append(adclass) - except Exception: # Some problem opening this - pass - - # For every candidate in the list, remove the ones that are base - # classes for other candidates. That way we keep only the more - # specific ones. - final_candidates = [] - for cnd in candidates: - if any(cnd in x.mro() for x in candidates if x != cnd): - continue - final_candidates.append(cnd) - - if len(final_candidates) > 1: - raise AstroDataError("More than one class is candidate for this dataset") - elif not final_candidates: - raise AstroDataError("No class matches this dataset") - - return final_candidates[0].read(source) - - def createFromScratch(self, phu, extensions=None): - """Creates an AstroData object from a collection of objects. - - Parameters - ---------- - phu : `fits.PrimaryHDU` or `fits.Header` or `dict` or `list` - FITS primary HDU or header, or something that can be used to create - a fits.Header (a dict, a list of "cards"). - extensions : list of HDUs - List of HDU objects. - - """ - lst = fits.HDUList() - if phu is not None: - if isinstance(phu, fits.PrimaryHDU): - lst.append(deepcopy(phu)) - elif isinstance(phu, fits.Header): - lst.append(fits.PrimaryHDU(header=deepcopy(phu))) - elif isinstance(phu, (dict, list, tuple)): - p = fits.PrimaryHDU() - p.header.update(phu) - lst.append(p) - else: - raise ValueError("phu must be a PrimaryHDU or a valid header object") - - # TODO: Verify the contents of extensions... - if extensions is not None: - for ext in extensions: - lst.append(ext) - - return self.getAstroData(lst) diff --git a/astrodata/fits.py b/astrodata/fits.py deleted file mode 100644 index 281ec0d6ca..0000000000 --- a/astrodata/fits.py +++ /dev/null @@ -1,860 +0,0 @@ -import gc -import logging -import os -import traceback -import warnings -from collections import OrderedDict -from copy import deepcopy -from io import BytesIO -from itertools import product as cart_product, zip_longest - -import asdf -import astropy -import jsonschema -import numpy as np -from astropy import units as u -from astropy.io import fits -from astropy.io.fits import (DELAYED, BinTableHDU, Column, HDUList, - ImageHDU, PrimaryHDU, TableHDU) -from astropy.nddata import NDData -# NDDataRef is still not in the stable astropy, but this should be the one -# we use in the future... -# from astropy.nddata import NDData, NDDataRef as NDDataObject -from astropy.table import Table -from gwcs.wcs import WCS as gWCS - -from .nddata import ADVarianceUncertainty, NDAstroData as NDDataObject -from .wcs import fitswcs_to_gwcs, gwcs_to_fits - -DEFAULT_EXTENSION = 'SCI' -NO_DEFAULT = object() -LOGGER = logging.getLogger(__name__) - - -class FitsHeaderCollection: - """Group access to a list of FITS Header-like objects. - - It exposes a number of methods (``set``, ``get``, etc.) that operate over - all the headers at the same time. It can also be iterated. - - Parameters - ---------- - headers : list of `astropy.io.fits.Header` - List of Header objects. - - """ - def __init__(self, headers): - self._headers = list(headers) - - def _insert(self, idx, header): - self._headers.insert(idx, header) - - def __iter__(self): - yield from self._headers - - def __setitem__(self, key, value): - if isinstance(value, tuple): - self.set(key, value=value[0], comment=value[1]) - else: - self.set(key, value=value) - - def set(self, key, value=None, comment=None): - for header in self._headers: - header.set(key, value=value, comment=comment) - - def __getitem__(self, key): - missing_at = [] - ret = [] - for n, header in enumerate(self._headers): - try: - ret.append(header[key]) - except KeyError: - missing_at.append(n) - ret.append(None) - if missing_at: - error = KeyError("The keyword couldn't be found at headers: {}" - .format(tuple(missing_at))) - error.missing_at = missing_at - error.values = ret - raise error - return ret - - def get(self, key, default=None): - try: - return self[key] - except KeyError as err: - vals = err.values - for n in err.missing_at: - vals[n] = default - return vals - - def __delitem__(self, key): - self.remove(key) - - def remove(self, key): - deleted = 0 - for header in self._headers: - try: - del header[key] - deleted = deleted + 1 - except KeyError: - pass - if not deleted: - raise KeyError(f"'{key}' is not on any of the extensions") - - def get_comment(self, key): - return [header.comments[key] for header in self._headers] - - def set_comment(self, key, comment): - def _inner_set_comment(header): - if key not in header: - raise KeyError(f"Keyword {key!r} not available") - - header.set(key, comment=comment) - - for n, header in enumerate(self._headers): - try: - _inner_set_comment(header) - except KeyError as err: - raise KeyError(err.args[0] + f" at header {n}") - - def __contains__(self, key): - return any(tuple(key in h for h in self._headers)) - - -def new_imagehdu(data, header, name=None): - # Assigning data in a delayed way, won't reset BZERO/BSCALE in the header, - # for some reason. Need to investigated. Maybe astropy.io.fits bug. Figure - # out WHY were we delaying in the first place. - # i = ImageHDU(data=DELAYED, header=header.copy(), name=name) - # i.data = data - return ImageHDU(data=data, header=header.copy(), name=name) - - -def table_to_bintablehdu(table, extname=None): - """ - Convert an astropy Table object to a BinTableHDU before writing to disk. - - Parameters - ---------- - table: astropy.table.Table instance - the table to be converted to a BinTableHDU - extname: str - name to go in the EXTNAME field of the FITS header - - Returns - ------- - BinTableHDU - - """ - # remove header to avoid warning from table_to_hdu - table_header = table.meta.pop('header', None) - - # table_to_hdu sets units only if the unit conforms to the FITS standard, - # otherwise it issues a warning, which we catch here. - with warnings.catch_warnings(): - warnings.simplefilter('ignore', UserWarning) - hdu = fits.table_to_hdu(table) - - # And now we try to set the units that do not conform to the standard, - # using unit.to_string() without the format='fits' argument. - for col in table.itercols(): - if col.unit and not hdu.columns[col.name].unit: - hdu.columns[col.name].unit = col.unit.to_string() - - if table_header is not None: - # Update with cards from table.meta, but skip structural FITS - # keywords since those have been set by table_to_hdu - exclude = ('SIMPLE', 'XTENSION', 'BITPIX', 'NAXIS', 'EXTEND', 'PCOUNT', - 'GCOUNT', 'TFIELDS', 'TFORM', 'TSCAL', 'TZERO', 'TNULL', - 'TTYPE', 'TUNIT', 'TDISP', 'TDIM', 'THEAP', 'TBCOL') - hdr = fits.Header([card for card in table_header.cards - if not card.keyword.startswith(exclude)]) - update_header(hdu.header, hdr) - # reset table's header - table.meta['header'] = table_header - if extname: - hdu.header['EXTNAME'] = (extname, 'added by AstroData') - return hdu - - -def header_for_table(table): - table_header = table.meta.pop('header', None) - fits_header = fits.table_to_hdu(table).header - if table_header: - table.meta['header'] = table_header # restore original meta - fits_header = update_header(table_header, fits_header) - return fits_header - - -def add_header_to_table(table): - header = header_for_table(table) - table.meta['header'] = header - return header - - -def _process_table(table, name=None, header=None): - if isinstance(table, (BinTableHDU, TableHDU)): - obj = Table(table.data, meta={'header': header or table.header}) - for i, col in enumerate(obj.columns, start=1): - try: - obj[col].unit = u.Unit(obj.meta['header'][f'TUNIT{i}']) - except (KeyError, TypeError, ValueError): - pass - elif isinstance(table, Table): - obj = Table(table) - if header is not None: - obj.meta['header'] = deepcopy(header) - elif 'header' not in obj.meta: - obj.meta['header'] = header_for_table(obj) - else: - raise ValueError(f"{table.__class__} is not a recognized table type") - - if name is not None: - obj.meta['header']['EXTNAME'] = name - - return obj - - -def card_filter(cards, include=None, exclude=None): - for card in cards: - if include is not None and card[0] not in include: - continue - elif exclude is not None and card[0] in exclude: - continue - yield card - - -def update_header(headera, headerb): - cardsa = tuple(tuple(cr) for cr in headera.cards) - cardsb = tuple(tuple(cr) for cr in headerb.cards) - - if cardsa == cardsb: - return headera - - # Ok, headerb differs somehow. Let's try to bring the changes to headera - # Updated keywords that should be unique - difference = set(cardsb) - set(cardsa) - headera.update(card_filter(difference, exclude={'HISTORY', 'COMMENT', ''})) - # Check the HISTORY and COMMENT cards, just in case - for key in ('HISTORY', 'COMMENT'): - fltcardsa = card_filter(cardsa, include={key}) - fltcardsb = card_filter(cardsb, include={key}) - # assume we start with two headers that are mostly the same and - # that will have added comments/history at the end (in headerb) - for (ca, cb) in zip_longest(fltcardsa, fltcardsb): - if ca is None: - headera.update((cb,)) - - return headera - - -def fits_ext_comp_key(ext): - """Returns a pair (int, str) that will be used to sort extensions.""" - if isinstance(ext, PrimaryHDU): - # This will guarantee that the primary HDU goes first - ret = (-1, "") - else: - # When two extensions share version number, we'll use their names - # to sort them out. Choose a suitable key so that: - # - # - SCI extensions come first - # - unnamed extensions come last - # - # We'll resort to add 'z' in front of the usual name to force - # SCI to be the "smallest" - name = ext.name - if name == '': - name = "zzzz" - elif name != DEFAULT_EXTENSION: - name = "z" + name - - ver = ext.header.get('EXTVER') - if ver in (-1, None): - # In practice, this number should be larger than any EXTVER found - # in real life HDUs, pushing unnumbered HDUs to the end. - ver = 2**32-1 - - # For the general case, just return version and name, to let them - # be sorted naturally - ret = (ver, name) - - return ret - - -class FitsLazyLoadable: - - def __init__(self, obj): - self._obj = obj - self.lazy = True - - def _create_result(self, shape): - return np.empty(shape, dtype=self.dtype) - - def _scale(self, data): - bscale = self._obj._orig_bscale - bzero = self._obj._orig_bzero - if bscale == 1 and bzero == 0: - return data - return (bscale * data + bzero).astype(self.dtype) - - def __getitem__(self, sl): - # TODO: We may want (read: should) create an empty result array before scaling - return self._scale(self._obj.section[sl]) - - @property - def header(self): - return self._obj.header - - @property - def data(self): - res = self._create_result(self.shape) - res[:] = self._scale(self._obj.data) - return res - - @property - def shape(self): - return self._obj.shape - - @property - def dtype(self): - """ - Need to to some overriding of astropy.io.fits since it doesn't - know about BITPIX=8 - """ - bitpix = self._obj._orig_bitpix - if self._obj._orig_bscale == 1 and self._obj._orig_bzero == 0: - dtype = fits.BITPIX2DTYPE[bitpix] - else: - # this method from astropy will return the dtype if the data - # needs to be converted to unsigned int or scaled to float - dtype = self._obj._dtype_for_bitpix() - - if dtype is None: - if bitpix < 0: - dtype = np.dtype('float{}'.format(abs(bitpix))) - if (self._obj.header['EXTNAME'] == 'DQ' or self._obj._uint and - self._obj._orig_bscale == 1 and bitpix == 8): - dtype = np.uint16 - return dtype - - -def _prepare_hdulist(hdulist, default_extension='SCI', extname_parser=None): - new_list = [] - highest_ver = 0 - recognized = set() - - if len(hdulist) > 1 or (len(hdulist) == 1 and hdulist[0].data is None): - # MEF file - # First get HDUs for which EXTVER is defined - for n, hdu in enumerate(hdulist): - if extname_parser: - extname_parser(hdu) - ver = hdu.header.get('EXTVER') - if ver not in (-1, None) and hdu.name: - highest_ver = max(highest_ver, ver) - elif not isinstance(hdu, PrimaryHDU): - continue - - new_list.append(hdu) - recognized.add(hdu) - - # Then HDUs that miss EXTVER - for hdu in hdulist: - if hdu in recognized: - continue - elif isinstance(hdu, ImageHDU): - highest_ver += 1 - if 'EXTNAME' not in hdu.header: - hdu.header['EXTNAME'] = (default_extension, - 'Added by AstroData') - if hdu.header.get('EXTVER') in (-1, None): - hdu.header['EXTVER'] = (highest_ver, 'Added by AstroData') - - new_list.append(hdu) - recognized.add(hdu) - else: - # Uh-oh, a single image FITS file - new_list.append(PrimaryHDU(header=hdulist[0].header)) - image = ImageHDU(header=hdulist[0].header, data=hdulist[0].data) - # Fudge due to apparent issues with assigning ImageHDU from data - image._orig_bscale = hdulist[0]._orig_bscale - image._orig_bzero = hdulist[0]._orig_bzero - - for keyw in ('SIMPLE', 'EXTEND'): - if keyw in image.header: - del image.header[keyw] - image.header['EXTNAME'] = (default_extension, 'Added by AstroData') - image.header['EXTVER'] = (1, 'Added by AstroData') - new_list.append(image) - - return HDUList(sorted(new_list, key=fits_ext_comp_key)) - - -def read_fits(cls, source, extname_parser=None): - """ - Takes either a string (with the path to a file) or an HDUList as input, and - tries to return a populated AstroData (or descendant) instance. - - It will raise exceptions if the file is not found, or if there is no match - for the HDUList, among the registered AstroData classes. - """ - - ad = cls() - - if isinstance(source, (str, os.PathLike)): - hdulist = fits.open(source, memmap=True, - do_not_scale_image_data=True, mode='readonly') - ad.path = source - else: - hdulist = source - try: - ad.path = source[0].header.get('ORIGNAME') - except AttributeError: - ad.path = None - - _file = hdulist._file - hdulist = _prepare_hdulist(hdulist, default_extension=DEFAULT_EXTENSION, - extname_parser=extname_parser) - if _file is not None: - hdulist._file = _file - - # Initialize the object containers to a bare minimum - if 'ORIGNAME' not in hdulist[0].header and ad.orig_filename is not None: - hdulist[0].header.set('ORIGNAME', ad.orig_filename, - 'Original filename prior to processing') - - ad.phu = hdulist[0].header - seen = {hdulist[0]} - skip_names = {DEFAULT_EXTENSION, 'REFCAT', 'MDF'} - - def associated_extensions(ver): - for hdu in hdulist: - if hdu.header.get('EXTVER') == ver and hdu.name not in skip_names: - yield hdu - - # Only SCI HDUs - sci_units = [hdu for hdu in hdulist[1:] if hdu.name == DEFAULT_EXTENSION] - - seen_vers = [] - for idx, hdu in enumerate(sci_units): - seen.add(hdu) - ver = hdu.header.get('EXTVER', -1) - if ver > -1 and seen_vers.count(ver) == 1: - LOGGER.warning(f"Multiple SCI extension with EXTVER {ver}") - seen_vers.append(ver) - parts = { - 'data': hdu, - 'uncertainty': None, - 'mask': None, - 'wcs': None, - 'other': [], - } - - # For each SCI HDU find if it has an associated variance, mask, wcs - for extra_unit in associated_extensions(ver): - seen.add(extra_unit) - name = extra_unit.name - if name == 'DQ': - parts['mask'] = extra_unit - elif name == 'VAR': - parts['uncertainty'] = extra_unit - elif name == 'WCS': - parts['wcs'] = extra_unit - else: - parts['other'].append(extra_unit) - - header = parts['data'].header - lazy = hdulist._file is not None and hdulist._file.memmap - - for part_name in ('data', 'mask', 'uncertainty'): - if parts[part_name] is not None: - if lazy: - # Use FitsLazyLoadable to delay loading of the data - parts[part_name] = FitsLazyLoadable(parts[part_name]) - else: - # Otherwise use the data array - #parts[part_name] = parts[part_name].data - # TODO: we open the file with do_not_scale_data=True, so - # the data array does not have the correct data values. - # AstroData handles scaling internally, and we can ensure - # it does that by making the data a FitsLazyLoadable; the - # side-effect of this is that the is_lazy() function will - # return True, but this has minimal knock-on effects. - # Hopefully astropy will handle this better in future. - if hdulist._file is not None: # probably compressed - parts[part_name] = FitsLazyLoadable(parts[part_name]) - else: # for astrodata.create() files - parts[part_name] = parts[part_name].data - - # handle the variance if not lazy - if (parts['uncertainty'] is not None and - not isinstance(parts['uncertainty'], FitsLazyLoadable)): - parts['uncertainty'] = ADVarianceUncertainty(parts['uncertainty']) - - # Create the NDData object - nd = NDDataObject( - data=parts['data'], - uncertainty=parts['uncertainty'], - mask=parts['mask'], - meta={'header': header}, - ) - - ad.append(nd, name=DEFAULT_EXTENSION) - - # This is used in the writer to keep track of the extensions that - # were read from the current object. - nd.meta['parent_ad'] = id(ad) - - for other in parts['other']: - if not other.name: - warnings.warn(f"Skip HDU {other} because it has no EXTNAME") - else: - setattr(ad[-1], other.name, other) - - if parts['wcs'] is not None: - # Load the gWCS object from the ASDF extension - nd.wcs = asdftablehdu_to_wcs(parts['wcs']) - if nd.wcs is None: - # Fallback to the data header - nd.wcs = fitswcs_to_gwcs(nd) - if nd.wcs is None: - # In case WCS info is in the PHU - nd.wcs = fitswcs_to_gwcs(hdulist[0].header) - - for other in hdulist: - if other in seen: - continue - name = other.header.get('EXTNAME') - try: - ad.append(other, name=name) - except ValueError as e: - warnings.warn(f"Discarding {name} :\n {e}") - - return ad - - -def ad_to_hdulist(ad): - """Creates an HDUList from an AstroData object.""" - hdul = HDUList() - hdul.append(PrimaryHDU(header=ad.phu, data=DELAYED)) - - # Find the maximum EXTVER for extensions that belonged with this - # object if it was read from a FITS file - maxver = max((nd.meta['header'].get('EXTVER', 0) for nd in ad._nddata - if nd.meta.get('parent_ad') == id(ad)), - default=0) - - for ext in ad._nddata: - header = ext.meta['header'].copy() - - if not isinstance(header, fits.Header): - header = fits.Header(header) - - if ext.meta.get('parent_ad') == id(ad): - # If the extension belonged with this object, use its - # original EXTVER - ver = header['EXTVER'] - else: - # Otherwise renumber the extension - ver = header['EXTVER'] = maxver + 1 - maxver += 1 - - wcs = ext.wcs - - if isinstance(wcs, gWCS): - # We don't have access to the AD tags so see if it's an image - # Catch ValueError as any sort of failure - try: - wcs_dict = gwcs_to_fits(ext, ad.phu) - except (ValueError, NotImplementedError) as e: - LOGGER.warning(e) - else: - # Must delete keywords if image WCS has been downscaled - # from a higher number of dimensions - for i in range(1, 5): - for kw in (f'CDELT{i}', f'CRVAL{i}', f'CUNIT{i}', - f'CTYPE{i}', f'NAXIS{i}'): - if kw in header: - del header[kw] - for j in range(1, 5): - for kw in (f'CD{i}_{j}', f'PC{i}_{j}', f'CRPIX{j}'): - if kw in header: - del header[kw] - # Delete this if it's left over from a previous save - if 'FITS-WCS' in header: - del header['FITS-WCS'] - try: - extensions = wcs_dict.pop('extensions') - except KeyError: - pass - else: - for k, v in extensions.items(): - ext.meta['other'][k] = v - header.update(wcs_dict) - # Use "in" here as the dict entry may be (value, comment) - if 'APPROXIMATE' not in wcs_dict.get('FITS-WCS', ''): - wcs = None # There's no need to create a WCS extension - - hdul.append(new_imagehdu(ext.data, header, 'SCI')) - if ext.uncertainty is not None: - hdul.append(new_imagehdu(ext.uncertainty.array, header, 'VAR')) - if ext.mask is not None: - hdul.append(new_imagehdu(ext.mask, header, 'DQ')) - - if isinstance(wcs, gWCS): - hdul.append(wcs_to_asdftablehdu(ext.wcs, extver=ver)) - - for name, other in ext.meta.get('other', {}).items(): - if isinstance(other, Table): - hdu = table_to_bintablehdu(other, extname=name) - elif isinstance(other, np.ndarray): - hdu = new_imagehdu(other, header, name=name) - elif isinstance(other, NDDataObject): - hdu = new_imagehdu(other.data, ext.meta['header']) - else: - raise ValueError("I don't know how to write back an object " - f"of type {type(other)}") - - hdu.ver = ver - hdul.append(hdu) - - if ad._tables is not None: - for name, table in sorted(ad._tables.items()): - hdul.append(table_to_bintablehdu(table, extname=name)) - - # Additional FITS compatibility, add to PHU - hdul[0].header['NEXTEND'] = len(hdul) - 1 - - return hdul - - -def write_fits(ad, filename, overwrite=False): - """Writes the AstroData object to a FITS file.""" - hdul = ad_to_hdulist(ad) - hdul.writeto(filename, overwrite=overwrite) - - -def windowedOp(func, sequence, kernel, shape=None, dtype=None, - with_uncertainty=False, with_mask=False, **kwargs): - """Apply function on a NDData obbjects, splitting the data in chunks to - limit memory usage. - - Parameters - ---------- - func : callable - The function to apply. - sequence : list of NDData - List of NDData objects. - kernel : tuple of int - Shape of the blocks. - shape : tuple of int - Shape of inputs. Defaults to ``sequence[0].shape``. - dtype : str or dtype - Type of the output array. Defaults to ``sequence[0].dtype``. - with_uncertainty : bool - Compute uncertainty? - with_mask : bool - Compute mask? - **kwargs - Additional args are passed to ``func``. - - """ - - def generate_boxes(shape, kernel): - if len(shape) != len(kernel): - raise AssertionError("Incompatible shape ({}) and kernel ({})" - .format(shape, kernel)) - ticks = [[(x, x+step) for x in range(0, axis, step)] - for axis, step in zip(shape, kernel)] - return list(cart_product(*ticks)) - - if shape is None: - if len({x.shape for x in sequence}) > 1: - raise ValueError("Can't calculate final shape: sequence elements " - "disagree on shape, and none was provided") - shape = sequence[0].shape - - if dtype is None: - dtype = sequence[0].window[:1, :1].data.dtype - - result = NDDataObject( - np.empty(shape, dtype=dtype), - variance=np.zeros(shape, dtype=dtype) if with_uncertainty else None, - mask=np.empty(shape, dtype=np.uint16) if with_mask else None, - meta=sequence[0].meta, - wcs=sequence[0].wcs, - ) - # Delete other extensions because we don't know what to do with them - result.meta['other'] = OrderedDict() - - # The Astropy logger's "INFO" messages aren't warnings, so have to fudge - log_level = astropy.logger.conf.log_level - astropy.log.setLevel(astropy.logger.WARNING) - - boxes = generate_boxes(shape, kernel) - - try: - for coords in boxes: - section = tuple([slice(start, end) for (start, end) in coords]) - out = func([element.window[section] for element in sequence], - **kwargs) - result.set_section(section, out) - - # propagate additional attributes - if out.meta.get('other'): - for k, v in out.meta['other'].items(): - if len(boxes) > 1: - result.meta['other'][k, coords] = v - else: - result.meta['other'][k] = v - - gc.collect() - finally: - astropy.log.setLevel(log_level) # and reset - - # Now if the input arrays where splitted in chunks, we need to gather - # the data arrays for the additional attributes. - other = result.meta['other'] - if other: - if len(boxes) > 1: - for (name, coords), obj in list(other.items()): - if not isinstance(obj, NDData): - raise ValueError('only NDData objects are handled here') - if name not in other: - other[name] = NDDataObject(np.empty(shape, - dtype=obj.data.dtype)) - section = tuple([slice(start, end) for (start, end) in coords]) - other[name].set_section(section, obj) - del other[name, coords] - - for name in other: - # To set the name of our object we need to save it as an ndarray, - # otherwise for a NDData one AstroData would use the name of the - # AstroData object. - other[name] = other[name].data - - return result - - -# --------------------------------------------------------------------------- -# gWCS <-> FITS WCS helper functions go here -# --------------------------------------------------------------------------- -# Could parametrize some naming conventions in the following two functions if -# done elsewhere for hard-coded names like 'SCI' in future, but they only have -# to be self-consistent with one another anyway. - -def wcs_to_asdftablehdu(wcs, extver=None): - """ - Serialize a gWCS object as a FITS TableHDU (ASCII) extension. - - The ASCII table is actually a mini ASDF file. The constituent AstroPy - models must have associated ASDF "tags" that specify how to serialize them. - - In the event that serialization as pure ASCII fails (this should not - happen), a binary table representation will be used as a fallback. - """ - - # Create a small ASDF file in memory containing the WCS object - # representation because there's no public API for generating only the - # relevant YAML subsection and an ASDF file handles the "tags" properly. - try: - af = asdf.AsdfFile({"wcs": wcs}) - except jsonschema.exceptions.ValidationError: - # (The original traceback also gets printed here) - raise TypeError("Cannot serialize model(s) for 'WCS' extension {}" - .format(extver or '')) - - # ASDF can only dump YAML to a binary file object, so do that and read - # the contents back from it for storage in a FITS extension: - with BytesIO() as fd: - with af: - # Generate the YAML, dumping any binary arrays as text: - af.write_to(fd, all_array_storage='inline') - fd.seek(0) - wcsbuf = fd.read() - - # Convert the bytes to readable lines of text for storage (falling back to - # saving as binary in the unexpected event that this is not possible): - try: - wcsbuf = wcsbuf.decode('ascii').splitlines() - except UnicodeDecodeError: - # This should not happen, but if the ASDF contains binary data in - # spite of the 'inline' option above, we have to dump the bytes to - # a non-human-readable binary table rather than an ASCII one: - LOGGER.warning("Could not convert WCS {} ASDF to ASCII; saving table " - "as binary".format(extver or '')) - hduclass = BinTableHDU - fmt = 'B' - wcsbuf = np.frombuffer(wcsbuf, dtype=np.uint8) - else: - hduclass = TableHDU - fmt = 'A{}'.format(max(len(line) for line in wcsbuf)) - - # Construct the FITS table extension: - col = Column(name='gWCS', format=fmt, array=wcsbuf, - ascii=hduclass is TableHDU) - return hduclass.from_columns([col], name='WCS', ver=extver) - - -def asdftablehdu_to_wcs(hdu): - """ - Recreate a gWCS object from its serialization in a FITS table extension. - - Returns None (issuing a warning) if the extension cannot be parsed, so - the rest of the file can still be read. - """ - - ver = hdu.header.get('EXTVER', -1) - - if isinstance(hdu, (TableHDU, BinTableHDU)): - try: - colarr = hdu.data['gWCS'] - except KeyError: - LOGGER.warning("Ignoring 'WCS' extension {} with no 'gWCS' table " - "column".format(ver)) - return - - # If this table column contains text strings as expected, join the rows - # as separate lines of a string buffer and encode the resulting YAML as - # bytes that ASDF can parse. If AstroData has produced another format, - # it will be a binary dump due to the unexpected presence of non-ASCII - # data, in which case we just extract unmodified bytes from the table. - if colarr.dtype.kind in ('U', 'S'): - sep = os.linesep - # Just in case io.fits ever produces 'S' on Py 3 (not the default): - # join lines as str & avoid a TypeError with unicode linesep; could - # also use astype('U') but it assumes an encoding implicitly. - if colarr.dtype.kind == 'S' and not isinstance(sep, bytes): - colarr = np.char.decode(np.char.rstrip(colarr), - encoding='ascii') - wcsbuf = sep.join(colarr).encode('ascii') - else: - wcsbuf = colarr.tobytes() - - # Convert the stored text to a Bytes file object that ASDF can open: - with BytesIO(wcsbuf) as fd: - - # Try to extract a 'wcs' entry from the YAML: - try: - af = asdf.open(fd) - except Exception: - LOGGER.warning("Ignoring 'WCS' extension {}: failed to parse " - "ASDF.\nError was as follows:\n{}" - .format(ver, traceback.format_exc())) - return - else: - with af: - try: - wcs = af.tree['wcs'] - except KeyError: - LOGGER.warning("Ignoring 'WCS' extension {}: missing " - "'wcs' dict entry.".format(ver)) - return - - else: - LOGGER.warning("Ignoring non-FITS-table 'WCS' extension {}" - .format(ver)) - return - - return wcs diff --git a/astrodata/nddata.py b/astrodata/nddata.py deleted file mode 100644 index ce15ab930b..0000000000 --- a/astrodata/nddata.py +++ /dev/null @@ -1,513 +0,0 @@ -""" -This module implements a derivative class based on NDData with some Mixins, -implementing windowing and on-the-fly data scaling. -""" - - -import warnings -from copy import deepcopy -from functools import reduce - -import numpy as np - -from astropy.io.fits import ImageHDU -from astropy.modeling import Model, models -from astropy.nddata import (NDArithmeticMixin, NDData, NDSlicingMixin, - VarianceUncertainty) -from gwcs.wcs import WCS as gWCS -from .wcs import remove_axis_from_frame - -INTEGER_TYPES = (int, np.integer) - -__all__ = ['NDAstroData'] - - -class ADVarianceUncertainty(VarianceUncertainty): - """ - Subclass VarianceUncertainty to check for negative values. - """ - @VarianceUncertainty.array.setter - def array(self, value): - if value is not None and np.any(value < 0): - warnings.warn("Negative variance values found. Setting to zero.", - RuntimeWarning) - value = np.where(value >= 0., value, 0.) - VarianceUncertainty.array.fset(self, value) - - -class AstroDataMixin: - """ - A Mixin for ``NDData``-like classes (such as ``Spectrum1D``) to enable - them to behave similarly to ``AstroData`` objects. - - These behaviors are: - 1. ``mask`` attributes are combined with bitwise, not logical, or, - since the individual bits are important. - 2. The WCS must be a ``gwcs.WCS`` object and slicing results in - the model being modified. - 3. There is a settable ``variance`` attribute. - 4. Additional attributes such as OBJMASK can be extracted from - the .meta['other'] dict - """ - def __getattr__(self, attribute): - """ - Allow access to attributes stored in self.meta['other'], as we do - with AstroData objects. - """ - if attribute.isupper(): - try: - return self.meta['other'][attribute] - except KeyError: - pass - raise AttributeError(f"{self.__class__.__name__!r} object has no " - f"attribute {attribute!r}") - - def _arithmetic(self, operation, operand, propagate_uncertainties=True, - handle_mask=np.bitwise_or, handle_meta=None, - uncertainty_correlation=0, compare_wcs='first_found', - **kwds): - """ - Override the NDData method so that "bitwise_or" becomes the default - operation to combine masks, rather than "logical_or" - """ - return super()._arithmetic( - operation, operand, propagate_uncertainties=propagate_uncertainties, - handle_mask=handle_mask, handle_meta=handle_meta, - uncertainty_correlation=uncertainty_correlation, - compare_wcs=compare_wcs, **kwds) - - def _slice_wcs(self, slices): - """ - The ``__call__()`` method of gWCS doesn't appear to conform to the - APE 14 interface for WCS implementations, and doesn't react to - slicing properly. We override NDSlicing's method to do what we want. - """ - if not isinstance(self.wcs, gWCS): - return self.wcs - - # Sanitize the slices, catching some errors early - if not isinstance(slices, (tuple, list)): - slices = (slices,) - slices = list(slices) - ndim = len(self.shape) - if len(slices) > ndim: - raise ValueError(f"Too many dimensions specified in slice {slices}") - - if Ellipsis in slices: - if slices.count(Ellipsis) > 1: - raise IndexError("Only one ellipsis can be specified in a slice") - ell_index = slices.index(Ellipsis) - slices[ell_index:ell_index+1] = [slice(None)] * (ndim - len(slices) + 1) - slices.extend([slice(None)] * (ndim-len(slices))) - - mods = [] - mapped_axes = [] - for i, (slice_, length) in enumerate(zip(slices[::-1], self.shape[::-1])): - model = [] - if isinstance(slice_, slice): - if slice_.step and slice_.step > 1: - raise IndexError("Cannot slice with a step") - if slice_.start: - start = (length + slice_.start) if slice_.start < 0 else slice_.start - if start > 0: - model.append(models.Shift(start)) - mapped_axes.append(max(mapped_axes) + 1 if mapped_axes else 0) - elif isinstance(slice_, INTEGER_TYPES): - model.append(models.Const1D((length + slice_) if slice_ < 0 else slice_)) - mapped_axes.append(-1) - elif slice_ is None: # equivalent to slice(None, None, None) - mapped_axes.append(max(mapped_axes) + 1 if mapped_axes else 0) - else: - raise IndexError("Slice not an integer or range") - if model: - mods.append(reduce(Model.__or__, model)) - else: - # If the previous model was an Identity, we can hang this - # one onto that without needing to append a new Identity - if i > 0 and isinstance(mods[-1], models.Identity): - mods[-1] = models.Identity(mods[-1].n_inputs + 1) - else: - mods.append(models.Identity(1)) - - slicing_model = reduce(Model.__and__, mods) - if mapped_axes != list(np.arange(ndim)): - slicing_model = models.Mapping( - tuple(max(ax, 0) for ax in mapped_axes)) | slicing_model - slicing_model.inverse = models.Mapping( - tuple(ax for ax in mapped_axes if ax != -1), n_inputs=ndim) - - if isinstance(slicing_model, models.Identity) and slicing_model.n_inputs == ndim: - return self.wcs # Unchanged! - new_wcs = deepcopy(self.wcs) - input_frame = new_wcs.input_frame - for axis, mapped_axis in reversed(list(enumerate(mapped_axes))): - if mapped_axis == -1: - input_frame = remove_axis_from_frame(input_frame, axis) - new_wcs.pipeline[0].frame = input_frame - new_wcs.insert_transform(new_wcs.input_frame, slicing_model, after=True) - return new_wcs - - @property - def variance(self): - """ - A convenience property to access the contents of ``uncertainty``. - """ - arr = self.uncertainty - if arr is not None: - return arr.array - - @variance.setter - def variance(self, value): - self.uncertainty = (ADVarianceUncertainty(value) if value is not None - else None) - - @property - def wcs(self): - return super().wcs - - @wcs.setter - def wcs(self, value): - if value is not None and not isinstance(value, gWCS): - raise TypeError("wcs value must be None or a gWCS object") - self._wcs = value - - @property - def shape(self): - return self._data.shape - - @property - def size(self): - return self._data.size - - -class FakeArray: - - def __init__(self, very_faked): - self.data = very_faked - self.shape = (100, 100) # Won't matter. This is just to fool NDData - self.dtype = np.float32 # Same here - - def __getitem__(self, index): - # FAKE NEWS! - return None - - def __array__(self): - return self.data - - -class NDWindowing: - - def __init__(self, target): - self._target = target - - def __getitem__(self, slice): - return NDWindowingAstroData(self._target, window=slice) - - -class NDWindowingAstroData(AstroDataMixin, NDArithmeticMixin, NDSlicingMixin, NDData): - """ - Allows "windowed" access to some properties of an ``NDAstroData`` instance. - In particular, ``data``, ``uncertainty``, ``variance``, and ``mask`` return - clipped data. - """ - def __init__(self, target, window): - self._target = target - self._window = window - - def __getattr__(self, attribute): - """ - Allow access to attributes stored in self.meta['other'], as we do - with AstroData objects. - """ - if attribute.isupper(): - try: - return self._target._get_simple(attribute, section=self._window) - except KeyError: - pass - raise AttributeError(f"{self.__class__.__name__!r} object has no " - f"attribute {attribute!r}") - - @property - def unit(self): - return self._target.unit - - @property - def wcs(self): - return self._target._slice_wcs(self._window) - - @property - def data(self): - return self._target._get_simple('_data', section=self._window) - - @property - def uncertainty(self): - return self._target._get_uncertainty(section=self._window) - - @property - def variance(self): - if self.uncertainty is not None: - return self.uncertainty.array - - @property - def mask(self): - return self._target._get_simple('_mask', section=self._window) - - -def is_lazy(item): - return isinstance(item, ImageHDU) or (hasattr(item, 'lazy') and item.lazy) - - -class NDAstroData(AstroDataMixin, NDArithmeticMixin, NDSlicingMixin, NDData): - """ - Implements ``NDData`` with all Mixins, plus some ``AstroData`` specifics. - - This class implements an ``NDData``-like container that supports reading - and writing as implemented in the ``astropy.io.registry`` and also slicing - (indexing) and simple arithmetics (add, subtract, divide and multiply). - - A very important difference between ``NDAstroData`` and ``NDData`` is that - the former attempts to load all its data lazily. There are also some - important differences in the interface (eg. ``.data`` lets you reset its - contents after initialization). - - Documentation is provided where our class differs. - - See also - -------- - NDData - NDArithmeticMixin - NDSlicingMixin - - Examples - -------- - - The mixins allow operation that are not possible with ``NDData`` or - ``NDDataBase``, i.e. simple arithmetics:: - - >>> from astropy.nddata import StdDevUncertainty - >>> import numpy as np - >>> data = np.ones((3,3), dtype=np.float) - >>> ndd1 = NDAstroData(data, uncertainty=StdDevUncertainty(data)) - >>> ndd2 = NDAstroData(data, uncertainty=StdDevUncertainty(data)) - >>> ndd3 = ndd1.add(ndd2) - >>> ndd3.data - array([[2., 2., 2.], - [2., 2., 2.], - [2., 2., 2.]]) - >>> ndd3.uncertainty.array - array([[1.41421356, 1.41421356, 1.41421356], - [1.41421356, 1.41421356, 1.41421356], - [1.41421356, 1.41421356, 1.41421356]]) - - see ``NDArithmeticMixin`` for a complete list of all supported arithmetic - operations. - - But also slicing (indexing) is possible:: - - >>> ndd4 = ndd3[1,:] - >>> ndd4.data - array([2., 2., 2.]) - >>> ndd4.uncertainty.array - array([1.41421356, 1.41421356, 1.41421356]) - - See ``NDSlicingMixin`` for a description how slicing works (which - attributes) are sliced. - - """ - def __init__(self, data, uncertainty=None, mask=None, wcs=None, - meta=None, unit=None, copy=False, window=None, variance=None): - - if variance is not None: - if uncertainty is not None: - raise ValueError() - uncertainty = ADVarianceUncertainty(variance) - - super().__init__(FakeArray(data) if is_lazy(data) else data, - None if is_lazy(uncertainty) else uncertainty, - mask, wcs, meta, unit, copy) - - if is_lazy(data): - self.data = data - if is_lazy(uncertainty): - self.uncertainty = uncertainty - - def __deepcopy__(self, memo): - new = self.__class__( - self._data if is_lazy(self._data) else deepcopy(self.data, memo), - self._uncertainty if is_lazy(self._uncertainty) else None, - self._mask if is_lazy(self._mask) else deepcopy(self.mask, memo), - deepcopy(self.wcs, memo), None, self.unit - ) - new.meta = deepcopy(self.meta, memo) - # Needed to avoid recursion because of uncertainty's weakref to self - if not is_lazy(self._uncertainty): - new.variance = deepcopy(self.variance) - return new - - @property - def window(self): - """ - Interface to access a section of the data, using lazy access whenever - possible. - - Returns - -------- - An instance of ``NDWindowing``, which provides ``__getitem__``, - to allow the use of square brackets when specifying the window. - Ultimately, an ``NDWindowingAstrodata`` instance is returned. - - Examples - --------- - - >>> ad[0].nddata.window[100:200, 100:200] # doctest: +SKIP - - - """ - return NDWindowing(self) - - def _get_uncertainty(self, section=None): - """Return the ADVarianceUncertainty object, or a slice of it.""" - if self._uncertainty is not None: - if is_lazy(self._uncertainty): - if section is None: - self.uncertainty = ADVarianceUncertainty(self._uncertainty.data) - return self.uncertainty - else: - return ADVarianceUncertainty(self._uncertainty[section]) - elif section is not None: - return self._uncertainty[section] - else: - return self._uncertainty - - def _get_simple(self, target, section=None): - """Only use 'section' for image-like objects that have the same shape - as the NDAstroData object; otherwise, return the whole object""" - source = getattr(self, target) - if source is not None: - if is_lazy(source): - if section is None: - ret = np.empty(source.shape, dtype=source.dtype) - ret[:] = source.data - setattr(self, target, ret) - else: - ret = source[section] - return ret - elif hasattr(source, 'shape'): - if section is None or source.shape != self.shape: - return np.array(source, copy=False) - else: - return np.array(source, copy=False)[section] - else: - return source - - @property - def data(self): - """ - An array representing the raw data stored in this instance. - It implements a setter. - """ - return self._get_simple('_data') - - @data.setter - def data(self, value): - if value is None: - raise ValueError("Cannot have None as the data value for an NDData object") - - if is_lazy(value): - self.meta['header'] = value.header - self._data = value - - @property - def uncertainty(self): - return self._get_uncertainty() - - @uncertainty.setter - def uncertainty(self, value): - if value is not None and not is_lazy(value): - if value._parent_nddata is not None: - value = value.__class__(value, copy=False) - value.parent_nddata = self - self._uncertainty = value - - @property - def mask(self): - return self._get_simple('_mask') - - @mask.setter - def mask(self, value): - self._mask = value - - @property - def variance(self): - """ - A convenience property to access the contents of ``uncertainty``, - squared (as the uncertainty data is stored as standard deviation). - """ - arr = self._get_uncertainty() - if arr is not None: - return arr.array - - @variance.setter - def variance(self, value): - self.uncertainty = (ADVarianceUncertainty(value) if value is not None - else None) - - def set_section(self, section, input): - """ - Sets only a section of the data. This method is meant to prevent - fragmentation in the Python heap, by reusing the internal structures - instead of replacing them with new ones. - - Args - ----- - section : ``slice`` - The area that will be replaced - input : ``NDData``-like instance - This object needs to implement at least ``data``, ``uncertainty``, - and ``mask``. Their entire contents will replace the data in the - area defined by ``section``. - - Examples - --------- - - >>> sec = NDData(np.zeros((100,100))) # doctest: +SKIP - >>> ad[0].nddata.set_section((slice(None,100),slice(None,100)), sec) # doctest: +SKIP - - """ - self.data[section] = input.data - if self.uncertainty is not None: - self.uncertainty.array[section] = input.uncertainty.array - if self.mask is not None: - self.mask[section] = input.mask - - def __repr__(self): - if is_lazy(self._data): - return self.__class__.__name__ + '(Memmapped)' - else: - return super().__repr__() - - @property - def T(self): - return self.transpose() - - def transpose(self): - unc = self.uncertainty - new_wcs = deepcopy(self.wcs) - inframe = new_wcs.input_frame - new_wcs.insert_transform(inframe, models.Mapping(tuple(reversed(range(inframe.naxes)))), after=True) - return self.__class__( - self.data.T, - uncertainty=None if unc is None else unc.__class__(unc.array.T), - mask=None if self.mask is None else self.mask.T, wcs=new_wcs, - meta=self.meta, copy=False - ) - - def _slice(self, item): - """Additionally slice things like OBJMASK""" - kwargs = super()._slice(item) - if 'other' in kwargs['meta']: - kwargs['meta'] = deepcopy(self.meta) - for k, v in kwargs['meta']['other'].items(): - if isinstance(v, np.ndarray) and v.shape == self.shape: - kwargs['meta']['other'][k] = v[item] - return kwargs diff --git a/astrodata/provenance.py b/astrodata/provenance.py deleted file mode 100644 index be24a5ed29..0000000000 --- a/astrodata/provenance.py +++ /dev/null @@ -1,303 +0,0 @@ -import json - -from astropy.table import Table -from datetime import datetime, timezone - - -def add_provenance(ad, filename, md5, primitive, timestamp=None): - """ - Add the given provenance entry to the full set of provenance records on - this object. - - Provenance is added even if the incoming md5 is None or ''. This indicates - source data for the provenance that are not on disk. - - Parameters - ---------- - ad : `astrodata.AstroData` - filename : str - md5 : str - primitive : str - timestamp : `datetime.datetime` - - """ - # Handle data where the md5 is None. It will need to be a string type to - # store in the FITS table. - md5 = '' if md5 is None else md5 - - if timestamp is None: - timestamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() - - if hasattr(ad, 'PROVENANCE'): - existing_provenance = ad.PROVENANCE - for row in existing_provenance: - if row[1] == filename and \ - row[2] == md5 and \ - row[3] == primitive: - # nothing needed, we already have it - return - - if not hasattr(ad, 'PROVENANCE'): - timestamp_data = [timestamp] - filename_data = [filename] - md5_data = [md5] - provenance_added_by_data = [primitive] - ad.PROVENANCE = Table( - [timestamp_data, filename_data, md5_data, provenance_added_by_data], - names=('timestamp', 'filename', 'md5', 'provenance_added_by'), - dtype=('S28', 'S128', 'S128', 'S128')) - else: - provenance = ad.PROVENANCE - provenance.add_row((timestamp, filename, md5, primitive)) - - -def add_history(ad, timestamp_start, timestamp_stop, primitive, args): - """ - Add the given History entry to the full set of history records on this - object. - - Parameters - ---------- - ad : `astrodata.AstroData` - AstroData object to add history record to. - timestamp_start : `datetime.datetime` - Date of the start of this operation. - timestamp_stop : `datetime.datetime` - Date of the end of this operation. - primitive : str - Name of the primitive performed. - args : str - Arguments used for the primitive call. - - """ - # If the ad instance has the old 'PROVHISTORY' extenstion name, rename it - # now to 'HISTORY' - if hasattr(ad, 'PROVHISTORY'): - ad.HISTORY = ad.PROVHISTORY - del ad.PROVHISTORY - - # I modified these indices, so making this method adaptive to existing - # histories with the old ordering. This also makes modifying the order - # in future easier - primitive_col_idx, args_col_idx, timestamp_start_col_idx, \ - timestamp_stop_col_idx = find_history_column_indices(ad) - - if hasattr(ad, 'HISTORY') and None not in (primitive_col_idx, args_col_idx, - timestamp_stop_col_idx, - timestamp_start_col_idx): - for row in ad.HISTORY: - if timestamp_start == row[timestamp_start_col_idx] and \ - timestamp_stop == row[timestamp_stop_col_idx] and \ - primitive == row[primitive_col_idx] and \ - args == row[args_col_idx]: - # already in the history, skip - return - - colsize = len(args)+1 - if hasattr(ad, 'HISTORY'): - colsize = max(colsize, (max(len(ph[args_col_idx]) - for ph in ad.HISTORY) + 1) - if args_col_idx is not None else 16) - - timestamp_start_arr = [ph[timestamp_start_col_idx] - if timestamp_start_col_idx is not None else '' - for ph in ad.HISTORY] - timestamp_stop_arr = [ph[timestamp_stop_col_idx] - if timestamp_stop_col_idx is not None else '' - for ph in ad.HISTORY] - primitive_arr = [ph[primitive_col_idx] - if primitive_col_idx is not None else '' - for ph in ad.HISTORY] - args_arr = [ph[args_col_idx] if args_col_idx is not None else '' - for ph in ad.HISTORY] - else: - timestamp_start_arr = [] - timestamp_stop_arr = [] - primitive_arr = [] - args_arr = [] - - timestamp_start_arr.append(timestamp_start) - timestamp_stop_arr.append(timestamp_stop) - primitive_arr.append(primitive) - args_arr.append(args) - - dtype = ("S128", "S%d" % colsize, "S28", "S28") - ad.HISTORY = Table([primitive_arr, args_arr, timestamp_start_arr, - timestamp_stop_arr], - names=('primitive', 'args', 'timestamp_start', - 'timestamp_stop'), - dtype=dtype) - - -def clone_provenance(provenance_data, ad): - """ - For a single input's provenance, copy it into the output - `AstroData` object as appropriate. - - This takes a dictionary with a source filename, md5 and both its - original provenance and history information. It duplicates - the provenance data into the outgoing `AstroData` ad object. - - Parameters - ---------- - provenance_data : - Pointer to the `~astrodata.AstroData` table with the provenance - information. *Note* this may be the output `~astrodata.AstroData` - as well, so we need to handle that. - ad : `astrodata.AstroData` - Outgoing `~astrodata.AstroData` object to add provenance data to. - - - """ - pd = [(prov[1], prov[2], prov[3], prov[0]) for prov in provenance_data] - for p in pd: - add_provenance(ad, p[0], p[1], p[2], timestamp=p[3]) - - -def clone_history(history_data, ad): - """ - For a single input's history, copy it into the output - `AstroData` object as appropriate. - - This takes a dictionary with a source filename, md5 and both its - original provenance and history information. It duplicates - the history data into the outgoing `AstroData` ad object. - - Parameters - ---------- - history_data : - pointer to the `AstroData` table with the history information. - *Note* this may be the output `~astrodata.AstroData` as well, so we - need to handle that. - ad : `astrodata.AstroData` - Outgoing `~astrodata.AstroData` object to add history data - to. - - """ - primitive_col_idx, args_col_idx, timestamp_start_col_idx, \ - timestamp_stop_col_idx = find_history_column_indices(ad) - hd = [(hist[timestamp_start_col_idx], hist[timestamp_stop_col_idx], - hist[primitive_col_idx], hist[args_col_idx]) - for hist in history_data] - for h in hd: - add_history(ad, h[0], h[1], h[2], h[3]) - - -def find_history_column_indices(ad): - if hasattr(ad, 'HISTORY'): - primitive_col_idx = None - args_col_idx = None - timestamp_start_col_idx = None - timestamp_stop_col_idx = None - for idx, colname in enumerate(ad.HISTORY.colnames): - if colname == 'primitive': - primitive_col_idx = idx - elif colname == 'args': - args_col_idx = idx - elif colname == 'timestamp_start': - timestamp_start_col_idx = idx - elif colname == 'timestamp_stop': - timestamp_stop_col_idx = idx - else: - # defaults - primitive_col_idx = 0 - args_col_idx = 1 - timestamp_start_col_idx = 2 - timestamp_stop_col_idx = 3 - - return primitive_col_idx, args_col_idx, timestamp_start_col_idx, \ - timestamp_stop_col_idx - - -def provenance_summary(ad, provenance=True, history=True): - """ - Generate a pretty text display of the provenance information for an - `~astrodata.core.AstroData`. - - This pulls the provenance and history information from a - `~astrodata.core.AstroData` object and formats it for readability. The - primitive arguments in the history are wrapped across multiple lines to - keep the overall width manageable. - - Parameters - ---------- - ad : :class:`~astrodata.core.AstroData` - Input data to read provenance from - provenance : bool - True to show provenance - history : bool - True to show the history with associated parameters and timestamps - - Returns - ------- - str representation of the provenance and history - """ - retval = "" - if provenance: - if hasattr(ad, 'PROVENANCE'): - retval = f"Provenance\n----------\n{ad.PROVENANCE}\n" - else: - retval = "No Provenance found\n" - if history: - if provenance: - retval += "\n" # extra blank line between - if hasattr(ad, 'PROVHISTORY'): - retval += "Warning: File uses old PROVHISTORY extname." - ad.HISTORY = ad.PROVHISTORY - if hasattr(ad, 'HISTORY'): - retval += "History\n-------\n" - - primitive_col_idx, args_col_idx, timestamp_start_col_idx, \ - timestamp_stop_col_idx = find_history_column_indices(ad) - - primitive_col_size = 9 - timestamp_start_col_size = 28 - timestamp_stop_col_size = 28 - args_col_size = 16 - - # infer args size by finding the max for the folded json values - for row in ad.HISTORY: - argsstr = row[args_col_idx] - args = json.loads(argsstr) - argspp = json.dumps(args, indent=4) - for line in argspp.split('\n'): - args_col_size = max(args_col_size, len(line)) - primitive_col_size = max(primitive_col_size, - len(row[primitive_col_idx])) - - # Titles - retval += f'{"Primitive":<{primitive_col_size}} ' \ - f'{"Args":<{args_col_size}} ' \ - f'{"Start":<{timestamp_start_col_size}} {"Stop"}\n' - # now the lines - retval += f'{"":{"-"}<{primitive_col_size}} ' \ - f'{"":{"-"}<{args_col_size}} ' \ - f'{"":{"-"}<{timestamp_start_col_size}} ' \ - f'{"":{"-"}<{timestamp_stop_col_size}}\n' - - # Rows, looping over args lines - for row in ad.HISTORY: - primitive = row[primitive_col_idx] - args = row[args_col_idx] - start = row[timestamp_start_col_idx] - stop = row[timestamp_stop_col_idx] - first = True - try: - parseargs = json.loads(args) - args = json.dumps(parseargs, indent=4) - except Exception: - pass # ok, just use whatever non-json was in there - for argrow in args.split('\n'): - if first: - retval += f'{primitive:<{primitive_col_size}} ' \ - f'{argrow:<{args_col_size}} ' \ - f'{start:<{timestamp_start_col_size}} ' \ - f'{stop}\n' - else: - retval += f'{"":<{primitive_col_size}} {argrow}\n' - # prep for additional arg rows without duplicating the - # other values - first = False - else: - retval += "No Provenance History found.\n" - return retval diff --git a/astrodata/testing.py b/astrodata/testing.py deleted file mode 100644 index b8b96925f1..0000000000 --- a/astrodata/testing.py +++ /dev/null @@ -1,562 +0,0 @@ -""" -Fixtures to be used in tests in DRAGONS -""" - -import os -import shutil -import urllib -import xml.etree.ElementTree as et - -import numpy as np -import pytest -from astropy.table import Table -from astropy.utils.data import download_file - -from geminidr.gemini.lookups.timestamp_keywords import timestamp_keys -from gempy.library import astrotools as at - -URL = 'https://archive.gemini.edu/file/' - - -def assert_most_close(actual, desired, max_miss, rtol=1e-7, atol=0, - equal_nan=True, verbose=True): - """ - Raises an AssertionError if the number of elements in two objects that are - not equal up to desired tolerance is greater than expected. - - See Also - -------- - :func:`~numpy.testing.assert_allclose` - - Parameters - ---------- - actual : array_like - Array obtained. - desired : array_like - Array desired. - max_miss : iny - Maximum number of mismatched elements. - rtol : float, optional - Relative tolerance. - atol : float, optional - Absolute tolerance. - equal_nan : bool, optional. - If True, NaNs will compare equal. - verbose : bool, optional - If True, the conflicting values are appended to the error message. - Raises - ------ - AssertionError - If actual and desired are not equal up to specified precision. - """ - from numpy.testing import assert_allclose - - try: - assert_allclose(actual, desired, atol=atol, equal_nan=equal_nan, - err_msg='', rtol=rtol, verbose=verbose) - - except AssertionError as e: - n_miss = e.args[0].split('\n')[3].split(':')[-1].split('(')[0].split('/')[0] - n_miss = int(n_miss.strip()) - - if n_miss > max_miss: - error_message = ( - "%g mismatching elements are more than the " % n_miss + - "expected %g." % max_miss + - '\n'.join(e.args[0].split('\n')[3:])) - - raise AssertionError(error_message) - - -def assert_most_equal(actual, desired, max_miss, verbose=True): - """ - Raises an AssertionError if more than `n` elements in two objects are not - equal. For more information, check :func:`numpy.testing.assert_equal`. - - Parameters - ---------- - actual : array_like - The object to check. - desired : array_like - The expected object. - max_miss : int - Maximum number of mismatched elements. - verbose : bool, optional - If True, the conflicting values are appended to the error message. - - Raises - ------ - AssertionError - If actual and desired are not equal. - """ - from numpy.testing import assert_equal - - try: - assert_equal(actual, desired, err_msg='', verbose=verbose) - except AssertionError as e: - - n_miss = e.args[0].split('\n')[3].split(':')[-1].split('(')[0].split('/')[0] - n_miss = int(n_miss.strip()) - - if n_miss > max_miss: - error_message = ( - "%g mismatching elements are more than the " % n_miss + - "expected %g." % max_miss + - '\n'.join(e.args[0].split('\n')[3:])) - - raise AssertionError(error_message) - - -def assert_same_class(ad, ad_ref): - """ - Compare if two :class:`~astrodata.AstroData` (or any subclass) have the - same class. - - Parameters - ---------- - ad : :class:`astrodata.AstroData` or any subclass - AstroData object to be checked. - ad_ref : :class:`astrodata.AstroData` or any subclass - AstroData object used as reference - """ - from astrodata import AstroData - - assert isinstance(ad, AstroData) - assert isinstance(ad_ref, AstroData) - assert isinstance(ad, type(ad_ref)) - - -def compare_models(model1, model2, rtol=1e-7, atol=0., check_inverse=True): - """ - Check that any two models are the same, within some tolerance on parameters - (using the same defaults as numpy.assert_allclose()). - - This is constructed like a test, rather than returning True/False, in order - to provide more useful information as to how the models differ when a test - fails (and with more concise syntax). - - If `check_inverse` is True (the default), only first-level inverses are - compared, to avoid unending recursion, since the inverse of an inverse - should be the supplied input model, if defined. The types of any inverses - (and their inverses in turn) are required to match whether or not their - parameters etc. are compared. - - This function might not completely guarantee that model1 & model2 are - identical for some models whose evaluation depends on class-specific - parameters controlling how the array of model `parameters` is interpreted - (eg. the orders in SIP?), but it does cover our common use of compound - models involving orthonormal polynomials etc. - """ - - from astropy.modeling import Model - from numpy.testing import assert_allclose - - if not (isinstance(model1, Model) and isinstance(model2, Model)): - raise TypeError('Inputs must be Model instances') - - if model1 is model2: - return - - # Require each model to be composed of same number of constituent models: - assert model1.n_submodels == model2.n_submodels - - # Treat everything like an iterable compound model: - if model1.n_submodels == 1: - model1 = [model1] - model2 = [model2] - - # Compare the constituent model definitions: - for m1, m2 in zip(model1, model2): - assert type(m1) == type(m2) - assert len(m1.parameters) == len(m2.parameters) - # NB. For 1D models the degrees match if the numbers of parameters do - if hasattr(m1, 'x_degree'): - assert m1.x_degree == m2.x_degree - if hasattr(m1, 'y_degree'): - assert m1.y_degree == m2.y_degree - if hasattr(m1, 'domain'): - assert m1.domain == m2.domain - if hasattr(m1, 'x_domain'): - assert m1.x_domain == m2.x_domain - if hasattr(m1, 'y_domain'): - assert m1.y_domain == m2.y_domain - - # Compare the model parameters (coefficients): - assert_allclose(model1.parameters, model2.parameters, rtol=rtol, atol=atol) - - # Now check for any inverse models and require them both to have the same - # type or be undefined: - try: - inverse1 = model1.inverse - except NotImplementedError: - inverse1 = None - try: - inverse2 = model2.inverse - except NotImplementedError: - inverse2 = None - - assert type(inverse1) == type(inverse2) - - # Compare inverses only if they exist and are not the forward model itself: - if inverse1 is None or (inverse1 is model1 and inverse2 is model2): - check_inverse = False - - # Recurse over the inverse models (but not their inverses in turn): - if check_inverse: - compare_models(inverse1, inverse2, rtol=rtol, atol=atol, - check_inverse=False) - - -def download_from_archive(filename, sub_path='raw_files', env_var='DRAGONS_TEST', url=None): - """ - Download file from the archive (or an alternative URL) and store it in the - local cache. - - Parameters - ---------- - filename : str - The filename, e.g. N20160524S0119.fits - sub_path : str - By default the file is stored at the root of the cache directory, but - using ``path`` allows to specify a sub-directory. - env_var: str - Environment variable containing the path to the cache directory. - url : str, optional - Alternative URL of the file to be fetched, if not using the Gemini - archive (a full path is required, since the last part of the path - component may differ from the nominal `filename` for some services, - such as Google Drive). - - Returns - ------- - str - Name of the cached file with the path added to it. - """ - # Find cache path and make sure it exists - root_cache_path = os.getenv(env_var) - - if root_cache_path is None: - raise ValueError(f'Environment variable not set: {env_var}') - - root_cache_path = os.path.expanduser(root_cache_path) - - if sub_path is not None: - cache_path = os.path.join(root_cache_path, sub_path) - - if not os.path.exists(cache_path): - os.makedirs(cache_path) - - if url is None: - url = URL + filename - - # Now check if the local file exists and download if not - local_path = os.path.join(cache_path, filename) - if not os.path.exists(local_path): - tmp_path = download_file(url, cache=False) - shutil.move(tmp_path, local_path) - - # `download_file` ignores Access Control List - fixing it - os.chmod(local_path, 0o664) - - return local_path - - -def get_associated_calibrations(filename, nbias=5): - """ - Queries Gemini Observatory Archive for associated calibrations to reduce - the data that will be used for testing. - - Parameters - ---------- - filename : str - Input file name - """ - url = f"https://archive.gemini.edu/calmgr/{filename}" - tree = et.parse(urllib.request.urlopen(url)) - root = tree.getroot() - prefix = root.tag[:root.tag.rfind('}') + 1] - - rows = [] - for node in tree.iter(prefix + 'calibration'): - cal_type = node.find(prefix + 'caltype').text - cal_filename = node.find(prefix + 'filename').text - if not ('processed_' in cal_filename or 'specphot' in cal_filename): - rows.append((cal_filename, cal_type)) - - tbl = Table(rows=rows, names=['filename', 'caltype']) - tbl.sort('filename') - tbl.remove_rows(np.where(tbl['caltype'] == 'bias')[0][nbias:]) - return tbl - - -class ADCompare: - """ - Compare two AstroData instances to determine whether they are basically - the same. Various properties (both data and metadata) can be compared - """ - # These are the keywords relating to a FITS WCS that we won't check - # because we check the gWCS objects instead and to the DRAGONS PROCxxxx - # headers, because that would mean updating every reference file every time - # the version changes (for PROCSVER) - fits_keys = {'WCSAXES', 'WCSDIM', 'RADESYS', 'BITPIX', 'PROCSOFT', - 'PROCSVER', 'PROCMODE'} - for i in range(1, 6): - fits_keys.update([f'CUNIT{i}', f'CTYPE{i}', f'CDELT{i}', f'CRVAL{i}', - f'CRPIX{i}']) - fits_keys.update([f'CD{i}_{j}' for i in range(1, 6) for j in range(1, 6)]) - - def __init__(self, ad1, ad2): - self.ad1 = ad1 - self.ad2 = ad2 - - def run_comparison(self, max_miss=0, rtol=1e-7, atol=0, compare=None, - ignore=None, ignore_fits_wcs=True, ignore_kw=None, - raise_exception=True): - """ - Perform a comparison between the two AD objects in this instance. - - Parameters - ---------- - max_miss: int - maximum number of elements in each array that can disagree - rtol: float - relative tolerance allowed between array elements - atol: float - absolute tolerance allowed between array elements - compare: list/None - list of comparisons to perform - ignore: list/None - list of comparisons to ignore - ignore_fits_wcs: bool - ignore FITS keywords relating to WCS (to allow a comparison - between an in-memory AD and one on disk if you're not interested - in these, without needed to save to disk) - ignore_kw: sequence/None - additional keywords to ignore in headers - raise_exception: bool - raise an AssertionError if the comparison fails? If False, - the errordict is returned, which may be useful if a very - specific mismatch is permitted - - Raises - ------- - AssertionError if the AD objects do not agree. - """ - self.max_miss = max_miss - self.rtol = rtol - self.atol = atol - self.ignore_kw = self.fits_keys if ignore_fits_wcs else set([]) - self.ignore_kw.add('') - if ignore_kw: - self.ignore_kw.update(ignore_kw) - if compare is None: - compare = ('filename', 'tags', 'numext', 'refcat', 'phu', - 'hdr', 'attributes', 'wcs') - if ignore is not None: - compare = [c for c in compare if c not in ignore] - - errordict = {} - for func_name in compare: - errorlist = getattr(self, func_name)() - if errorlist: - errordict[func_name] = errorlist - if errordict and raise_exception: - raise AssertionError(self.format_errordict(errordict)) - return errordict - - def numext(self): - """Check the number of extensions is equal""" - numext1, numext2 = len(self.ad1), len(self.ad2) - if numext1 != numext2: - return [f'{numext1} v {numext2}'] - - def filename(self): - """Check the filenames are equal""" - fname1, fname2 = self.ad1.filename, self.ad2.filename - if fname1 != fname2: - return [f'{fname1} v {fname2}'] - - def tags(self): - """Check the tags are equal""" - tags1, tags2 = self.ad1.tags, self.ad2.tags - if tags1 != tags2: - return [f'{tags1}\n v {tags2}'] - - def phu(self): - """Check the PHUs agree""" - # Ignore NEXTEND as only recently added and len(ad) handles it - errorlist = self._header(self.ad1.phu, self.ad2.phu, - ignore=self.ignore_kw.union({'NEXTEND'})) - if errorlist: - return errorlist - - def hdr(self): - """Check the extension headers agree""" - errorlist = [] - for i, (hdr1, hdr2) in enumerate(zip(self.ad1.hdr, self.ad2.hdr)): - elist = self._header(hdr1, hdr2, ignore=self.ignore_kw) - if elist: - errorlist.extend([f'Slice {i} HDR mismatch'] + elist) - return errorlist - - def _header(self, hdr1, hdr2, ignore=None): - """General method for comparing headers, ignoring some keywords - - Parameters - ---------- - hdr1, hdr2: dict - The headers to compare. - ignore : `list` of `str` - A list of strings corresponding to header keywords which should be - excluded from matching between both headers. - - """ - errorlist = [] - s1 = set(hdr1.keys()) - {'HISTORY', 'COMMENT'} - s2 = set(hdr2.keys()) - {'HISTORY', 'COMMENT'} - if ignore: - s1 -= set(ignore) - s2 -= set(ignore) - if s1 != s2: - if s1 - s2: - errorlist.append(f'Header 1 contains keywords {s1 - s2}') - if s2 - s1: - errorlist.append(f'Header 2 contains keywords {s2 - s1}') - - ignore_list = ['GEM-TLM', 'HISTORY', 'COMMENT', ''] - # Include keywords from `ignore` parameter. - if ignore: - ignore_list.extend(ignore) - - for kw in hdr1: - # GEM-TLM is "time last modified" - if (kw not in timestamp_keys.values() and kw not in ignore_list and - kw not in self.ignore_kw): - try: - v1, v2 = hdr1[kw], hdr2[kw] - except KeyError: # Missing keyword in AD2 - continue - try: - if abs(v1 - v2) >= 0.01: - errorlist.append(f'{kw} value mismatch: {v1} v {v2}') - except TypeError: - if v1 != v2: - errorlist.append(f'{kw} value inequality: {v1} v {v2}') - return errorlist - - def refcat(self): - """Check both ADs have REFCATs (or not) and that the lengths agree""" - refcat1 = getattr(self.ad1, 'REFCAT', None) - refcat2 = getattr(self.ad2, 'REFCAT', None) - if (refcat1 is None) ^ (refcat2 is None): - return [f'presence: {refcat1 is not None} v {refcat2 is not None}'] - elif refcat1 is not None: # and refcat2 must also exist - len1, len2 = len(refcat1), len(refcat2) - if len1 != len2: - return [f'lengths: {len1} v {len2}'] - - def attributes(self): - """Check extension-level attributes""" - errorlist = [] - for i, (ext1, ext2) in enumerate(zip(self.ad1, self.ad2)): - elist = self._attributes(ext1, ext2) - if elist: - errorlist.extend([f'Slice {i} attribute mismatch'] + elist) - return errorlist - - def _attributes(self, ext1, ext2): - """Helper method for checking attributes""" - errorlist = [] - for attr in ['data', 'mask', 'variance', 'OBJMASK', 'OBJCAT']: - attr1 = getattr(ext1, attr, None) - attr2 = getattr(ext2, attr, None) - if (attr1 is None) ^ (attr2 is None): - errorlist.append(f'Attribute error for {attr}: ' - f'{attr1 is not None} v {attr2 is not None}') - elif attr1 is not None: - if isinstance(attr1, Table): - if len(attr1) != len(attr2): - errorlist.append(f'attr lengths differ: ' - f'{len(attr1)} v {len(attr2)}') - else: # everything else is pixel-like - if attr1.dtype.name != attr2.dtype.name: - errorlist.append(f'Datatype mismatch for {attr}: ' - f'{attr1.dtype} v {attr2.dtype}') - if attr1.shape != attr2.shape: - errorlist.append(f'Shape mismatch for {attr}: ' - f'{attr1.shape} v {attr2.shape}') - if 'int' in attr1.dtype.name: - try: - assert_most_equal(attr1, attr2, max_miss=self.max_miss) - except AssertionError as e: - errorlist.append(f'Inequality for {attr}: '+str(e)) - else: - try: - assert_most_close(attr1, attr2, max_miss=self.max_miss, - rtol=self.rtol, atol=self.atol) - except AssertionError as e: - errorlist.append(f'Mismatch for {attr}: '+str(e)) - return errorlist - - def wcs(self): - """Check WCS agrees""" - def compare_frames(frame1, frame2): - """Compare the important stuff of two CoordinateFrame instances""" - for attr in ("naxes", "axes_type", "axes_order", "unit", "axes_names"): - assert getattr(frame1, attr) == getattr(frame2, attr) - - errorlist = [] - for i, (ext1, ext2) in enumerate(zip(self.ad1, self.ad2)): - wcs1, wcs2 = ext1.wcs, ext2.wcs - if (wcs1 is None) != (wcs2 is None): - errorlist.append(f'Slice {i} WCS presence mismatch ' - f'{wcs1 is not None} {wcs2 is not None}') - continue - elif wcs1 is None: # and wcs2 is also None - continue - frames1, frames2 = wcs1.available_frames, wcs2.available_frames - if frames1 != frames2: - errorlist.append(f'Slice {i} frames differ: {frames1} v {frames2}') - return errorlist - for frame in frames1: - frame1, frame2 = getattr(wcs1, frame), getattr(wcs2, frame) - try: - compare_frames(frame1, frame2) - except AssertionError: - errorlist.compare(f'Slice {i} {frame} differs: ' - f'{frame1} v {frame2}') - corners = at.get_corners(ext1.shape) - world1, world2 = wcs1(*zip(*corners)), wcs2(*zip(*corners)) - try: - np.testing.assert_allclose(world1, world2) - except AssertionError: - errorlist.append(f'Slice {i} world coords differ: {world1} v {world2}') - return errorlist - - def format_errordict(self, errordict): - """Format the errordict into a str for reporting""" - errormsg = f'Comparison between {self.ad1.filename} and {self.ad2.filename}' - for k, v in errordict.items(): - errormsg += f'\nComparison failure in {k}' - errormsg += '\n' + ('-' * (22 + len(k))) + '\n' - errormsg += '\n '.join(v) - return errormsg - -def ad_compare(ad1, ad2, **kwargs): - """ - Compares the tags, headers, and pixel values of two images. This is simply - a wrapper for ADCompare.run_comparison() for backward-compatibility. - - Parameters - ---------- - ad1: AstroData - first AD objects - ad2: AstroData - second AD object - - Returns - ------- - bool: are the two AD instances basically the same? - """ - compare = ADCompare(ad1, ad2).run_comparison(**kwargs) - return compare == {} diff --git a/astrodata/tests/__init__.py b/astrodata/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/astrodata/tests/test_core.py b/astrodata/tests/test_core.py deleted file mode 100644 index ab80e42fcd..0000000000 --- a/astrodata/tests/test_core.py +++ /dev/null @@ -1,273 +0,0 @@ -import operator -from copy import deepcopy - -import numpy as np -import pytest -from numpy.testing import assert_array_equal - -import astrodata -from astropy.io import fits -from astropy.nddata import NDData, VarianceUncertainty -from astropy.table import Table - -SHAPE = (4, 5) - - -@pytest.fixture -def ad1(): - hdr = fits.Header({'INSTRUME': 'darkimager', 'OBJECT': 'M42'}) - phu = fits.PrimaryHDU(header=hdr) - hdu = fits.ImageHDU(data=np.ones(SHAPE), name='SCI') - return astrodata.create(phu, [hdu]) - - -@pytest.fixture -def ad2(): - phu = fits.PrimaryHDU() - hdu = fits.ImageHDU(data=np.ones(SHAPE) * 2, name='SCI') - return astrodata.create(phu, [hdu]) - - -def test_attributes(ad1): - assert ad1[0].shape == SHAPE - - data = ad1[0].data - assert_array_equal(data, 1) - assert data.mean() == 1 - assert np.median(data) == 1 - - assert ad1.phu['INSTRUME'] == 'darkimager' - assert ad1.instrument() == 'darkimager' - assert ad1.object() == 'M42' - - -@pytest.mark.parametrize('op, res, res2', [ - (operator.add, 3, 3), - (operator.sub, -1, 1), - (operator.mul, 2, 2), - (operator.truediv, 0.5, 2) -]) -def test_arithmetic(op, res, res2, ad1, ad2): - for data in (ad2, ad2.data): - result = op(ad1, data) - assert_array_equal(result.data, res) - assert isinstance(result, astrodata.AstroData) - assert len(result) == 1 - assert isinstance(result[0].data, np.ndarray) - assert isinstance(result[0].hdr, fits.Header) - - result = op(data, ad1) - assert_array_equal(result.data, res2) - - for data in (ad2[0], ad2[0].data): - result = op(ad1[0], data) - assert_array_equal(result.data, res) - assert isinstance(result, astrodata.AstroData) - assert len(result) == 1 - assert isinstance(result[0].data, np.ndarray) - assert isinstance(result[0].hdr, fits.Header) - - # FIXME: should work also with ad2[0].data, but crashes - result = op(ad2[0], ad1[0]) - assert_array_equal(result.data, res2) - - -@pytest.mark.parametrize('op, res, res2', [ - (operator.iadd, 3, 3), - (operator.isub, -1, 1), - (operator.imul, 2, 2), - (operator.itruediv, 0.5, 2) -]) -def test_arithmetic_inplace(op, res, res2, ad1, ad2): - for data in (ad2, ad2.data): - ad = deepcopy(ad1) - op(ad, data) - assert_array_equal(ad.data, res) - assert isinstance(ad, astrodata.AstroData) - assert len(ad) == 1 - assert isinstance(ad[0].data, np.ndarray) - assert isinstance(ad[0].hdr, fits.Header) - - # data2 = deepcopy(ad2[0]) - # op(data2, ad1) - # assert_array_equal(data2, res2) - - for data in (ad2[0], ad2[0].data): - ad = deepcopy(ad1) - op(ad[0], data) - assert_array_equal(ad.data, res) - assert isinstance(ad, astrodata.AstroData) - assert len(ad) == 1 - assert isinstance(ad[0].data, np.ndarray) - assert isinstance(ad[0].hdr, fits.Header) - - -@pytest.mark.parametrize('op, res', [ - (operator.add, (3, 7)), - (operator.sub, (-1, 3)), - (operator.mul, (2, 10)), - (operator.truediv, (0.5, 2.5)) -]) -def test_arithmetic_multiple_ext(op, res, ad1): - ad1.append(np.ones(SHAPE, dtype=np.uint16) + 4) - - result = op(ad1, 2) - assert len(result) == 2 - assert_array_equal(result[0].data, res[0]) - assert_array_equal(result[1].data, res[1]) - - for i, ext in enumerate(ad1): - result = op(ext, 2) - assert len(result) == 1 - assert_array_equal(result[0].data, res[i]) - - -@pytest.mark.parametrize('op, res', [ - (operator.iadd, (3, 7)), - (operator.isub, (-1, 3)), - (operator.imul, (2, 10)), - (operator.itruediv, (0.5, 2.5)) -]) -def test_arithmetic_inplace_multiple_ext(op, res, ad1): - ad1.append(np.ones(SHAPE, dtype=np.uint16) + 4) - - ad = deepcopy(ad1) - result = op(ad, 2) - assert len(result) == 2 - assert_array_equal(result[0].data, res[0]) - assert_array_equal(result[1].data, res[1]) - - # Making a deepcopy will create a single nddata object but not sliced - # as it is now independant from its parent - for i, ext in enumerate(ad1): - ext = deepcopy(ext) - result = op(ext, 2) - assert len(result) == 1 - assert_array_equal(result[0].data, res[i]) - - # Now work directly on the input object, will creates single ad objects - for i, ext in enumerate(ad1): - result = op(ext, 2) - assert len(result) == 1 - assert_array_equal(result.data, res[i]) - - -@pytest.mark.parametrize('op, arg, res', [('add', 100, 101), - ('subtract', 100, -99), - ('multiply', 3, 3), - ('divide', 2, 0.5)]) -def test_operations(ad1, op, arg, res): - result = getattr(ad1, op)(arg) - assert_array_equal(result.data, res) - assert isinstance(result, astrodata.AstroData) - assert isinstance(result[0].data, np.ndarray) - assert isinstance(result[0].hdr, fits.Header) - assert len(result) == 1 - - -def test_operate(): - ad = astrodata.create({}) - nd = NDData(data=[[1, 2], [3, 4]], - uncertainty=VarianceUncertainty(np.ones((2, 2))), - mask=np.identity(2), - meta={'header': fits.Header()}) - ad.append(nd) - - ad.operate(np.sum, axis=1) - assert_array_equal(ad[0].data, [3, 7]) - assert_array_equal(ad[0].variance, [2, 2]) - assert_array_equal(ad[0].mask, [1, 1]) - - -def test_write_and_read(tmpdir, capsys): - ad = astrodata.create({}) - nd = NDData(data=[[1, 2], [3, 4]], - uncertainty=VarianceUncertainty(np.ones((2, 2))), - mask=np.identity(2), - meta={'header': fits.Header()}) - ad.append(nd) - - tbl = Table([np.zeros(10), np.ones(10)], names=('a', 'b')) - - with pytest.raises(ValueError, - match='Tables should be set directly as attribute'): - ad.append(tbl, name='BOB') - - ad.BOB = tbl - - tbl = Table([np.zeros(5) + 2, np.zeros(5) + 3], names=('c', 'd')) - - match = "Cannot append table 'BOB' because it would hide a top-level table" - with pytest.raises(ValueError, match=match): - ad[0].BOB = tbl - - ad[0].BOB2 = tbl - ad[0].MYVAL_WITH_A_VERY_LONG_NAME = np.arange(10) - - match = "You can only append NDData derived instances at the top level" - with pytest.raises(TypeError, match=match): - ad[0].MYNDD = NDData(data=np.ones(10), meta={'header': fits.Header()}) - - testfile = str(tmpdir.join('testfile.fits')) - ad.write(testfile) - - ad = astrodata.open(testfile) - ad.info() - captured = capsys.readouterr() - assert captured.out.splitlines()[3:] == [ - 'Pixels Extensions', - 'Index Content Type Dimensions Format', - '[ 0] science NDAstroData (2, 2) int64', - ' .variance ADVarianceUncerta (2, 2) float64', - ' .mask ndarray (2, 2) uint16', - ' .BOB2 Table (5, 2) n/a', - ' .MYVAL_WITH_A_VERY_LO ndarray (10,) int64', - '', - 'Other Extensions', - ' Type Dimensions', - '.BOB Table (10, 2)' - ] - assert_array_equal(ad[0].nddata.data[0], nd.data[0]) - assert_array_equal(ad[0].nddata.variance[0], nd.uncertainty.array[0]) - assert_array_equal(ad[0].nddata.mask[0], nd.mask[0]) - - -def test_reset(ad1): - data = np.ones(SHAPE) + 3 - with pytest.raises(ValueError): - ad1.reset(data) - - ext = ad1[0] - - with pytest.raises(ValueError): - ext.reset(data, mask=np.ones((2, 2)), check=True) - - with pytest.raises(ValueError): - ext.reset(data, variance=np.ones((2, 2)), check=True) - - ext.reset(data, mask=np.ones(SHAPE), variance=np.ones(SHAPE)) - assert_array_equal(ext.data, 4) - assert_array_equal(ext.variance, 1) - assert_array_equal(ext.mask, 1) - - ext.reset(data, mask=None, variance=None) - assert ext.mask is None - assert ext.uncertainty is None - - ext.reset(np.ma.array(data, mask=np.ones(SHAPE))) - assert_array_equal(ext.data, 4) - assert_array_equal(ext.mask, 1) - - with pytest.raises(TypeError): - ext.reset(data, mask=1) - - with pytest.raises(TypeError): - ext.reset(data, variance=1) - - nd = NDData(data=data, - uncertainty=VarianceUncertainty(np.ones((2, 2)) * 3), - mask=np.ones((2, 2)) * 2, meta={'header': {}}) - ext.reset(nd) - assert_array_equal(ext.data, 4) - assert_array_equal(ext.variance, 3) - assert_array_equal(ext.mask, 2) diff --git a/astrodata/tests/test_fits.py b/astrodata/tests/test_fits.py deleted file mode 100644 index 8812713e05..0000000000 --- a/astrodata/tests/test_fits.py +++ /dev/null @@ -1,989 +0,0 @@ -import copy -import os -import warnings - -import numpy as np -import pytest -from numpy.testing import assert_allclose, assert_array_equal - -import astrodata -from astrodata.utils import AstroDataDeprecationWarning -from astrodata.nddata import ADVarianceUncertainty, NDAstroData -from astrodata.testing import download_from_archive, compare_models -import astropy -from astropy import units as u -from astropy.io import fits -from astropy.table import Table -from astropy.modeling import models - -# test_files = [ -# "N20160727S0077.fits", # NIFS DARK -# "N20170529S0168.fits", # GMOS-N SPECT -# "N20190116G0054i.fits", # GRACES SPECT -# "N20190120S0287.fits", # NIRI IMAGE -# "N20190206S0279.fits", # GNIRS SPECT XD -# "S20150609S0023.fits", # GSAOI DARK -# "S20170103S0032.fits", # F2 IMAGE -# "S20170505S0031.fits", # GSAOI FLAT -# "S20170505S0095.fits", # GSAOI IMAGE -# "S20171116S0078.fits", # GMOS-S MOS NS -# "S20180223S0229.fits", # GMOS IFU ACQUISITION -# "S20190213S0084.fits", # F2 IMAGE -# ] - - -@pytest.fixture(scope='module') -def NIFS_DARK(): - """ - No. Name Ver Type Cards Dimensions Format - 0 PRIMARY 1 PrimaryHDU 144 () - 1 1 ImageHDU 108 (2048, 2048) float32 - """ - return download_from_archive("N20160727S0077.fits") - - -@pytest.fixture(scope='module') -def GMOSN_SPECT(): - """ - No. Name Ver Type Cards Dimensions Format - 0 PRIMARY 1 PrimaryHDU 180 () - 1 -1 ImageHDU 108 (288, 512) int16 (rescales to uint16) - 2 -1 ImageHDU 108 (288, 512) int16 (rescales to uint16) - 3 -1 ImageHDU 108 (288, 512) int16 (rescales to uint16) - 4 -1 ImageHDU 108 (288, 512) int16 (rescales to uint16) - 5 -1 ImageHDU 72 (288, 512) int16 (rescales to uint16) - 6 -1 ImageHDU 72 (288, 512) int16 (rescales to uint16) - 7 -1 ImageHDU 72 (288, 512) int16 (rescales to uint16) - 8 -1 ImageHDU 72 (288, 512) int16 (rescales to uint16) - 9 -1 ImageHDU 38 (288, 512) int16 (rescales to uint16) - 10 -1 ImageHDU 38 (288, 512) int16 (rescales to uint16) - 11 -1 ImageHDU 38 (288, 512) int16 (rescales to uint16) - 12 -1 ImageHDU 38 (288, 512) int16 (rescales to uint16) - """ - return download_from_archive("N20170529S0168.fits") - - -@pytest.fixture(scope='module') -def GSAOI_DARK(): - """ - No. Name Ver Type Cards Dimensions Format - 0 PRIMARY 1 PrimaryHDU 289 () - 1 1 ImageHDU 144 (2048, 2048) float32 - 2 2 ImageHDU 144 (2048, 2048) float32 - 3 3 ImageHDU 144 (2048, 2048) float32 - 4 4 ImageHDU 144 (2048, 2048) float32 - """ - return download_from_archive("S20150609S0023.fits") - - -@pytest.fixture(scope='module') -def GRACES_SPECT(): - """ - No. Name Ver Type Cards Dimensions Format - 0 PRIMARY 1 PrimaryHDU 183 (190747, 28) float32 - """ - return download_from_archive("N20190116G0054i.fits") - - -def test_extver(tmp_path): - """Test that EXTVER is written sequentially for new extensions, - and preserved with slicing. - """ - testfile = tmp_path / 'test.fits' - - ad = astrodata.create({}) - for _ in range(10): - ad.append(np.zeros((4, 5))) - ad.write(testfile) - - ad = astrodata.open(testfile) - ext = ad[2] - assert ext.hdr['EXTNAME'] == 'SCI' - assert ext.hdr['EXTVER'] == 3 - - ext = ad[4] - assert ext.hdr['EXTNAME'] == 'SCI' - assert ext.hdr['EXTVER'] == 5 - - ext = ad[:8][4] - assert ext.hdr['EXTNAME'] == 'SCI' - assert ext.hdr['EXTVER'] == 5 - - -def test_extver2(tmp_path): - """Test renumbering of EXTVER.""" - testfile = tmp_path / 'test.fits' - - ad = astrodata.create(fits.PrimaryHDU()) - data = np.arange(5) - ad.append(fits.ImageHDU(data=data, header=fits.Header({'EXTVER': 2}))) - ad.append(fits.ImageHDU(data=data + 2, header=fits.Header({'EXTVER': 5}))) - ad.append(fits.ImageHDU(data=data + 5)) - ad.append(fits.ImageHDU(data=data + 7, header=fits.Header({'EXTVER': 3}))) - ad.write(testfile) - - ad = astrodata.open(testfile) - assert [hdr['EXTVER'] for hdr in ad.hdr] == [1, 2, 3, 4] - - -def test_extver3(tmp_path, GSAOI_DARK): - """Test that original EXTVER are preserved and extensions added - from another object are renumbered. - """ - testfile = tmp_path / 'test.fits' - - ad1 = astrodata.open(GSAOI_DARK) - ad2 = astrodata.open(GSAOI_DARK) - - del ad1[2] - ad1.append(ad2[2]) - ad1.append(np.zeros(10)) - - ad1.write(testfile) - - ad = astrodata.open(testfile) - assert [hdr['EXTVER'] for hdr in ad.hdr] == [1, 2, 4, 5, 6] - - -@pytest.mark.dragons_remote_data -def test_can_add_and_del_extension(GMOSN_SPECT): - ad = astrodata.open(GMOSN_SPECT) - original_size = len(ad) - - ourarray = np.array([(1, 2, 3), (11, 12, 13), (21, 22, 23)]) - ad.append(ourarray) - assert len(ad) == original_size + 1 - - del ad[original_size] - assert len(ad) == original_size - - -@pytest.mark.dragons_remote_data -def test_slice(GMOSN_SPECT): - ad = astrodata.open(GMOSN_SPECT) - assert ad.is_sliced is False - - n_ext = len(ad) - with pytest.raises(IndexError, match="Index out of range"): - ad[n_ext + 1] - - with pytest.raises(ValueError, match='Invalid index: FOO'): - ad['FOO'] - - # single - metadata = ('SCI', 2) - ext = ad[1] - assert ext.id == 2 - assert ext.is_single is True - assert ext.is_sliced is True - assert ext.hdr['EXTNAME'] == metadata[0] - assert ext.hdr['EXTVER'] == metadata[1] - assert not ext.is_settable('filename') - assert ext.data[0, 0] == 387 - - # when astrofaker is imported this will be recognized as AstroFakerGmos - # instead of AstroData - match = r"'Astro.*' object has no attribute 'FOO'" - with pytest.raises(AttributeError, match=match): - ext.FOO - - # setting uppercase attr adds to the extension: - ext.FOO = 1 - assert ext.FOO == 1 - assert ext.exposed == {'FOO'} - assert ext.nddata.meta['other']['FOO'] == 1 - del ext.FOO - - with pytest.raises(AttributeError): - del ext.BAR - - match = "Can't append objects to slices, use 'ext.NAME = obj' instead" - with pytest.raises(TypeError, match=match): - ext.append(np.zeros(5)) - - # but lowercase just adds a normal attribute to the object - ext.bar = 1 - assert ext.bar == 1 - assert 'bar' not in ext.nddata.meta['other'] - del ext.bar - - with pytest.raises(TypeError, match="Can't slice a single slice!"): - ext[1] - - -@pytest.mark.dragons_remote_data -def test_slice_single_element(GMOSN_SPECT): - ad = astrodata.open(GMOSN_SPECT) - assert ad.is_sliced is False - - metadata = ('SCI', 2) - - ext = ad[1:2] - assert ext.is_single is False - assert ext.is_sliced is True - assert ext.indices == [1] - assert isinstance(ext.data, list) and len(ext.data) == 1 - - ext = ext[0] - assert ext.id == 2 - assert ext.is_single is True - assert ext.is_sliced is True - assert ext.hdr['EXTNAME'] == metadata[0] - assert ext.hdr['EXTVER'] == metadata[1] - - -@pytest.mark.dragons_remote_data -def test_slice_multiple(GMOSN_SPECT): - ad = astrodata.open(GMOSN_SPECT) - - metadata = ('SCI', 2), ('SCI', 3) - slc = ad[1, 2] - assert len(slc) == 2 - assert slc.is_sliced is True - assert len(slc.data) == 2 - assert slc.data[0][0, 0] == 387 - assert slc.data[1][0, 0] == 383 - assert slc.shape == [(512, 288), (512, 288)] - - for ext, md in zip(slc, metadata): - assert (ext.hdr['EXTNAME'], ext.hdr['EXTVER']) == md - - with pytest.raises(ValueError, match="Cannot return id"): - slc.id - - assert slc[0].id == 2 - assert slc[1].id == 3 - - match = "Can't remove items from a sliced object" - with pytest.raises(TypeError, match=match): - del slc[0] - - match = "Can't append objects to slices, use 'ext.NAME = obj' instead" - with pytest.raises(TypeError, match=match): - slc.append(np.zeros(5), name='ARR') - - match = "This attribute can only be assigned to a single-slice object" - with pytest.raises(TypeError, match=match): - slc.ARR = np.zeros(5) - - # iterate over single slice - metadata = ('SCI', 1) - for ext in ad[0]: - assert (ext.hdr['EXTNAME'], ext.hdr['EXTVER']) == metadata - - # slice negative - assert ad.data[-1] is ad[-1].data - - match = "This attribute can only be assigned to a single-slice object" - with pytest.raises(TypeError, match=match): - slc.FOO = 1 - - with pytest.raises(TypeError, - match="Can't delete attributes on non-single slices"): - del slc.FOO - - ext.bar = 1 - assert ext.bar == 1 - del ext.bar - - -@pytest.mark.dragons_remote_data -def test_slice_data(GMOSN_SPECT): - ad = astrodata.open(GMOSN_SPECT) - - slc = ad[1, 2] - match = "Trying to assign to an AstroData object that is not a single slice" - with pytest.raises(ValueError, match=match): - slc.data = 1 - with pytest.raises(ValueError, match=match): - slc.uncertainty = 1 - with pytest.raises(ValueError, match=match): - slc.mask = 1 - with pytest.raises(ValueError, match=match): - slc.variance = 1 - - assert slc.uncertainty == [None, None] - assert slc.mask == [None, None] - - ext = ad[1] - match = "Trying to assign data to be something with no shape" - with pytest.raises(AttributeError, match=match): - ext.data = 1 - - # set/get on single slice - ext.data = np.ones(10) - assert_array_equal(ext.data, 1) - - ext.variance = np.ones(10) - assert_array_equal(ext.variance, 1) - ext.variance = None - assert ext.variance is None - - ext.uncertainty = ADVarianceUncertainty(np.ones(10)) - assert_array_equal(ext.variance, 1) - assert_array_equal(slc.variance[0], 1) - - ext.mask = np.zeros(10) - assert_array_equal(ext.mask, 0) - assert_array_equal(slc.mask[0], 0) - - assert slc.nddata[0].data is ext.data - assert slc.nddata[0].uncertainty is ext.uncertainty - assert slc.nddata[0].mask is ext.mask - - -@pytest.mark.dragons_remote_data -def test_phu(NIFS_DARK): - ad = astrodata.open(NIFS_DARK) - - # The result of this depends if gemini_instruments was imported or not - # assert ad.descriptors == ('instrument', 'object', 'telescope') - # assert ad.tags == set() - - assert ad.instrument() == 'NIFS' - assert ad.object() == 'Dark' - assert ad.telescope() == 'Gemini-North' - - ad.phu['DETECTOR'] = 'FooBar' - ad.phu['ARBTRARY'] = 'BarBaz' - - assert ad.phu['DETECTOR'] == 'FooBar' - assert ad.phu['ARBTRARY'] == 'BarBaz' - - if ad.instrument().upper() not in ['GNIRS', 'NIRI', 'F2']: - del ad.phu['DETECTOR'] - assert 'DETECTOR' not in ad.phu - - -@pytest.mark.dragons_remote_data -def test_paths(tmpdir, NIFS_DARK): - ad = astrodata.open(NIFS_DARK) - assert ad.orig_filename == 'N20160727S0077.fits' - - srcdir = os.path.dirname(NIFS_DARK) - assert ad.filename == 'N20160727S0077.fits' - assert ad.path == os.path.join(srcdir, 'N20160727S0077.fits') - - ad.filename = 'newfile.fits' - assert ad.filename == 'newfile.fits' - assert ad.path == os.path.join(srcdir, 'newfile.fits') - - testfile = os.path.join(str(tmpdir), 'temp.fits') - ad.path = testfile - assert ad.filename == 'temp.fits' - assert ad.path == testfile - assert ad.orig_filename == 'N20160727S0077.fits' - ad.write() - assert os.path.exists(testfile) - os.remove(testfile) - - testfile = os.path.join(str(tmpdir), 'temp2.fits') - ad.write(testfile) - assert os.path.exists(testfile) - - # overwriting is forbidden by default - with pytest.raises(OSError): - ad.write(testfile) - - ad.write(testfile, overwrite=True) - assert os.path.exists(testfile) - os.remove(testfile) - - ad.path = None - assert ad.filename is None - with pytest.raises(ValueError): - ad.write() - - -@pytest.mark.dragons_remote_data -def test_from_hdulist(NIFS_DARK): - with fits.open(NIFS_DARK) as hdul: - assert 'ORIGNAME' not in hdul[0].header - ad = astrodata.open(hdul) - assert ad.path is None - assert ad.instrument() == 'NIFS' - assert ad.object() == 'Dark' - assert ad.telescope() == 'Gemini-North' - assert len(ad) == 1 - assert ad[0].shape == (2048, 2048) - - with fits.open(NIFS_DARK) as hdul: - # Make sure that when ORIGNAME is set, astrodata use it - hdul[0].header['ORIGNAME'] = 'N20160727S0077.fits' - ad = astrodata.open(hdul) - assert ad.path == 'N20160727S0077.fits' - - -def test_from_hdulist2(): - tablehdu = fits.table_to_hdu(Table([[1]])) - tablehdu.name = 'REFCAT' - - hdul = fits.HDUList([ - fits.PrimaryHDU(header=fits.Header({'INSTRUME': 'FISH'})), - fits.ImageHDU(data=np.zeros(10), name='SCI', ver=1), - fits.ImageHDU(data=np.ones(10), name='VAR', ver=1), - fits.ImageHDU(data=np.zeros(10, dtype='uint16'), name='DQ', ver=1), - tablehdu, - fits.BinTableHDU.from_columns( - [fits.Column(array=['a', 'b'], format='A', name='col')], ver=1, - ), # This HDU will be skipped because it has no EXTNAME - ]) - - with pytest.warns(UserWarning, - match='Skip HDU .* because it has no EXTNAME'): - ad = astrodata.open(hdul) - - assert len(ad) == 1 - assert ad.phu['INSTRUME'] == 'FISH' - assert_array_equal(ad[0].data, 0) - assert_array_equal(ad[0].variance, 1) - assert_array_equal(ad[0].mask, 0) - assert len(ad.REFCAT) == 1 - assert ad.exposed == {'REFCAT'} - assert ad[0].exposed == {'REFCAT'} - - -def test_from_hdulist3(): - hdul = fits.HDUList([ - fits.PrimaryHDU(), - fits.ImageHDU(data=np.zeros(10), name='SCI', ver=1), - fits.TableHDU.from_columns( - [fits.Column(array=['a', 'b'], format='A', name='col')], - name='ASCIITAB', - ), - ]) - - ad = astrodata.open(hdul) - - assert hasattr(ad, 'ASCIITAB') - assert len(ad.ASCIITAB) == 2 - - -def test_can_make_and_write_ad_object(tmpdir): - # Creates data and ad object - phu = fits.PrimaryHDU() - hdu = fits.ImageHDU(data=np.arange(10)) - ad = astrodata.create(phu) - ad.append(hdu, name='SCI') - - hdr = fits.Header({'EXTNAME': 'SCI', 'EXTVER': 1, 'FOO': 'BAR'}) - ad.append(hdu, header=hdr) - assert ad[1].hdr['FOO'] == 'BAR' - - # Write file and test it exists properly - testfile = str(tmpdir.join('created_fits_file.fits')) - ad.write(testfile) - - # Opens file again and tests data is same as above - adnew = astrodata.open(testfile) - assert np.array_equal(adnew[0].data, np.arange(10)) - - -def test_can_append_table_and_access_data(capsys, tmpdir): - tbl = Table([np.zeros(10), np.ones(10)], names=['col1', 'col2']) - phu = fits.PrimaryHDU() - ad = astrodata.create(phu) - - with pytest.raises(ValueError, - match='Tables should be set directly as attribute'): - ad.append(tbl, name='BOB') - - ad.BOB = tbl - assert ad.exposed == {'BOB'} - - assert ad.tables == {'BOB'} - assert np.all(ad.table()['BOB'] == tbl) - - ad.info() - captured = capsys.readouterr() - assert '.BOB Table (10, 2)' in captured.out - - # Write file and test it exists properly - testfile = str(tmpdir.join('created_fits_file.fits')) - ad.write(testfile) - adnew = astrodata.open(testfile) - assert adnew.exposed == {'BOB'} - assert len(adnew.BOB) == 10 - - del ad.BOB - assert ad.tables == set() - with pytest.raises(AttributeError): - del ad.BOB - - -@pytest.mark.dragons_remote_data -def test_attributes(GSAOI_DARK): - ad = astrodata.open(GSAOI_DARK) - assert ad.shape == [(2048, 2048)] * 4 - assert [arr.shape for arr in ad.data] == [(2048, 2048)] * 4 - assert [arr.dtype for arr in ad.data] == ['f'] * 4 - assert ad.uncertainty == [None] * 4 - assert ad.variance == [None] * 4 - assert ad.mask == [None] * 4 - - ad[0].variance = np.ones(ad[0].shape) - assert isinstance(ad[0].uncertainty, ADVarianceUncertainty) - assert_array_equal(ad[0].uncertainty.array, 1) - assert_array_equal(ad[0].variance, 1) - assert_array_equal(ad.variance[0], 1) - - assert all(isinstance(nd, NDAstroData) for nd in ad.nddata) - assert [nd.shape for nd in ad.nddata] == [(2048, 2048)] * 4 - - match = "Trying to assign to an AstroData object that is not a single slice" - with pytest.raises(ValueError, match=match): - ad.data = 1 - with pytest.raises(ValueError, match=match): - ad.variance = 1 - with pytest.raises(ValueError, match=match): - ad.uncertainty = 1 - with pytest.raises(ValueError, match=match): - ad.mask = 1 - - -@pytest.mark.dragons_remote_data -def test_set_a_keyword_on_phu_deprecated(NIFS_DARK): - ad = astrodata.open(NIFS_DARK) - # Test that setting DETECTOR as an attribute doesn't modify the header - ad.phu.DETECTOR = 'FooBar' - assert ad.phu.DETECTOR == 'FooBar' - assert ad.phu['DETECTOR'] == 'NIFS' - - -# Regression: -# Make sure that references to associated -# extension objects are copied across -@pytest.mark.dragons_remote_data -def test_do_arith_and_retain_features(NIFS_DARK): - ad = astrodata.open(NIFS_DARK) - ad[0].NEW_FEATURE = np.array([1, 2, 3, 4, 5]) - ad2 = ad * 5 - assert_array_equal(ad[0].NEW_FEATURE, ad2[0].NEW_FEATURE) - - -def test_update_filename(): - phu = fits.PrimaryHDU() - ad = astrodata.create(phu) - - ad.filename = 'myfile.fits' - - # This will also set ORIGNAME='myfile.fits' - ad.update_filename(suffix='_suffix1') - assert ad.filename == 'myfile_suffix1.fits' - - ad.update_filename(suffix='_suffix2', strip=True) - assert ad.filename == 'myfile_suffix2.fits' - - ad.update_filename(suffix='_suffix1', strip=False) - assert ad.filename == 'myfile_suffix2_suffix1.fits' - - ad.filename = 'myfile.fits' - ad.update_filename(prefix='prefix_', strip=True) - assert ad.filename == 'prefix_myfile.fits' - - ad.update_filename(suffix='_suffix', strip=True) - assert ad.filename == 'prefix_myfile_suffix.fits' - - ad.update_filename(prefix='', suffix='_suffix2', strip=True) - assert ad.filename == 'myfile_suffix2.fits' - - # Now check that updates are based on existing filename - # (so "myfile" shouldn't appear) - ad.filename = 'file_suffix1.fits' - ad.update_filename(suffix='_suffix2') - assert ad.filename == 'file_suffix1_suffix2.fits' - - # A suffix shouldn't have an underscore, so should assume that - # "file_suffix1" is the root - ad.update_filename(suffix='_suffix3', strip=True) - assert ad.filename == 'file_suffix1_suffix3.fits' - - -def test_update_filename2(): - phu = fits.PrimaryHDU() - ad = astrodata.create(phu) - - with pytest.raises(ValueError): - # Not possible when ad.filename is None - ad.update_filename(suffix='_suffix1') - - # filename is taken from ORIGNAME by default - ad.phu['ORIGNAME'] = 'origfile.fits' - ad.update_filename(suffix='_suffix') - assert ad.filename == 'origfile_suffix.fits' - - ad.phu['ORIGNAME'] = 'temp.fits' - ad.filename = 'origfile.fits' - ad.update_filename(suffix='_bar', strip=True) - assert ad.filename == 'origfile_bar.fits' - - -@pytest.mark.dragons_remote_data -def test_read_a_keyword_from_phu_deprecated(): - """Test deprecated methods to access headers""" - ad = astrodata.open(download_from_archive('N20110826S0336.fits')) - - with pytest.raises(AttributeError): - assert ad.phu.DETECTOR == 'GMOS + Red1' - - with pytest.raises(AttributeError): - assert ad.hdr.CCDNAME == [ - 'EEV 9273-16-03', 'EEV 9273-20-04', 'EEV 9273-20-03' - ] - - # and when accessing missing extension - with pytest.raises(AttributeError): - ad.ABC - - -def test_read_invalid_file(tmpdir, caplog): - testfile = str(tmpdir.join('test.fits')) - with open(testfile, 'w'): - # create empty file - pass - - with pytest.raises(astrodata.AstroDataError): - astrodata.open(testfile) - - assert caplog.records[0].message.endswith('is zero size') - - -def test_read_empty_file(tmpdir): - testfile = str(tmpdir.join('test.fits')) - hdr = fits.Header({'INSTRUME': 'darkimager', 'OBJECT': 'M42'}) - fits.PrimaryHDU(header=hdr).writeto(testfile) - ad = astrodata.open(testfile) - assert len(ad) == 0 - assert ad.object() == 'M42' - assert ad.instrument() == 'darkimager' - - -def test_read_file(tmpdir): - testfile = str(tmpdir.join('test.fits')) - hdr = fits.Header({'INSTRUME': 'darkimager', 'OBJECT': 'M42'}) - fits.PrimaryHDU(header=hdr).writeto(testfile) - ad = astrodata.open(testfile) - assert len(ad) == 0 - assert ad.object() == 'M42' - assert ad.instrument() == 'darkimager' - - -@pytest.mark.dragons_remote_data -def test_header_collection(GMOSN_SPECT): - ad = astrodata.create({}) - assert ad.hdr is None - - ad = astrodata.open(GMOSN_SPECT) - assert len(ad) == 12 - assert len([hdr for hdr in ad.hdr]) == 12 - - # get - assert 'FRAMEID' in ad.hdr - assert 'FOO' not in ad.hdr - assert ad.hdr.get('FRAMEID') == [ - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11' - ] - with pytest.raises(KeyError): - ad.hdr['FOO'] - assert ad.hdr.get('FOO') == [None] * 12 - assert ad.hdr.get('FOO', default='BAR') == ['BAR'] * 12 - - # del/remove - assert ad.hdr['GAIN'] == [1.0] * 12 - del ad.hdr['GAIN'] - with pytest.raises(KeyError): - ad.hdr['GAIN'] - with pytest.raises(KeyError): - del ad.hdr['GAIN'] - - # set - assert ad.hdr['RDNOISE'] == [1.0] * 12 - ad.hdr['RDNOISE'] = 2.0 - assert ad.hdr['RDNOISE'] == [2.0] * 12 - - # comment - assert ad.hdr.get_comment('DATATYPE') == ['Type of Data'] * 12 - ad.hdr.set_comment('DATATYPE', 'Hello!') - assert ad.hdr.get_comment('DATATYPE') == ['Hello!'] * 12 - ad.hdr['RDNOISE'] = (2.0, 'New comment') - assert ad.hdr.get_comment('RDNOISE') == ['New comment'] * 12 - with pytest.raises(KeyError, - match="Keyword 'FOO' not available at header 0"): - ad.hdr.set_comment('FOO', 'A comment') - - ad = astrodata.open(GMOSN_SPECT) - hdr = ad.hdr - assert len(list(hdr)) == 12 - hdr._insert(1, fits.Header({'INSTRUME': 'darkimager', 'OBJECT': 'M42'})) - assert len(list(hdr)) == 13 - - -@pytest.mark.dragons_remote_data -def test_header_deprecated(GMOSN_SPECT): - ad = astrodata.open(GMOSN_SPECT) - with pytest.warns(AstroDataDeprecationWarning): - warnings.simplefilter('always', AstroDataDeprecationWarning) - header = ad.header - assert header[0]['ORIGNAME'] == 'N20170529S0168.fits' - assert header[1]['EXTNAME'] == 'SCI' - assert header[1]['EXTVER'] == 1 - - with pytest.warns(AstroDataDeprecationWarning): - warnings.simplefilter('always', AstroDataDeprecationWarning) - header = ad[0].header - assert header[0]['ORIGNAME'] == 'N20170529S0168.fits' - - -@pytest.mark.dragons_remote_data -def test_read_no_extensions(GRACES_SPECT): - ad = astrodata.open(GRACES_SPECT) - assert len(ad) == 1 - # header is duplicated for .phu and extension's header - assert len(ad.phu) == 181 - assert len(ad[0].hdr) == 185 - assert ad[0].hdr['EXTNAME'] == 'SCI' - assert ad[0].hdr['EXTVER'] == 1 - - -def test_add_var_and_dq(): - shape = (3, 4) - fakedata = np.arange(np.prod(shape)).reshape(shape) - - ad = astrodata.create({'OBJECT': 'M42'}) - ad.append(fakedata) - assert ad[0].hdr['EXTNAME'] == 'SCI' # default value for EXTNAME - - with pytest.raises(ValueError, match="Only one Primary HDU allowed"): - ad.append(fits.PrimaryHDU(data=fakedata), name='FOO') - - with pytest.raises(ValueError, - match="Arbitrary image extensions can " - "only be added in association to a 'SCI'"): - ad.append(np.zeros(shape), name='FOO') - - with pytest.raises(ValueError, - match="'VAR' need to be associated to a 'SCI' one"): - ad.append(np.ones(shape), name='VAR') - - with pytest.raises(AttributeError, - match="SCI extensions should be appended with .append"): - ad[0].SCI = np.ones(shape) - - -def test_add_table(): - shape = (3, 4) - fakedata = np.arange(np.prod(shape)).reshape(shape) - - ad = astrodata.create({'OBJECT': 'M42'}) - ad.append(fakedata) - - ad.TABLE1 = Table([['a', 'b', 'c'], [1, 2, 3]]) - assert ad.tables == {'TABLE1'} - - ad.TABLE2 = Table([['a', 'b', 'c'], [1, 2, 3]]) - assert ad.tables == {'TABLE1', 'TABLE2'} - - ad.MYTABLE = Table([['a', 'b', 'c'], [1, 2, 3]]) - assert ad.tables == {'TABLE1', 'TABLE2', 'MYTABLE'} - - ad[0].TABLE3 = Table([['aa', 'bb', 'cc'], [1, 2, 3]]) - ad[0].TABLE4 = Table([['aa', 'bb', 'cc'], [1, 2, 3]]) - ad[0].OTHERTABLE = Table([['aa', 'bb', 'cc'], [1, 2, 3]]) - - assert list(ad[0].OTHERTABLE['col0']) == ['aa', 'bb', 'cc'] - - assert ad.tables == {'TABLE1', 'TABLE2', 'MYTABLE'} - assert ad[0].tables == {'TABLE1', 'TABLE2', 'MYTABLE'} - assert ad[0].ext_tables == {'OTHERTABLE', 'TABLE3', 'TABLE4'} - assert ad[0].exposed == {'MYTABLE', 'OTHERTABLE', 'TABLE1', 'TABLE2', - 'TABLE3', 'TABLE4'} - - with pytest.raises(AttributeError): - ad.ext_tables - - assert set(ad[0].nddata.meta['other'].keys()) == {'OTHERTABLE', - 'TABLE3', 'TABLE4'} - assert_array_equal(ad[0].TABLE3['col0'], ['aa', 'bb', 'cc']) - assert_array_equal(ad[0].TABLE4['col0'], ['aa', 'bb', 'cc']) - assert_array_equal(ad[0].OTHERTABLE['col0'], ['aa', 'bb', 'cc']) - - -@pytest.mark.dragons_remote_data -def test_copy(GSAOI_DARK, capsys): - ad = astrodata.open(GSAOI_DARK) - ad.TABLE = Table([['a', 'b', 'c'], [1, 2, 3]]) - ad[0].MYTABLE = Table([['aa', 'bb', 'cc'], [1, 2, 3]]) - - ad.info() - captured = capsys.readouterr() - - ad2 = copy.deepcopy(ad) - ad2.info() - captured2 = capsys.readouterr() - - # Compare that objects have the same attributes etc. with their - # .info representation - assert captured.out == captured2.out - - ext = ad[0] - ext.info() - captured = capsys.readouterr() - - ext2 = copy.deepcopy(ext) - ext2.info() - captured2 = capsys.readouterr() - - # Same for extension, except that first line is different (no - # filename in the copied ext) - assert captured.out.splitlines()[1:] == captured2.out.splitlines()[1:] - - -@pytest.mark.dragons_remote_data -def test_crop(GSAOI_DARK): - ad = astrodata.open(GSAOI_DARK) - assert set(ad.shape) == {(2048, 2048)} - - ad.crop(0, 0, 5, 10) - assert len(ad.nddata) == 4 - assert set(ad.shape) == {(11, 6)} - - -@pytest.mark.dragons_remote_data -def test_crop_ext(GSAOI_DARK): - ad = astrodata.open(GSAOI_DARK) - ext = ad[0] - ext.uncertainty = ADVarianceUncertainty(np.ones(ext.shape)) - ext.mask = np.ones(ext.shape, dtype=np.uint8) - - # FIXME: cannot test cropping attached array because that does not work - # with numpy arrays, and can only attach NDData instance at the top level - # ext.append(np.zeros(ext.shape, dtype=int), name='FOO') - - ext.BAR = 1 - - ext.crop(0, 0, 5, 10) - assert ext.shape == (11, 6) - assert_allclose(ext.data[0], [-1.75, -0.75, -4.75, 2.375, -0.25, 1.375]) - assert_array_equal(ext.uncertainty.array, 1) - assert_array_equal(ext.mask, 1) - - # assert ext.FOO.shape == (11, 6) - # assert_array_equal(ext.FOO, 0) - assert ext.BAR == 1 - - -@pytest.mark.xfail(not astropy.utils.minversion(astropy, '4.0.1'), - reason='requires astropy >=4.0.1 for correct serialization') -def test_round_trip_gwcs(tmpdir): - """ - Add a 2-step gWCS instance to NDAstroData, save to disk, reload & compare. - """ - - from gwcs import coordinate_frames as cf - from gwcs import WCS - - arr = np.zeros((10, 10), dtype=np.float32) - ad1 = astrodata.create(fits.PrimaryHDU(), [fits.ImageHDU(arr, name='SCI')]) - - # Transformation from detector pixels to pixels in some reference row, - # removing relative distortions in wavelength: - det_frame = cf.Frame2D(name='det_mosaic', axes_names=('x', 'y'), - unit=(u.pix, u.pix)) - dref_frame = cf.Frame2D(name='dist_ref_row', axes_names=('xref', 'y'), - unit=(u.pix, u.pix)) - - # A made-up example model that looks vaguely like some real distortions: - fdist = models.Chebyshev2D(2, 2, - c0_0=4.81125, c1_0=5.43375, c0_1=-0.135, - c1_1=-0.405, c0_2=0.30375, c1_2=0.91125, - x_domain=[0., 9.], y_domain=[0., 9.]) - - # This is not an accurate inverse, but will do for this test: - idist = models.Chebyshev2D(2, 2, - c0_0=4.89062675, c1_0=5.68581232, - c2_0=-0.00590263, c0_1=0.11755526, - c1_1=0.35652358, c2_1=-0.01193828, - c0_2=-0.29996306, c1_2=-0.91823397, - c2_2=0.02390594, - x_domain=[-1.5, 12.], y_domain=[0., 9.]) - - # The resulting 2D co-ordinate mapping from detector to ref row pixels: - distrans = models.Mapping((0, 1, 1)) | (fdist & models.Identity(1)) - distrans.inverse = models.Mapping((0, 1, 1)) | (idist & models.Identity(1)) - - # Transformation from reference row pixels to linear, row-stacked spectra: - spec_frame = cf.SpectralFrame(axes_order=(0,), unit=u.nm, - axes_names='lambda', name='wavelength') - row_frame = cf.CoordinateFrame(1, 'SPATIAL', axes_order=(1,), unit=u.pix, - axes_names='y', name='row') - rss_frame = cf.CompositeFrame([spec_frame, row_frame]) - - # Toy wavelength model & approximate inverse: - fwcal = models.Chebyshev1D(2, c0=500.075, c1=0.05, c2=0.001, domain=[0, 9]) - iwcal = models.Chebyshev1D(2, c0=4.59006292, c1=4.49601817, c2=-0.08989608, - domain=[500.026, 500.126]) - - # The resulting 2D co-ordinate mapping from ref pixels to wavelength: - wavtrans = fwcal & models.Identity(1) - wavtrans.inverse = iwcal & models.Identity(1) - - # The complete WCS chain for these 2 transformation steps: - ad1[0].nddata.wcs = WCS([(det_frame, distrans), - (dref_frame, wavtrans), - (rss_frame, None) - ]) - - # Save & re-load the AstroData instance with its new WCS attribute: - testfile = str(tmpdir.join('round_trip_gwcs.fits')) - ad1.write(testfile) - ad2 = astrodata.open(testfile) - - wcs1 = ad1[0].nddata.wcs - wcs2 = ad2[0].nddata.wcs - - # # Temporary workaround for issue #9809, to ensure the test is correct: - # wcs2.forward_transform[1].x_domain = (0, 9) - # wcs2.forward_transform[1].y_domain = (0, 9) - # wcs2.forward_transform[3].domain = (0, 9) - # wcs2.backward_transform[0].domain = (500.026, 500.126) - # wcs2.backward_transform[3].x_domain = (-1.5, 12.) - # wcs2.backward_transform[3].y_domain = (0, 9) - - # Did we actually get a gWCS instance back? - assert isinstance(wcs2, WCS) - - # Do the transforms have the same number of submodels, with the same types, - # degrees, domains & parameters? Here the inverse gets checked redundantly - # as both backward_transform and forward_transform.inverse, but it would be - # convoluted to ensure that both are correct otherwise (since the transforms - # get regenerated as new compound models each time they are accessed). - compare_models(wcs1.forward_transform, wcs2.forward_transform) - compare_models(wcs1.backward_transform, wcs2.backward_transform) - - # Do the instances have matching co-ordinate frames? - for f in wcs1.available_frames: - assert repr(getattr(wcs1, f)) == repr(getattr(wcs2, f)) - - # Also compare a few transformed values, as the "proof of the pudding": - y, x = np.mgrid[0:9:2, 0:9:2] - np.testing.assert_allclose(wcs1(x, y), wcs2(x, y), rtol=1e-7, atol=0.) - - y, w = np.mgrid[0:9:2, 500.025:500.12:0.0225] - np.testing.assert_allclose(wcs1.invert(w, y), wcs2.invert(w, y), - rtol=1e-7, atol=0.) - - -@pytest.mark.parametrize('dtype', ['int8', 'uint8', 'int16', 'uint16', - 'int32', 'uint32', 'int64', 'uint64']) -def test_uint_data(dtype, tmp_path): - testfile = tmp_path / 'test.fits' - data = np.arange(10, dtype=np.int16) - fits.writeto(testfile, data) - - ad = astrodata.open(str(testfile)) - assert ad[0].data.dtype == data.dtype - assert_array_equal(ad[0].data, data) - - -if __name__ == '__main__': - pytest.main() diff --git a/astrodata/tests/test_nddata.py b/astrodata/tests/test_nddata.py deleted file mode 100644 index 3562b3dfcf..0000000000 --- a/astrodata/tests/test_nddata.py +++ /dev/null @@ -1,166 +0,0 @@ -import warnings - -import numpy as np -import pytest -from numpy.testing import assert_array_almost_equal, assert_array_equal - -from astrodata.fits import windowedOp -from astrodata import wcs as adwcs -from astrodata.nddata import ADVarianceUncertainty, NDAstroData -from astropy.io import fits -from astropy.nddata import NDData, VarianceUncertainty -from astropy.modeling import models -from astropy.table import Table -from gwcs.wcs import WCS as gWCS -from gwcs.coordinate_frames import Frame2D - - -@pytest.fixture -def testnd(): - shape = (5, 5) - hdr = fits.Header({'CRPIX1': 1, 'CRPIX2': 2}) - nd = NDAstroData(data=np.arange(np.prod(shape)).reshape(shape), - variance=np.ones(shape) + 0.5, - mask=np.zeros(shape, dtype=bool), - wcs=gWCS(models.Shift(1) & models.Shift(2), - input_frame=adwcs.pixel_frame(2), - output_frame=adwcs.pixel_frame(2, name='world')), - unit='ct') - nd.meta['other'] = {'OBJMASK': np.arange(np.prod(shape)).reshape(shape), - 'OBJCAT': Table([[1,2,3]], names=[['number']])} - nd.mask[3, 4] = True - return nd - - -def test_var(testnd): - data = np.zeros(5) - var = np.array([1.2, 2, 1.5, 1, 1.3]) - nd1 = NDAstroData(data=data, uncertainty=ADVarianceUncertainty(var)) - nd2 = NDAstroData(data=data, variance=var) - assert_array_equal(nd1.variance, nd2.variance) - - -def test_window(testnd): - win = testnd.window[2:4, 3:5] - assert win.unit == 'ct' - #assert_array_equal(win.wcs.wcs.crpix, [1, 2]) - assert_array_equal(win.data, [[13, 14], [18, 19]]) - assert_array_equal(win.mask, [[False, False], [False, True]]) - assert_array_almost_equal(win.uncertainty.array, 1.5) - assert_array_almost_equal(win.variance, 1.5) - - -def test_windowedOp(testnd): - - def stack(arrays): - arrays = [x for x in arrays] - data = np.array([arr.data for arr in arrays]).sum(axis=0) - unc = np.array([arr.uncertainty.array for arr in arrays]).sum(axis=0) - mask = np.array([arr.mask for arr in arrays]).sum(axis=0) - return NDAstroData(data=data, variance=unc, mask=mask) - - result = windowedOp(stack, [testnd, testnd], - kernel=(3, 3), - with_uncertainty=True, - with_mask=True) - assert_array_equal(result.data, testnd.data * 2) - assert_array_equal(result.uncertainty.array, testnd.uncertainty.array * 2) - assert result.mask[3, 4] == 2 - - nd2 = NDAstroData(data=np.zeros((4, 4))) - with pytest.raises(ValueError, match=r"Can't calculate final shape.*"): - result = windowedOp(stack, [testnd, nd2], kernel=(3, 3)) - - with pytest.raises(AssertionError, match=r"Incompatible shape.*"): - result = windowedOp(stack, [testnd, testnd], kernel=[3], shape=(5, 5)) - - -def test_transpose(testnd): - testnd.variance[0, -1] = 10 - ndt = testnd.T - assert_array_equal(ndt.data[0], [0, 5, 10, 15, 20]) - assert ndt.variance[-1, 0] == 10 - assert ndt.wcs(1, 2) == testnd.wcs(2, 1) - - -def test_set_section(testnd): - sec = NDData(np.zeros((2, 2)), - uncertainty=VarianceUncertainty(np.ones((2, 2)))) - testnd.set_section((slice(0, 2), slice(1, 3)), sec) - assert_array_equal(testnd[:2, 1:3].data, 0) - assert_array_equal(testnd[:2, 1:3].variance, 1) - - -def test_uncertainty_negative_numbers(): - arr = np.zeros(5) - - # No warning if all 0 - with warnings.catch_warnings(record=True) as w: - ADVarianceUncertainty(arr) - assert len(w) == 0 - - arr[2] = -0.001 - - with pytest.warns(RuntimeWarning, match='Negative variance values found.'): - result = ADVarianceUncertainty(arr) - - assert not np.all(arr >= 0) - assert isinstance(result, ADVarianceUncertainty) - assert result.array[2] == 0 - - # check that it always works with a VarianceUncertainty instance - result.array[2] = -0.001 - - with pytest.warns(RuntimeWarning, match='Negative variance values found.'): - result2 = ADVarianceUncertainty(result) - - assert not np.all(arr >= 0) - assert not np.all(result.array >= 0) - assert isinstance(result2, ADVarianceUncertainty) - assert result2.array[2] == 0 - - -def test_wcs_slicing(): - nd = NDAstroData(np.zeros((50, 50))) - in_frame = Frame2D(name="in_frame") - out_frame = Frame2D(name="out_frame") - nd.wcs = gWCS([(in_frame, models.Identity(2)), - (out_frame, None)]) - assert nd.wcs(10, 10) == (10, 10) - assert nd[10:].wcs(10, 10) == (10, 20) - assert nd[..., 10:].wcs(10, 10) == (20, 10) - assert nd[:, 5].wcs(10) == (5, 10) - assert nd[20, -10:].wcs(0) == (40, 20) - - -def test_access_to_other_planes(testnd): - assert hasattr(testnd, 'OBJMASK') - assert testnd.OBJMASK.shape == testnd.data.shape - assert hasattr(testnd, 'OBJCAT') - assert isinstance(testnd.OBJCAT, Table) - assert len(testnd.OBJCAT) == 3 - - -def test_access_to_other_planes_when_windowed(testnd): - ndwindow = testnd.window[1:, 1:] - assert ndwindow.data.shape == (4, 4) - assert ndwindow.data[0, 0] == testnd.shape[1] + 1 - assert ndwindow.OBJMASK.shape == (4, 4) - assert ndwindow.OBJMASK[0, 0] == testnd.shape[1] + 1 - assert isinstance(ndwindow.OBJCAT, Table) - assert len(ndwindow.OBJCAT) == 3 - - -# Basically the same test as above but using slicing. -def test_access_to_other_planes_when_sliced(testnd): - ndwindow = testnd[1:, 1:] - assert ndwindow.data.shape == (4, 4) - assert ndwindow.data[0, 0] == testnd.shape[1] + 1 - assert ndwindow.OBJMASK.shape == (4, 4) - assert ndwindow.OBJMASK[0, 0] == testnd.shape[1] + 1 - assert isinstance(ndwindow.OBJCAT, Table) - assert len(ndwindow.OBJCAT) == 3 - - -if __name__ == '__main__': - pytest.main() diff --git a/astrodata/tests/test_object_construction.py b/astrodata/tests/test_object_construction.py deleted file mode 100644 index ed9630686b..0000000000 --- a/astrodata/tests/test_object_construction.py +++ /dev/null @@ -1,430 +0,0 @@ -import warnings - -import astrodata -import astropy.units as u -import numpy as np -import pytest -from astrodata import AstroData, factory -from astrodata.testing import download_from_archive -from astropy.io import fits -from astropy.nddata import NDData, VarianceUncertainty -from astropy.table import Table -from numpy.testing import assert_array_equal - - -@pytest.fixture() -def testfile1(): - """ - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (2304, 1056) uint16 - [ 1] science NDAstroData (2304, 1056) uint16 - [ 2] science NDAstroData (2304, 1056) uint16 - """ - return download_from_archive("N20110826S0336.fits") - - -@pytest.fixture -def testfile2(): - """ - Pixels Extensions - Index Content Type Dimensions Format - [ 0] science NDAstroData (4608, 1056) uint16 - [ 1] science NDAstroData (4608, 1056) uint16 - [ 2] science NDAstroData (4608, 1056) uint16 - [ 3] science NDAstroData (4608, 1056) uint16 - [ 4] science NDAstroData (4608, 1056) uint16 - [ 5] science NDAstroData (4608, 1056) uint16 - """ - return download_from_archive("N20160524S0119.fits") - - -class AstroDataMyInstrument(AstroData): - __keyword_dict = dict( - array_name='AMPNAME', - array_section='CCDSECT' - ) - - @staticmethod - def _matches_data(source): - return source[0].header.get('INSTRUME', '').upper() == 'MYINSTRUMENT' - - -def setup_module(): - factory.addClass(AstroDataMyInstrument) - - -def teardown_module(): - factory._registry.remove(AstroDataMyInstrument) - - -def test_create_with_no_data(): - for phu in (fits.PrimaryHDU(), fits.Header(), {}): - ad = astrodata.create(phu) - assert isinstance(ad, astrodata.AstroData) - assert len(ad) == 0 - assert ad.instrument() is None - assert ad.object() is None - - -def test_create_with_header(): - hdr = fits.Header({'INSTRUME': 'darkimager', 'OBJECT': 'M42'}) - for phu in (hdr, fits.PrimaryHDU(header=hdr), dict(hdr), list(hdr.cards)): - ad = astrodata.create(phu) - assert isinstance(ad, astrodata.AstroData) - assert len(ad) == 0 - assert ad.instrument() == 'darkimager' - assert ad.object() == 'M42' - - -def test_create_from_hdu(): - phu = fits.PrimaryHDU() - hdu = fits.ImageHDU(data=np.zeros((4, 5)), name='SCI') - ad = astrodata.create(phu, [hdu]) - - assert isinstance(ad, astrodata.AstroData) - assert len(ad) == 1 - assert isinstance(ad[0].data, np.ndarray) - assert ad[0].data is hdu.data - - -def test_create_invalid(): - with pytest.raises(ValueError): - astrodata.create('FOOBAR') - with pytest.raises(ValueError): - astrodata.create(42) - - -def test_append_image_hdu(): - ad = astrodata.create(fits.PrimaryHDU()) - ad.append(fits.ImageHDU(data=np.zeros((4, 5)))) - ad.append(fits.ImageHDU(data=np.zeros((4, 5))), name='SCI') - - with pytest.raises(ValueError, - match="Arbitrary image extensions can only be added " - "in association to a 'SCI'"): - ad.append(fits.ImageHDU(data=np.zeros((4, 5))), name='SCI2') - - assert len(ad) == 2 - - -def test_append_lowercase_name(): - ad = astrodata.create({}) - with pytest.warns(UserWarning, - match="extension name 'sci' should be uppercase"): - ad.append(NDData(np.zeros((4, 5))), name='sci') - - -def test_append_arrays(tmp_path): - testfile = tmp_path / 'test.fits' - - ad = astrodata.create({}) - ad.append(np.zeros(10)) - ad[0].ARR = np.arange(5) - - with pytest.raises(AttributeError): - ad[0].SCI = np.arange(5) - with pytest.raises(AttributeError): - ad[0].VAR = np.arange(5) - with pytest.raises(AttributeError): - ad[0].DQ = np.arange(5) - - match = ("Arbitrary image extensions can only be added in association " - "to a 'SCI'") - with pytest.raises(ValueError, match=match): - ad.append(np.zeros(10), name='FOO') - - with pytest.raises(ValueError, match=match): - ad.append(np.zeros(10), header=fits.Header({'EXTNAME': 'FOO'})) - - ad.write(testfile) - - ad = astrodata.open(testfile) - assert len(ad) == 1 - assert ad[0].nddata.meta['header']['EXTNAME'] == 'SCI' - assert_array_equal(ad[0].ARR, np.arange(5)) - - -@pytest.mark.dragons_remote_data -def test_can_read_data(testfile1): - ad = astrodata.open(testfile1) - assert len(ad) == 3 - assert ad.shape == [(2304, 1056), (2304, 1056), (2304, 1056)] - - -def test_can_read_write_pathlib(tmp_path): - testfile = tmp_path / 'test.fits' - - ad = astrodata.create({'INSTRUME': 'MYINSTRUMENT'}) - ad.append(np.zeros((4, 5))) - ad.write(testfile) - - ad = astrodata.open(testfile) - assert isinstance(ad, AstroDataMyInstrument) - assert len(ad) == 1 - assert ad.shape == [(4, 5)] - - -@pytest.mark.dragons_remote_data -def test_append_array_to_root_no_name(testfile2): - ad = astrodata.open(testfile2) - - lbefore = len(ad) - ones = np.ones((10, 10)) - ad.append(ones) - assert len(ad) == (lbefore + 1) - assert ad[-1].data is ones - assert ad[-1].hdr['EXTNAME'] == 'SCI' - - -@pytest.mark.dragons_remote_data -def test_append_array_to_root_with_name_sci(testfile2): - ad = astrodata.open(testfile2) - - lbefore = len(ad) - ones = np.ones((10, 10)) - ad.append(ones, name='SCI') - assert len(ad) == (lbefore + 1) - assert ad[-1].data is ones - assert ad[-1].hdr['EXTNAME'] == 'SCI' - - -@pytest.mark.dragons_remote_data -def test_append_array_to_root_with_arbitrary_name(testfile2): - ad = astrodata.open(testfile2) - assert len(ad) == 6 - - ones = np.ones((10, 10)) - with pytest.raises(ValueError): - ad.append(ones, name='ARBITRARY') - - -@pytest.mark.dragons_remote_data -def test_append_array_to_extension_with_name_sci(testfile2): - ad = astrodata.open(testfile2) - assert len(ad) == 6 - - ones = np.ones((10, 10)) - with pytest.raises(TypeError): - ad[0].append(ones, name='SCI') - - -@pytest.mark.dragons_remote_data -def test_append_array_to_extension_with_arbitrary_name(testfile2): - ad = astrodata.open(testfile2) - - lbefore = len(ad) - ones = np.ones((10, 10)) - ad[0].ARBITRARY = ones - - assert len(ad) == lbefore - assert ad[0].ARBITRARY is ones - - -@pytest.mark.dragons_remote_data -def test_append_nddata_to_root_no_name(testfile2): - ad = astrodata.open(testfile2) - - lbefore = len(ad) - ones = np.ones((10, 10)) - hdu = fits.ImageHDU(ones) - nd = NDData(hdu.data) - nd.meta['header'] = hdu.header - ad.append(nd) - assert len(ad) == (lbefore + 1) - - -@pytest.mark.dragons_remote_data -def test_append_nddata_to_root_with_arbitrary_name(testfile2): - ad = astrodata.open(testfile2) - assert len(ad) == 6 - - ones = np.ones((10, 10)) - hdu = fits.ImageHDU(ones) - nd = NDData(hdu.data) - nd.meta['header'] = hdu.header - hdu.header['EXTNAME'] = 'ARBITRARY' - with pytest.raises(ValueError): - ad.append(nd) - - -def test_append_table_to_extensions(tmp_path): - testfile = tmp_path / 'test.fits' - ad = astrodata.create({}) - ad.append(NDData(np.zeros((4, 5)))) - ad.append(NDData(np.zeros((4, 5)))) - ad.append(NDData(np.zeros((4, 5)), meta={'header': {'FOO': 'BAR'}})) - ad[0].TABLE1 = Table([[1]]) - ad[0].TABLE2 = Table([[22]]) - ad[1].TABLE2 = Table([[2]]) # extensions can have the same table name - ad[2].TABLE3 = Table([[3]]) - ad.write(testfile) - - ad = astrodata.open(testfile) - - # Check that slices do not report extension tables - assert ad.exposed == set() - assert ad[0].exposed == {'TABLE1', 'TABLE2'} - assert ad[1].exposed == {'TABLE2'} - assert ad[2].exposed == {'TABLE3'} - assert ad[1:].exposed == set() - - assert ad[2].hdr['FOO'] == 'BAR' - - match = ("Cannot append table 'TABLE1' because it would hide an " - "extension table") - with pytest.raises(ValueError, match=match): - ad.TABLE1 = Table([[1]]) - - -def test_append_table_and_write(tmp_path): - testfile = tmp_path / 'test.fits' - ad = astrodata.create({}) - ad.append(NDData(np.zeros((4, 5)))) - ad[0].TABLE1 = Table([[1]]) - ad.write(testfile) - ad.write(testfile, overwrite=True) - - ad = astrodata.open(testfile) - assert ad[0].exposed == {'TABLE1'} - - -def test_table_with_units(tmp_path): - testfile = tmp_path / 'test.fits' - ad = astrodata.create({}) - ad.append(NDData(np.zeros((4, 5)))) - ad[0].TABLE1 = Table([[1]]) - ad[0].TABLE1['col0'].unit = 'mag(cm2 electron / erg)' - - with warnings.catch_warnings(record=True) as w: - ad.write(testfile) - - assert len(w) == 0 - ad = astrodata.open(testfile) - assert ad[0].TABLE1['col0'].unit == u.Unit('mag(cm2 electron / erg)') - - -# Append / assign Gemini specific - -@pytest.mark.dragons_remote_data -def test_append_dq_var(testfile2): - ad = astrodata.open(testfile2) - - dq = np.zeros(ad[0].data.shape) - with pytest.raises(ValueError): - ad.append(dq, name='DQ') - with pytest.raises(AttributeError): - ad.DQ = dq - with pytest.raises(AttributeError): - ad[0].DQ = dq - - var = np.ones(ad[0].data.shape) - with pytest.raises(ValueError): - ad.append(var, name='VAR') - with pytest.raises(AttributeError): - ad.VAR = var - with pytest.raises(AttributeError): - ad[0].VAR = var - - -# Append AstroData slices - -@pytest.mark.dragons_remote_data -def test_append_single_slice(testfile1, testfile2): - ad = astrodata.open(testfile2) - ad2 = astrodata.open(testfile1) - - lbefore = len(ad2) - ad2.append(ad[1]) - - assert len(ad2) == (lbefore + 1) - assert np.all(ad2[-1].data == ad[1].data) - - # With a custom header - ad2.append(ad[1], header=fits.Header({'FOO': 'BAR'})) - assert ad2[-1].nddata.meta['header']['FOO'] == 'BAR' - - -@pytest.mark.dragons_remote_data -def test_append_non_single_slice(testfile1, testfile2): - ad = astrodata.open(testfile2) - ad2 = astrodata.open(testfile1) - - with pytest.raises(ValueError): - ad2.append(ad[1:]) - - -@pytest.mark.dragons_remote_data -def test_append_whole_instance(testfile1, testfile2): - ad = astrodata.open(testfile2) - ad2 = astrodata.open(testfile1) - - with pytest.raises(ValueError): - ad2.append(ad) - - -@pytest.mark.dragons_remote_data -def test_append_slice_to_extension(testfile1, testfile2): - ad = astrodata.open(testfile2) - ad2 = astrodata.open(testfile1) - - with pytest.raises(TypeError): - ad2[0].append(ad[0], name="FOOBAR") - - match = "Cannot append an AstroData slice to another slice" - with pytest.raises(ValueError, match=match): - ad[2].FOO = ad2[1] - - -@pytest.mark.dragons_remote_data -def test_delete_named_associated_extension(testfile2): - ad = astrodata.open(testfile2) - ad[0].MYTABLE = Table(([1, 2, 3], [4, 5, 6], [7, 8, 9]), - names=('a', 'b', 'c')) - assert 'MYTABLE' in ad[0] - del ad[0].MYTABLE - assert 'MYTABLE' not in ad[0] - - -@pytest.mark.dragons_remote_data -def test_delete_arbitrary_attribute_from_ad(testfile2): - ad = astrodata.open(testfile2) - - with pytest.raises(AttributeError): - ad.arbitrary - - ad.arbitrary = 15 - - assert ad.arbitrary == 15 - - del ad.arbitrary - - with pytest.raises(AttributeError): - ad.arbitrary - - -def test_build_ad_multiple_extensions(tmp_path): - """Build an AD object with multiple extensions and check that we retrieve - everything in the correct order after writing. - """ - shape = (4, 5) - testfile = tmp_path / 'test.fits' - - ad = astrodata.create({}) - for i in range(1, 4): - nd = NDData(np.zeros(shape) + i, - uncertainty=VarianceUncertainty(np.ones(shape)), - mask=np.zeros(shape, dtype='uint16')) - ad.append(nd) - ad[-1].OBJCAT = Table([[i]]) - ad[-1].MYARR = np.zeros(10) + i - - ad.REFCAT = Table([['ref']]) - ad.write(testfile) - - ad2 = astrodata.open(testfile) - - for ext, ext2 in zip(ad, ad2): - assert_array_equal(ext.data, ext2.data) - assert_array_equal(ext.MYARR, ext2.MYARR) - assert_array_equal(ext.OBJCAT['col0'], ext2.OBJCAT['col0']) diff --git a/astrodata/tests/test_provenance.py b/astrodata/tests/test_provenance.py deleted file mode 100644 index 832f0c7774..0000000000 --- a/astrodata/tests/test_provenance.py +++ /dev/null @@ -1,172 +0,0 @@ -from datetime import datetime, timedelta, timezone -import os - -import numpy as np -import pytest - -import astrodata -from astrodata import fits -from astrodata.testing import download_from_archive -from astrodata.provenance import (add_provenance, add_history, clone_provenance, - clone_history) - - -@pytest.fixture -def ad(): - phu = fits.PrimaryHDU() - hdu = fits.ImageHDU(data=np.ones((10, 10)), name='SCI') - return astrodata.create(phu, [hdu]) - - -@pytest.fixture -def ad2(): - phu = fits.PrimaryHDU() - hdu = fits.ImageHDU(data=np.ones((10, 10)), name='SCI') - return astrodata.create(phu, [hdu]) - - -def test_add_get_provenance(ad): - timestamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() - filename = "filename" - md5 = "md5" - primitive = "provenance_added_by" - - # if md5 is None, provenance is added with empty string as md5 - add_provenance(ad, filename, None, primitive) - assert len(ad.PROVENANCE) == 1 - assert tuple(ad.PROVENANCE[0])[1:] == (filename, '', primitive) - - add_provenance(ad, filename, md5, primitive, timestamp=timestamp) - assert len(ad.PROVENANCE) == 2 - assert tuple(ad.PROVENANCE[1]) == (timestamp, filename, md5, primitive) - - # entry is updated and a default timestamp is created - add_provenance(ad, filename, md5, primitive) - assert len(ad.PROVENANCE) == 2 - assert tuple(ad.PROVENANCE[1])[1:] == (filename, md5, primitive) - - # add new entry - add_provenance(ad, filename, 'md6', 'other primitive') - assert len(ad.PROVENANCE) == 3 - assert tuple(ad.PROVENANCE[1])[1:] == (filename, md5, primitive) - assert tuple(ad.PROVENANCE[2])[1:] == (filename, 'md6', 'other primitive') - - -def test_add_duplicate_provenance(ad): - timestamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() - filename = "filename" - md5 = "md5" - primitive = "provenance_added_by" - - add_provenance(ad, filename, md5, primitive, timestamp=timestamp) - add_provenance(ad, filename, md5, primitive, timestamp=timestamp) - - # was a dupe, so should have been skipped - assert len(ad.PROVENANCE) == 1 - - -def test_add_get_history(ad): - timestamp_start = datetime.now(timezone.utc).replace(tzinfo=None) - timestamp_end = (timestamp_start + - timedelta(days=1)).isoformat() - timestamp_start = timestamp_start.isoformat() - primitive = "primitive" - args = "args" - - add_history(ad, timestamp_start, timestamp_end, primitive, args) - assert len(ad.HISTORY) == 1 - assert tuple(ad.HISTORY[0]) == (primitive, args, timestamp_start, - timestamp_end) - - add_history(ad, timestamp_start, timestamp_end, 'another primitive', args) - assert len(ad.HISTORY) == 2 - assert tuple(ad.HISTORY[0]) == (primitive, args, timestamp_start, - timestamp_end) - assert tuple(ad.HISTORY[1]) == ('another primitive', args, - timestamp_start, timestamp_end) - - -def test_add_dupe_history(ad): - timestamp_start = datetime.now(timezone.utc).replace(tzinfo=None) - timestamp_end = (timestamp_start + timedelta(days=1)).isoformat() - timestamp_start = timestamp_start.isoformat() - primitive = "primitive" - args = "args" - - add_history(ad, timestamp_start, timestamp_end, primitive, args) - add_history(ad, timestamp_start, timestamp_end, primitive, args) - - # was a dupe, should have skipped 2nd add - assert len(ad.HISTORY) == 1 - - -def test_clone_provenance(ad, ad2): - timestamp = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() - filename = "filename" - md5 = "md5" - primitive = "provenance_added_by" - - add_provenance(ad, filename, md5, primitive, timestamp=timestamp) - - clone_provenance(ad.PROVENANCE, ad2) - - assert len(ad2.PROVENANCE) == 1 - assert tuple(ad2.PROVENANCE[0]) == (timestamp, filename, md5, primitive) - - -def test_clone_history(ad, ad2): - timestamp_start = datetime.now(timezone.utc).replace(tzinfo=None) - timestamp_end = (timestamp_start + timedelta(days=1)).isoformat() - timestamp_start = timestamp_start.isoformat() - primitive = "primitive" - args = "args" - - add_history(ad, timestamp_start, timestamp_end, primitive, args) - - clone_history(ad.HISTORY, ad2) - - assert len(ad2.HISTORY) == 1 - assert tuple(ad2.HISTORY[0]) == (primitive, args, timestamp_start, - timestamp_end) - - -@pytest.fixture(scope='module') -def BPM_PROVHISTORY(): - """ - BPM file with PROVHISTORY (old name for HISTORY) - """ - return download_from_archive("bpm_20220128_gmos-s_Ham_11_full_12amp.fits") - - -@pytest.mark.dragons_remote_data -def test_convert_provhistory(tmpdir, BPM_PROVHISTORY): - ad = astrodata.open(BPM_PROVHISTORY) - - # This file (should) use the old PROVHISTORY extname - assert hasattr(ad, 'PROVHISTORY') - - # When we add history, that should get converted to HISTORY - now = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() - add_history(ad, now, now, "primitive", "args") - assert not hasattr(ad, 'PROVHISTORY') - assert hasattr(ad, 'HISTORY') - - # and if we write the file, it should have a HISTORY extname - # and not a PROVHISTORY extname - testfile = os.path.join(str(tmpdir), 'temp.fits') - ad.path = testfile - ad.write() - assert os.path.exists(testfile) - - ad2 = astrodata.open(testfile) - assert hasattr(ad2, 'HISTORY') - assert not hasattr(ad2, 'PROVHISTORY') - - # and should have that new history record we added - hist = ad.HISTORY[-1] - assert hist['timestamp_start'] == now - assert hist['timestamp_stop'] == now - assert hist['primitive'] == 'primitive' - assert hist['args'] == 'args' - - os.remove(testfile) diff --git a/astrodata/tests/test_tags.py b/astrodata/tests/test_tags.py deleted file mode 100644 index 6e51e83d23..0000000000 --- a/astrodata/tests/test_tags.py +++ /dev/null @@ -1,162 +0,0 @@ -import os -import numpy as np -import pytest -from astropy.io import fits -from astropy.table import Table - -import astrodata -from astrodata import (astro_data_tag, astro_data_descriptor, TagSet, - AstroData, factory, returns_list) - -SHAPE = (4, 5) - - -class AstroDataMyInstrument(AstroData): - __keyword_dict = dict( - array_name='AMPNAME', - array_section='CCDSECT' - ) - - @staticmethod - def _matches_data(source): - return source[0].header.get('INSTRUME', '').upper() == 'MYINSTRUMENT' - - @astro_data_tag - def _tag_instrument(self): - return TagSet(['MYINSTRUMENT']) - - @astro_data_tag - def _tag_image(self): - if self.phu.get('GRATING') == 'MIRROR': - return TagSet(['IMAGE']) - - @astro_data_tag - def _tag_dark(self): - if self.phu.get('OBSTYPE') == 'DARK': - return TagSet(['DARK'], blocks=['IMAGE', 'SPECT']) - - @astro_data_tag - def _tag_raise(self): - raise KeyError # I guess if some keyword is missing... - - @returns_list - @astro_data_descriptor - def dispersion_axis(self): - return 1 - - @returns_list - @astro_data_descriptor - def gain(self): - return [1, 1] - - @returns_list - @astro_data_descriptor - def badguy(self): - return [1, 2, 3] - - @astro_data_descriptor - def array_name(self): - return self.phu.get(self._keyword_for('array_name')) - - @astro_data_descriptor - def detector_section(self): - return self.phu.get(self._keyword_for('array_section')) - - @astro_data_descriptor - def amp_read_area(self): - ampname = self.array_name() - detector_section = self.detector_section() - return "'{}':{}".format(ampname, detector_section) - - -def setup_module(): - factory.addClass(AstroDataMyInstrument) - - -def teardown_module(): - factory._registry.remove(AstroDataMyInstrument) - - -@pytest.fixture(scope='function') -def testfile(tmpdir): - hdr = fits.Header({ - 'INSTRUME': 'MYINSTRUMENT', - 'GRATING': 'MIRROR', - 'OBSTYPE': 'DARK', - 'AMPNAME': 'FOO', - 'CCDSECT': '1:1024', - }) - phu = fits.PrimaryHDU(header=hdr) - hdu = fits.ImageHDU(data=np.ones(SHAPE)) - hdu2 = fits.ImageHDU(data=np.ones(SHAPE) + 1) - ad = astrodata.create(phu, [hdu, hdu2]) - tbl = Table([np.zeros(10), np.ones(10)], names=['col1', 'col2']) - ad.MYCAT = tbl - filename = str(tmpdir.join('fakebias.fits')) - ad.write(filename) - yield filename - os.remove(filename) - - -def test_tags(testfile): - ad = astrodata.open(testfile) - assert ad.descriptors == ('amp_read_area', 'array_name', 'badguy', - 'detector_section', 'dispersion_axis', 'gain', - 'instrument', 'object', 'telescope') - assert ad.tags == {'DARK', 'MYINSTRUMENT'} - assert ad.amp_read_area() == "'FOO':1:1024" - - -def test_keyword_for(testfile): - ad = astrodata.open(testfile) - assert ad._keyword_for('array_name') == 'AMPNAME' - with pytest.raises(AttributeError, match="No match for 'foobar'"): - ad._keyword_for('foobar') - - -def test_returns_list(testfile): - ad = astrodata.open(testfile) - assert ad.dispersion_axis() == [1, 1] - assert ad[0].dispersion_axis() == 1 - - assert ad.gain() == [1, 1] - assert ad[0].gain() == 1 - - with pytest.raises(IndexError): - ad.badguy() - - -def test_info(testfile, capsys): - ad = astrodata.open(testfile) - ad.info() - captured = capsys.readouterr() - out = captured.out.splitlines() - assert out[0].endswith('fakebias.fits') - assert out[1:] == [ - 'Tags: DARK MYINSTRUMENT', - '', - 'Pixels Extensions', - 'Index Content Type Dimensions Format', - '[ 0] science NDAstroData (4, 5) float64', - '[ 1] science NDAstroData (4, 5) float64', - '', - 'Other Extensions', - ' Type Dimensions', - '.MYCAT Table (10, 2)' - ] - - ad[1].info() - captured = capsys.readouterr() - out = captured.out.splitlines() - assert out[0].endswith('fakebias.fits') - assert out[1:] == [ - 'Tags: DARK MYINSTRUMENT', - '', - 'Pixels Extensions', - 'Index Content Type Dimensions Format', - '[ 0] science NDAstroData (4, 5) float64', - '', - 'Other Extensions', - ' Type Dimensions', - '.MYCAT Table (10, 2)' - ] diff --git a/astrodata/tests/test_testing.py b/astrodata/tests/test_testing.py deleted file mode 100644 index d191cb889f..0000000000 --- a/astrodata/tests/test_testing.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Tests for the `astrodata.testing` module. -""" - -import os - -import astrodata -import numpy as np -import pytest -from astrodata.testing import assert_same_class, download_from_archive - - -def test_download_from_archive_raises_ValueError_if_envvar_does_not_exists(): - with pytest.raises(ValueError): - download_from_archive('N20180304S0126.fits', env_var='') - - -def test_download_from_archive_raises_IOError_if_path_is_not_accessible(): - env_var = 'MY_FAKE_ENV_VAR' - os.environ['MY_FAKE_ENV_VAR'] = "/not/accessible/path" - with pytest.raises(IOError): - download_from_archive('N20180304S0126.fits', env_var=env_var) - - -def test_download_from_archive(monkeypatch, tmpdir): - ncall = 0 - - def mock_download(remote_url, **kwargs): - nonlocal ncall - ncall += 1 - fname = remote_url.split('/')[-1] - tmpdir.join(fname).write('') # create fake file - return str(tmpdir.join(fname)) - - monkeypatch.setattr("astrodata.testing.download_file", mock_download) - monkeypatch.setenv("DRAGONS_TEST", str(tmpdir)) - - # first call will use our mock function above - fname = download_from_archive('N20170529S0168.fits') - assert os.path.exists(fname) - assert ncall == 1 - - # second call will use the cache so we check that our mock function is not - # called twice - fname = download_from_archive('N20170529S0168.fits') - assert os.path.exists(fname) - assert ncall == 1 - - -def test_assert_most_close(): - from astrodata.testing import assert_most_close - x = np.arange(10) - y = np.arange(10) - assert_most_close(x, y, 1) - - y[0] = -1 - assert_most_close(x, y, 1) - - with pytest.raises(AssertionError): - y[1] = -1 - assert_most_close(x, y, 1) - - -def test_assert_most_equal(): - from astrodata.testing import assert_most_equal - x = np.arange(10) - y = np.arange(10) - assert_most_equal(x, y, 1) - - y[0] = -1 - assert_most_equal(x, y, 1) - - with pytest.raises(AssertionError): - y[1] = -1 - assert_most_equal(x, y, 1) - - -def test_assert_same_class(): - ad = astrodata.create({}) - ad2 = astrodata.create({}) - assert_same_class(ad, ad2) - - with pytest.raises(AssertionError): - assert_same_class(ad, np.array([1])) diff --git a/astrodata/tests/test_utils.py b/astrodata/tests/test_utils.py deleted file mode 100644 index 585b2c0b71..0000000000 --- a/astrodata/tests/test_utils.py +++ /dev/null @@ -1,81 +0,0 @@ -import numpy as np -import pytest -import astropy.units as u -from astropy.io import fits -from astropy.table import Table -from astrodata.fits import header_for_table, card_filter, update_header -from astrodata import Section - - -def test_header_for_table(): - tbl = Table([np.arange(2 * 3 * 4).reshape(3, 2, 4), - [1.0, 2.0, 3.0], - ['aa', 'bb', 'cc'], - [[True, False], [True, False], [True, False]]], - names='abcd') - tbl['b'].unit = u.arcsec - hdr = header_for_table(tbl) - assert hdr['TFORM1'] == '8K' - assert hdr['TDIM1'] == '(4,2)' - assert hdr['TFORM4'] == '2L' - assert hdr['TUNIT2'] == 'arcsec' - - -def test_card_filter(): - hdr = fits.Header(dict(zip('ABCDE', range(5)))) - assert [c.keyword for c in card_filter(hdr.cards, include='ABC')] == \ - ['A', 'B', 'C'] - assert [c.keyword for c in card_filter(hdr.cards, exclude='AB')] == \ - ['C', 'D', 'E'] - - -def test_update_header(): - hdra = fits.Header({'INSTRUME': 'darkimager', 'OBJECT': 'M42'}) - hdra.add_comment('A super useful comment') - hdra.add_history('This is historic') - assert update_header(hdra, hdra) is hdra - - hdrb = fits.Header({'OBJECT': 'IO', 'EXPTIME': 42}) - hdrb.add_comment('A super useful comment') - hdrb.add_comment('Another comment') - hdrb.add_history('This is historic') - hdrb.add_history('And not so useful') - - hdr = update_header(hdra, hdrb) - # Check that comments have been merged - assert list(hdr['COMMENT']) == ['A super useful comment', 'Another comment'] - assert list(hdr['HISTORY']) == ['This is historic', 'And not so useful'] - - -def test_section_basics(): - s = Section.from_string("[1:1024,1:512") - assert tuple(s) == (0, 1024, 0, 512) - assert s.asIRAFsection() == "[1:1024,1:512]" - assert s.asslice() == (slice(0, 512), slice(0, 1024)) - assert (s.x1, s.x2, s.y1, s.y2) == (0, 1024, 0, 512) - assert (s[0], s[1], s[2], s[3]) == (0, 1024, 0, 512) - - s = Section.from_shape((8, 512, 1024)) - assert tuple(s) == (0, 1024, 0, 512, 0, 8) - assert s.z2 == 8 - - -def test_section_relations(): - s = Section.from_string("[1:1024,1:512") - s2 = Section.from_string("[1:1024,1:256]") - s3 = Section.from_string("[1:512,1:1024]") - s4 = Section.from_string("[513:1024,1:1024]") - - assert s.contains(s2) - assert not s.contains(s3) - - assert s.overlap(s2) == Section(0, 1024, 0, 256) - assert s.overlap(s3) == Section(0, 512, 0, 512) - assert s.overlap(s2) == s2.overlap(s) - assert s3.overlap(s4) is None - - assert s3.is_same_size(s4) - - ss = s2.shift(100, 200) - assert ss.is_same_size(s2) - assert ss == Section(100, 1124, 200, 456) diff --git a/astrodata/tests/test_wcs.py b/astrodata/tests/test_wcs.py deleted file mode 100644 index 33440f2f87..0000000000 --- a/astrodata/tests/test_wcs.py +++ /dev/null @@ -1,267 +0,0 @@ -import math -import os -import pytest -import numpy as np -from numpy.testing import assert_allclose - -from astropy.modeling import models -from astropy.wcs import WCS -from astropy.coordinates import SkyCoord -from astropy import units as u -from astropy.io.fits import Header -from gwcs import coordinate_frames as cf -from gwcs.wcs import WCS as gWCS - -import astrodata -from astrodata import wcs as adwcs -from astrodata.testing import download_from_archive -from gempy.library.transform import add_longslit_wcs - - -@pytest.fixture(scope='module') -def F2_IMAGE(): - """Any F2 image with CD3_3=1""" - return download_from_archive("S20130717S0365.fits") - - -@pytest.fixture(scope='module') -def NIRI_IMAGE(): - """Any NIRI image""" - return download_from_archive("N20180102S0392.fits") - - -@pytest.fixture(scope='module') -def GMOS_LONGSLIT(): - """Any GMOS longslit spectrum""" - return download_from_archive("N20180103S0332.fits") - - -@pytest.mark.parametrize("angle", [0, 20, 67, -35]) -@pytest.mark.parametrize("scale", [0.5, 1.0, 2.0]) -@pytest.mark.parametrize("xoffset,yoffset", [(0,0), (10,20)]) -def test_calculate_affine_matrices(angle, scale, xoffset, yoffset): - m = ((models.Scale(scale) & models.Scale(scale)) | - models.Rotation2D(angle) | - (models.Shift(xoffset) & models.Shift(yoffset))) - affine = adwcs.calculate_affine_matrices(m, (100, 100)) - assert_allclose(affine.offset, (yoffset, xoffset), atol=1e-10) - angle = math.radians(angle) - assert_allclose(affine.matrix, ((scale * math.cos(angle), scale * math.sin(angle)), - (-scale * math.sin(angle), scale * math.cos(angle))), - atol=1e-10) - - -@pytest.mark.dragons_remote_data -def test_reading_and_writing_sliced_image(F2_IMAGE): - ad = astrodata.open(F2_IMAGE) - result = ad[0].wcs(100, 100, 0) - ad[0].reset(ad[0].nddata[0]) - assert_allclose(ad[0].wcs(100, 100), result) - ad.write("test.fits", overwrite=True) - ad2 = astrodata.open("test.fits") - assert_allclose(ad2[0].wcs(100, 100), result) - ad2.write("test.fits", overwrite=True) - ad2 = astrodata.open("test.fits") - assert_allclose(ad2[0].wcs(100, 100), result) - - -def test_remove_axis_from_model(): - """A simple test that removes one of three &-linked models""" - model = models.Shift(0) & models.Shift(1) & models.Shift(2) - for axis in (0, 1, 2): - new_model, input_axis = adwcs.remove_axis_from_model(model, axis) - assert input_axis == axis - assert new_model.n_submodels == 2 - assert new_model.offset_0 + new_model.offset_1 == 3 - axis - - -def test_remove_axis_from_model_2(): - """A test with |-chained models""" - model = ((models.Shift(0) & models.Shift(1) & models.Shift(2)) | - (models.Scale(2) & models.Rotation2D(90))) - new_model, input_axis = adwcs.remove_axis_from_model(model, 0) - assert input_axis == 0 - assert new_model.n_submodels == 3 - assert new_model.offset_0 == 1 - assert new_model.offset_1 == 2 - assert new_model.angle_2 == 90 - - -def test_remove_axis_from_model_3(): - """A test with a Mapping""" - model1 = models.Mapping((1, 2, 0)) - model2 = models.Shift(0) & models.Shift(1) & models.Shift(2) - new_model, input_axis = adwcs.remove_axis_from_model(model1 | model2, 1) - assert input_axis == 2 - assert new_model.n_submodels == 3 - assert_allclose(new_model(0, 10), (10, 2)) - new_model, input_axis = adwcs.remove_axis_from_model(model2 | model1, 1) - assert input_axis == 2 - assert new_model.n_submodels == 3 - assert_allclose(new_model(0, 10), (11, 0)) - - -def test_remove_axis_from_model_4(): - """A test with a Mapping that creates a new axis""" - model1 = models.Shift(0) & models.Shift(1) & models.Shift(2) - model = models.Mapping((1, 0, 0)) | model1 - new_model, input_axis = adwcs.remove_axis_from_model(model, 1) - assert input_axis is None - assert new_model.n_submodels == 3 - assert_allclose(new_model(0, 10), (10, 2)) - - # Check that we can identify and remove the "Identity"-like residual Mapping - model = models.Mapping((0, 1, 0)) | model1 - new_model, input_axis = adwcs.remove_axis_from_model(model, 2) - assert input_axis is None - assert new_model.n_submodels == 2 - assert_allclose(new_model(0, 10), (0, 11)) - - -def test_remove_axis_from_model_5(): - """A test with fix_inputs""" - model1 = models.Shift(0) & models.Shift(1) & models.Shift(2) - model = models.fix_inputs(model1, {1: 6}) - new_model, input_axis = adwcs.remove_axis_from_model(model, 1) - assert input_axis is None - assert new_model.n_submodels == 2 - assert_allclose(new_model(0, 10), (0, 12)) - - new_model, input_axis = adwcs.remove_axis_from_model(model, 2) - assert input_axis == 2 - assert new_model.n_submodels == 3 - assert_allclose(new_model(0), (0, 7)) - - -@pytest.mark.dragons_remote_data -def test_remove_unused_world_axis(F2_IMAGE): - """A test with an intermediate frame""" - ad = astrodata.open(F2_IMAGE) - result = ad[0].wcs(1000, 1000, 0) - new_frame = cf.Frame2D(name="intermediate") - new_model = models.Shift(100) & models.Shift(200) & models.Identity(1) - ad[0].wcs.insert_frame(ad[0].wcs.input_frame, - new_model, new_frame) - ad[0].reset(ad[0].nddata[0]) - new_result = ad[0].wcs(900, 800) - assert_allclose(new_result, result) - adwcs.remove_unused_world_axis(ad[0]) - new_result = ad[0].wcs(900, 800) - assert_allclose(new_result, result[-2:]) - for frame in ad[0].wcs.available_frames: - assert getattr(ad[0].wcs, frame).naxes == 2 - - -@pytest.mark.dragons_remote_data -def test_gwcs_creation(NIRI_IMAGE): - """Test that the gWCS object for an image agrees with the FITS WCS""" - ad = astrodata.open(NIRI_IMAGE) - w = WCS(ad[0].hdr) - for y in range(0, 1024, 200): - for x in range(0, 1024, 200): - wcs_sky = w.pixel_to_world(x, y) - gwcs_sky = SkyCoord(*ad[0].wcs(x, y), unit=u.deg) - assert wcs_sky.separation(gwcs_sky) < 0.01 * u.arcsec - - -@pytest.mark.dragons_remote_data -def test_adding_longslit_wcs(GMOS_LONGSLIT): - """Test that adding the longslit WCS doesn't interfere with the sky - coordinates of the WCS""" - ad = astrodata.open(GMOS_LONGSLIT) - frame_name = ad[4].hdr.get("RADESYS", ad[4].hdr["RADECSYS"]).lower() - crpix1 = ad[4].hdr["CRPIX1"] - 1 - crpix2 = ad[4].hdr["CRPIX2"] - 1 - gwcs_sky = SkyCoord(*ad[4].wcs(crpix1, crpix2), unit=u.deg, frame=frame_name) - add_longslit_wcs(ad) - gwcs_coords = ad[4].wcs(crpix1, crpix2) - new_gwcs_sky = SkyCoord(*gwcs_coords[1:], unit=u.deg, frame=frame_name) - assert gwcs_sky.separation(new_gwcs_sky) < 0.01 * u.arcsec - # The sky coordinates should not depend on the x pixel value - gwcs_coords = ad[4].wcs(0, crpix2) - new_gwcs_sky = SkyCoord(*gwcs_coords[1:], unit=u.deg, frame=frame_name) - assert gwcs_sky.separation(new_gwcs_sky) < 0.01 * u.arcsec - - # The sky coordinates also should not depend on the extension - # there are shifts of order 1 pixel because of the rotations of CCDs 1 - # and 3, which are incorporated into their raw WCSs. Remember that the - # 12 WCSs are independent at this stage, they don't all map onto the - # WCS of the reference extension - for ext in ad: - gwcs_coords = ext.wcs(0, crpix2) - new_gwcs_sky = SkyCoord(*gwcs_coords[1:], unit=u.deg, frame=frame_name) - assert gwcs_sky.separation(new_gwcs_sky) < 0.1 * u.arcsec - - # This is equivalent to writing to disk and reading back in - wcs_dict = astrodata.wcs.gwcs_to_fits(ad[4].nddata, ad.phu) - new_gwcs = astrodata.wcs.fitswcs_to_gwcs(Header(wcs_dict)) - gwcs_coords = new_gwcs(crpix1, crpix2) - new_gwcs_sky = SkyCoord(*gwcs_coords[1:], unit=u.deg, frame=frame_name) - assert gwcs_sky.separation(new_gwcs_sky) < 0.01 * u.arcsec - gwcs_coords = new_gwcs(0, crpix2) - new_gwcs_sky = SkyCoord(*gwcs_coords[1:], unit=u.deg, frame=frame_name) - assert gwcs_sky.separation(new_gwcs_sky) < 0.01 * u.arcsec - -# Coordinates of projection center and new projection center -@pytest.mark.parametrize("coords", ([(0, 0), (0.1, -0.1)], - [(120, -50), (119.5, -49.5)], - [(270, 89.9), (0, 89)])) -@pytest.mark.parametrize("flip", (True, False)) -def test_create_new_image_projection(flip, coords): - shifts = (100, 200) - shifts = models.Shift(shifts[0]) & models.Shift(shifts[1]) - pixscale = 1.0 - angle = 0 - matrix = np.asarray(models.Rotation2D(angle)(*(np.identity(2) * pixscale / 3600))) - if not flip: - matrix[0] *= -1 - lon, lat = coords[0] - projection = (models.AffineTransformation2D(matrix=matrix) | - models.Pix2Sky_Gnomonic() | - models.RotateNative2Celestial(lon=lon, lat=lat, lon_pole=180)) - transform = shifts | projection - new_transform = adwcs.create_new_image_projection(transform, coords[1]) - for x in (0, 1000): - for y in (0, 1000): - c1 = SkyCoord(*transform(x, y), unit='deg') - c2 = SkyCoord(*new_transform(x, y), unit='deg') - assert c1.separation(c2).arcsec < 0.5 - - -@pytest.mark.dragons_remote_data -def test_loglinear_axis(NIRI_IMAGE): - """Test that we can add a log-linear axis and write and read it""" - ad = astrodata.open(NIRI_IMAGE) - coords = ad[0].wcs(200, 300) - ad[0].data = np.repeat(ad[0].data[:, :, np.newaxis], 5, axis=2) - new_input_frame = adwcs.pixel_frame(3) - loglinear_frame = cf.SpectralFrame(axes_order=(0,), unit=u.nm, - axes_names=("AWAV",), name="Wavelength in air") - celestial_frame = ad[0].wcs.output_frame - celestial_frame._axes_order = (1, 2) - new_output_frame = cf.CompositeFrame([loglinear_frame, celestial_frame], - name="world") - new_wcs = models.Exponential1D(amplitude=1, tau=2) & ad[0].wcs.forward_transform - ad[0].wcs = gWCS([(new_input_frame, new_wcs), - (new_output_frame, None)]) - new_coords = ad[0].wcs(2, 200, 300) - assert_allclose(coords, new_coords[1:]) - - #with change_working_dir(): - ad.write("test.fits", overwrite=True) - ad2 = astrodata.open("test.fits") - assert_allclose(ad2[0].wcs(2, 200, 300), new_coords) - - -@pytest.mark.preprocessed_data -def test_tabular1D_axis(path_to_inputs, change_working_dir): - """Check a FITS file with a tabular 1D axis is read correctly and - then rewritten to disk and read back in""" - ad = astrodata.open(os.path.join(path_to_inputs, "tab1dtest.fits")) - assert ad[0].wcs(0) == pytest.approx(3017.51065254) - assert ad[0].wcs(1021) == pytest.approx(4012.89510727) - with change_working_dir(): - ad.write("test.fits", overwrite=True) - ad2 = astrodata.open("test.fits") - assert ad2[0].wcs(0) == pytest.approx(3017.51065254) - assert ad2[0].wcs(1021) == pytest.approx(4012.89510727) diff --git a/astrodata/utils.py b/astrodata/utils.py deleted file mode 100644 index b35ae3247e..0000000000 --- a/astrodata/utils.py +++ /dev/null @@ -1,332 +0,0 @@ -import inspect -import warnings -from collections import namedtuple -from functools import wraps -from traceback import format_stack - -import numpy as np - -INTEGER_TYPES = (int, np.integer) - -__all__ = ('assign_only_single_slice', 'astro_data_descriptor', - 'AstroDataDeprecationWarning', 'astro_data_tag', 'deprecated', - 'normalize_indices', 'returns_list', 'TagSet', 'Section') - - -class AstroDataDeprecationWarning(DeprecationWarning): - pass - - -warnings.simplefilter("always", AstroDataDeprecationWarning) - - -def deprecated(reason): - def decorator_wrapper(fn): - @wraps(fn) - def wrapper(*args, **kw): - current_source = '|'.join(format_stack(inspect.currentframe())) - if current_source not in wrapper.seen: - wrapper.seen.add(current_source) - warnings.warn(reason, AstroDataDeprecationWarning) - return fn(*args, **kw) - wrapper.seen = set() - return wrapper - return decorator_wrapper - - -def normalize_indices(slc, nitems): - multiple = True - if isinstance(slc, slice): - start, stop, step = slc.indices(nitems) - indices = list(range(start, stop, step)) - elif (isinstance(slc, INTEGER_TYPES) or - (isinstance(slc, tuple) and - all(isinstance(i, INTEGER_TYPES) for i in slc))): - if isinstance(slc, INTEGER_TYPES): - slc = (int(slc),) # slc's type m - multiple = False - else: - multiple = True - # Normalize negative indices... - indices = [(x if x >= 0 else nitems + x) for x in slc] - else: - raise ValueError("Invalid index: {}".format(slc)) - - if any(i >= nitems for i in indices): - raise IndexError("Index out of range") - - return indices, multiple - - -class TagSet(namedtuple('TagSet', 'add remove blocked_by blocks if_present')): - """ - Named tuple that is used by tag methods to return which actions should be - performed on a tag set. All the attributes are optional, and any - combination of them can be used, allowing to create complex tag structures. - Read the documentation on the tag-generating algorithm if you want to - better understand the interactions. - - The simplest TagSet, though, tends to just add tags to the global set. - - It can be initialized by position, like any other tuple (the order of the - arguments is the one in which the attributes are listed below). It can - also be initialized by name. - - Attributes - ---------- - add : set of str, optional - Tags to be added to the global set - remove : set of str, optional - Tags to be removed from the global set - blocked_by : set of str, optional - Tags that will prevent this TagSet from being applied - blocks : set of str, optional - Other TagSets containing these won't be applied - if_present : set of str, optional - This TagSet will be applied only *all* of these tags are present - - Examples - --------- - >>> TagSet() - TagSet(add=set(), remove=set(), blocked_by=set(), blocks=set(), if_present=set()) - >>> TagSet({'BIAS', 'CAL'}) - TagSet(add={'BIAS', 'CAL'}, remove=set(), blocked_by=set(), blocks=set(), if_present=set()) - >>> TagSet(remove={'BIAS', 'CAL'}) - TagSet(add=set(), remove={'BIAS', 'CAL'}, blocked_by=set(), blocks=set(), if_present=set()) - - """ - def __new__(cls, add=None, remove=None, blocked_by=None, blocks=None, - if_present=None): - return super().__new__(cls, add or set(), - remove or set(), - blocked_by or set(), - blocks or set(), - if_present or set()) - - -def astro_data_descriptor(fn): - """ - Decorator that will mark a class method as an AstroData descriptor. - Useful to produce list of descriptors, for example. - - If used in combination with other decorators, this one *must* be the - one on the top (ie. the last one applying). It doesn't modify the - method in any other way. - - Args - ----- - fn : method - The method to be decorated - - Returns - -------- - The tagged method (not a wrapper) - """ - fn.descriptor_method = True - return fn - - -def returns_list(fn): - """ - Decorator to ensure that descriptors that should return a list (of one - value per extension) only returns single values when operating on - single slices; and vice versa. - - This is a common case, and you can use the decorator to simplify the - logic of your descriptors. - - Args - ----- - fn : method - The method to be decorated - - Returns - -------- - A function - """ - @wraps(fn) - def wrapper(self, *args, **kwargs): - ret = fn(self, *args, **kwargs) - if self.is_single: - if isinstance(ret, list): - # TODO: log a warning if the list is >1 element - if len(ret) > 1: - pass - return ret[0] - else: - return ret - else: - if isinstance(ret, list): - if len(ret) == len(self): - return ret - else: - raise IndexError( - "Incompatible numbers of extensions and elements in {}" - .format(fn.__name__)) - else: - return [ret] * len(self) - return wrapper - - -def assign_only_single_slice(fn): - """Raise `ValueError` if assigning to a non-single slice.""" - @wraps(fn) - def wrapper(self, *args, **kwargs): - if not self.is_single: - raise ValueError("Trying to assign to an AstroData object that " - "is not a single slice") - return fn(self, *args, **kwargs) - return wrapper - - -def astro_data_tag(fn): - """ - Decorator that marks methods of an `AstroData` derived class as part of the - tag-producing system. - - It wraps the method around a function that will ensure a consistent return - value: the wrapped method can return any sequence of sequences of strings, - and they will be converted to a TagSet. If the wrapped method - returns None, it will be turned into an empty TagSet. - - Args - ----- - fn : method - The method to be decorated - - Returns - -------- - A wrapper function - """ - @wraps(fn) - def wrapper(self): - try: - ret = fn(self) - if ret is not None: - if not isinstance(ret, TagSet): - raise TypeError("Tag function {} didn't return a TagSet" - .format(fn.__name__)) - - return TagSet(*tuple(set(s) for s in ret)) - except KeyError: - pass - - # Return empty TagSet for the "doesn't apply" case - return TagSet() - - wrapper.tag_method = True - return wrapper - - -class Section(tuple): - """A class to handle n-dimensional sections""" - - def __new__(cls, *args, **kwargs): - # Ensure that the order of keys is what we want - axis_names = [x for axis in "xyzuvw" - for x in (f"{axis}1", f"{axis}2")] - _dict = {k: v for k, v in zip(axis_names, args + - ('',) * len(kwargs))} - _dict.update(kwargs) - if list(_dict.values()).count('') or (len(_dict) % 2): - raise ValueError("Cannot initialize 'Section' object") - instance = tuple.__new__(cls, tuple(_dict.values())) - instance._axis_names = tuple(_dict.keys()) - if not all(np.diff(instance)[::2] > 0): - raise ValueError("Not all 'Section' end coordinates exceed the " - "start coordinates") - return instance - - @property - def __dict__(self): - return dict(zip(self._axis_names, self)) - - def __getnewargs__(self): - return tuple(self) - - def __getattr__(self, attr): - if attr in self._axis_names: - return self.__dict__[attr] - raise AttributeError(f"No such attribute '{attr}'") - - def __repr__(self): - return ("Section(" + - ", ".join([f"{k}={self.__dict__[k]}" - for k in self._axis_names]) + ")") - - @property - def ndim(self): - return len(self) // 2 - - @staticmethod - def from_shape(value): - """produce a Section object defining a given shape""" - return Section(*[y for x in reversed(value) for y in (0, x)]) - - @staticmethod - def from_string(value): - """The inverse of __str__, produce a Section object from a string""" - # if we were sent None, return None - if value is None: - return None - return Section(*[y for x in value.strip("[]").split(",") - for start, end in [x.split(":")] - for y in (None if start == '' else int(start)-1, - None if end == '' else int(end))]) - - def asIRAFsection(self, binning=None): - """Produce string of style '[x1:x2,y1:y2]' that is 1-indexed - and end-inclusive - - Parameters - ---------- - binning : iterable - A length-2 iterable of (x_binning, y_binning). Binning is assumed - to be 1 for all axes if not given. - """ - if binning is None: - binning = [1] * len(self._axis_names) - return ("[" + - ",".join([":".join([str(bin_*self.__dict__[axis]+1), - str(bin_*self.__dict__[axis.replace("1", "2")])]) - for axis, bin_ in zip(self._axis_names[::2], binning)]) - + "]") - - def asslice(self, add_dims=0): - """Return the Section object as a slice/list of slices. - Higher dimensionality can be achieved with the add_dims parameter.""" - return ((slice(None),) * add_dims + - tuple(slice(self.__dict__[axis], - self.__dict__[axis.replace("1", "2")]) - for axis in reversed(self._axis_names[::2]))) - - def contains(self, section): - """Return True if the supplied section is entirely within self""" - if self.ndim != section.ndim: - raise ValueError("Sections have different dimensionality") - return (all(s2 >= s1 for s1, s2 in zip(self[::2], section[::2])) and - all(s2 <= s1 for s1, s2 in zip(self[1::2], section[1::2]))) - - def is_same_size(self, section): - """Return True if the Sections are the same size""" - return np.array_equal(np.diff(self)[::2], np.diff(section)[::2]) - - def overlap(self, section): - """Determine whether the two sections overlap. If so, the Section - common to both is returned, otherwise None""" - if self.ndim != section.ndim: - raise ValueError("Sections have different dimensionality") - mins = [max(s1, s2) for s1, s2 in zip(self[::2], section[::2])] - maxs = [min(s1, s2) for s1, s2 in zip(self[1::2], section[1::2])] - try: - return self.__class__(*[v for pair in zip(mins, maxs) for v in pair]) - except ValueError: - return - - def shift(self, *shifts): - """Shift a section in each direction by the specified amount""" - if len(shifts) != self.ndim: - raise ValueError(f"Number of shifts {len(shifts)} incompatible " - f"with dimensionality {self.ndim}") - return self.__class__(*[x + s for x, s in - zip(self, [ss for s in shifts for ss in [s] * 2])]) diff --git a/astrodata/wcs.py b/astrodata/wcs.py deleted file mode 100644 index 898520eeff..0000000000 --- a/astrodata/wcs.py +++ /dev/null @@ -1,1033 +0,0 @@ -import functools -import re -from collections import namedtuple -from copy import deepcopy - -import numpy as np -from astropy import coordinates as coord -from astropy import units as u -from astropy.io import fits -from astropy.modeling import core, models, projections, CompoundModel -from astropy.table import Table -from gwcs import coordinate_frames as cf -from gwcs import utils as gwutils -from gwcs.utils import sky_pairs, specsystems -from gwcs.wcs import WCS as gWCS - -AffineMatrices = namedtuple("AffineMatrices", "matrix offset") - -FrameMapping = namedtuple("FrameMapping", "cls description") - -# Type of CoordinateFrame to construct for a FITS keyword and -# readable name so user knows what's going on -frame_mapping = {'WAVE': FrameMapping(cf.SpectralFrame, "Wavelength in vacuo"), - 'AWAV': FrameMapping(cf.SpectralFrame, "Wavelength in air")} - -re_ctype = re.compile("^CTYPE(\\d+)$", re.IGNORECASE) -re_cd = re.compile("^CD(\\d+)_\\d+$", re.IGNORECASE) - -#----------------------------------------------------------------------------- -# FITS-WCS -> gWCS -#----------------------------------------------------------------------------- -def pixel_frame(naxes, name="pixels"): - """ - Make a CoordinateFrame for pixels - - Parameters - ---------- - naxes: int - Number of axes - - Returns - ------- - CoordinateFrame - """ - axes_names = ('x', 'y', 'z', 'u', 'v', 'w')[:naxes] - return cf.CoordinateFrame(naxes=naxes, axes_type=['SPATIAL'] * naxes, - axes_order=tuple(range(naxes)), name=name, - axes_names=axes_names, unit=[u.pix] * naxes) - - -def fitswcs_to_gwcs(input): - """ - Create and return a gWCS object from a FITS header or NDData object. - If it can't construct one, it should quietly return None. - """ - # coordinate names for CelestialFrame - coordinate_outputs = {'alpha_C', 'delta_C'} - - # transform = gw.make_fitswcs_transform(hdr) - try: - transform = make_fitswcs_transform(input) - except Exception as e: - return - outputs = transform.outputs - try: - wcs_info = read_wcs_from_header(input.meta['header']) - except AttributeError: - wcs_info = read_wcs_from_header(input) - - in_frame = pixel_frame(transform.n_inputs) - out_frames = [] - for i, output in enumerate(outputs): - unit_name = wcs_info["CUNIT"][i] - try: - unit = u.Unit(unit_name) - except TypeError: - unit = None - try: - frame_type = output[:4].upper() - frame_info = frame_mapping[frame_type] - except KeyError: - if output in coordinate_outputs: - continue - frame = cf.CoordinateFrame(naxes=1, axes_type=("SPATIAL",), - axes_order=(i,), unit=unit, - axes_names=(output,), name=output) - else: - frame = frame_info.cls(axes_order=(i,), unit=unit, - axes_names=(frame_type,), - name=frame_info.description) - - out_frames.append(frame) - - if coordinate_outputs.issubset(outputs): - frame_name = wcs_info["RADESYS"] # FK5, etc. - axes_names = None - try: - ref_frame = getattr(coord, frame_name)() - # TODO? Work out how to stuff EQUINOX and OBS-TIME into the frame - except (AttributeError, TypeError): - # TODO: Replace quick fix as gWCS doesn't recognize GAPPT - if frame_name == "GAPPT": - ref_frame = coord.FK5() - else: - ref_frame = None - axes_names = ('lon', 'lat') - axes_order = (outputs.index('alpha_C'), outputs.index('delta_C')) - - # Call it 'world' if there are no other axes, otherwise 'sky' - name = 'SKY' if len(outputs) > 2 else 'world' - cel_frame = cf.CelestialFrame(reference_frame=ref_frame, name=name, - axes_names=axes_names, axes_order=axes_order) - out_frames.append(cel_frame) - - out_frame = (out_frames[0] if len(out_frames) == 1 - else cf.CompositeFrame(out_frames, name='world')) - return gWCS([(in_frame, transform), - (out_frame, None)]) - - -# ----------------------------------------------------------------------------- -# gWCS -> FITS-WCS -# ----------------------------------------------------------------------------- - -def gwcs_to_fits(ndd, hdr=None): - """ - Convert a gWCS object to a collection of FITS WCS keyword/value pairs, - if possible. If the FITS WCS is only approximate, this should be indicated - with a dict entry {'FITS-WCS': 'APPROXIMATE'}. If there is no suitable - FITS representation, then a ValueError or NotImplementedError can be - raised. - - Parameters - ---------- - ndd : `astropy.nddata.NDData` - The NDData whose wcs attribute we want converted - hdr : `astropy.io.fits.Header` - A Header object that may contain some useful keywords - - Returns - ------- - dict - values to insert into the FITS header to express this WCS - - """ - if hdr is None: - hdr = {} - - wcs = ndd.wcs - # Don't need to "copy" because any changes to transform use - # replace_submodel, which creates a new instance - transform = wcs.forward_transform - world_axes = list(wcs.output_frame.axes_names) - nworld_axes = len(world_axes) - tabular_axes = dict() - wcs_dict = {'NAXIS': len(ndd.shape), # in case it's not written to a file - 'WCSAXES': nworld_axes, - 'WCSDIM': nworld_axes} - wcs_dict.update({f'NAXIS{i}': length - for i, length in enumerate(ndd.shape[::-1], start=1)}) - wcs_dict.update({f'CD{i+1}_{j+1}': 0. for j in range(nworld_axes) - for i in range(nworld_axes)}) - pix_center = [0.5 * (length - 1) for length in ndd.shape[::-1]] - wcs_center = transform(*pix_center) - if nworld_axes == 1: - wcs_center = (wcs_center,) - - # Find and process the sky projection first - if {'lon', 'lat'}.issubset(world_axes): - if isinstance(wcs.output_frame, cf.CelestialFrame): - cel_frame = wcs.output_frame - elif isinstance(wcs.output_frame, cf.CompositeFrame): - for frame in wcs.output_frame.frames: - if isinstance(frame, cf.CelestialFrame): - cel_frame = frame - - # TODO: Non-ecliptic coordinate frames - cel_ref_frame = cel_frame.reference_frame - if not isinstance(cel_ref_frame, coord.builtin_frames.BaseRADecFrame): - raise NotImplementedError("Cannot write non-ecliptic frames yet") - wcs_dict['RADESYS'] = cel_ref_frame.name.upper() - - for m in transform: - if isinstance(m, models.RotateNative2Celestial): - nat2cel = m - if isinstance(m, models.Pix2SkyProjection): - m.name = 'pix2sky' - # Determine which sort of projection this is - for projcode in projections.projcodes: - if isinstance(m, getattr(models, f'Pix2Sky_{projcode}')): - break - else: - raise ValueError("Unknown projection class: {}". - format(m.__class__.__name__)) - - lon_axis = world_axes.index('lon') - lat_axis = world_axes.index('lat') - world_axes[lon_axis] = f'RA---{projcode}' - world_axes[lat_axis] = f'DEC--{projcode}' - wcs_dict[f'CRVAL{lon_axis+1}'] = nat2cel.lon.value - wcs_dict[f'CRVAL{lat_axis+1}'] = nat2cel.lat.value - - # Remove projection parts so we can calculate the CD matrix - if projcode: - nat2cel.name = 'nat2cel' - transform_inverse = transform.inverse - for m in transform_inverse: - if isinstance(m, models.RotateCelestial2Native): - m.name = 'cel2nat' - elif isinstance(m, models.Sky2PixProjection): - m.name = 'sky2pix' - transform_inverse = transform_inverse.replace_submodel('sky2pix', models.Identity(2)) - transform_inverse = transform_inverse.replace_submodel('cel2nat', models.Identity(2)) - transform = transform.replace_submodel('pix2sky', models.Identity(2)) - transform = transform.replace_submodel('nat2cel', models.Identity(2)) - transform.inverse = transform_inverse - - # Replace a log-linear axis with a linear axis representing the log - # and a Tabular axis with Identity to ensure the affinity check is passed - compound = isinstance(transform, CompoundModel) - if not compound: # just so we can iterate - transform = transform | models.Identity(nworld_axes) - for i in reversed(range(transform.n_submodels)): - m_this = transform[i] - if isinstance(m_this, models.Exponential1D): - if m_this.name is None: - m_this.name = "UNIQUE_NAME" - m_new = (models.Scale(1. / m_this.tau) | - models.Shift(np.log(m_this.amplitude))) - transform = transform.replace_submodel(m_this.name, m_new) - elif isinstance(m_this, (models.Tabular1D, models.Tabular2D)): - ndim = m_this.lookup_table.ndim - points = m_this.points - if not (ndim == 1 and np.allclose(points, np.arange(points[0].size)) or - ndim == 2 and np.allclose(points[0], np.arange(points[0].size)) - and np.allclose(points[1], np.arange(points[1].size))): - print("Tabular has different 'points' than expected") - continue - if m_this.name is None: - m_this.name = "UNIQUE_NAME" - tabular_axes[m_this.name] = m_this.lookup_table - # We need the model to produce CDij keywords that indicate which - # axes it depends on - if ndim == 1: - m_map = models.Identity(1) - else: - m_map = models.Mapping((0,), n_inputs=2) + models.Mapping((1,)) - m_map.inverse = models.Mapping((0, 0)) - transform = transform.replace_submodel(m_this.name, m_map) - if not compound: - transform = transform[:-1] - - # Deal with other axes - # TODO: AD should refactor to allow the descriptor to be used here - for i, axis_type in enumerate(wcs.output_frame.axes_type, start=1): - if f'CRVAL{i}' in wcs_dict: - continue - if axis_type == "SPECTRAL": - try: - wave_tab = tabular_axes["WAVE"] - except KeyError: - wcs_dict[f'CRVAL{i}'] = hdr.get('CENTWAVE', wcs_center[i-1]) - wcs_dict[f'CTYPE{i}'] = wcs.output_frame.axes_names[i-1] # AWAV/WAVE - else: - wcs_dict[f'CRVAL{i}'] = 0 - wcs_dict[f'CTYPE{i}'] = wcs.output_frame.axes_names[i-1][:4] + "-TAB" - if wave_tab.ndim == 1: # Greisen et al. (2006) - wcs_dict[f'PS{i}_0'] = wcs.output_frame.axes_names[i-1] - wcs_dict[f'PS{i}_1'] = ("WAVELENGTH", "Name of column") - wcs_dict[f'PS{i}_1'] = ("WAVELENGTH", "Name of column") - wcs_dict['extensions'] = {wcs.output_frame.axes_names[i-1]: - Table([wave_tab], names=('WAVELENGTH',))} - else: # make something up here - wcs_dict[f'PS{i}_0'] = wcs.output_frame.axes_names[i-1] - wcs_dict['extensions'] = {wcs.output_frame.axes_names[i-1]: wave_tab.T} - else: # Just something - wcs_dict[f'CRVAL{i}'] = wcs_center[i-1] - - # Flag if we can't construct a perfect CD matrix - if not model_is_affine(transform): - wcs_dict['FITS-WCS'] = ('APPROXIMATE', 'FITS WCS is approximate') - - affine = calculate_affine_matrices(transform, ndd.shape) - # Convert to x-first order - affine_matrix = np.flip(affine.matrix) - # Require an inverse to write out - wcs_dict.update({f'CD{i+1}_{j+1}': affine_matrix[i, j] - for j, _ in enumerate(ndd.shape) - for i, _ in enumerate(world_axes)}) - # Don't overwrite CTYPEi keywords we've already created - wcs_dict.update({f'CTYPE{i}': axis.upper()[:8] - for i, axis in enumerate(world_axes, start=1) - if f'CTYPE{i}' not in wcs_dict}) - - crval = [wcs_dict[f'CRVAL{i+1}'] for i, _ in enumerate(world_axes)] - try: - crval[lon_axis] = 0 - crval[lat_axis] = 0 - except NameError: - pass - - # Find any world axes that we previous logarithmed and fix the CDij - # matrix -- we follow FITS-III (Greisen et al. 2006; A&A 446, 747) - modified_wcs_center = transform(*pix_center) - if nworld_axes == 1: - modified_wcs_center = (modified_wcs_center,) - for world_axis, (wcs_val, modified_wcs_val) in enumerate( - zip(wcs_center, modified_wcs_center), start=1): - if wcs_val > 0 and np.isclose(modified_wcs_val, np.log(wcs_val)): - for j, _ in enumerate(ndd.shape, start=1): - wcs_dict[f'CD{world_axis}_{j}'] *= crval[world_axis-1] - wcs_dict[f'CTYPE{world_axis}'] = wcs_dict[f'CTYPE{world_axis}'][:4] + "-LOG" - crval[world_axis-1] = np.log(crval[world_axis-1]) - - # This (commented) line fails for un-invertable Tabular2D - #crpix = np.array(wcs.backward_transform(*crval)) + 1 - crpix = np.array(transform.inverse(*crval)) + 1 - - # Cope with a situation where the sky projection center is not in the slit - # We may be able to fix this in future, but FITS doesn't handle it well. - if len(ndd.shape) > 1: - crval2 = wcs(*(crpix - 1)) - try: - sky_center = coord.SkyCoord(nat2cel.lon.value, nat2cel.lat.value, unit=u.deg) - except NameError: - pass - else: - sky_center2 = coord.SkyCoord(crval2[lon_axis], crval2[lat_axis], unit=u.deg) - if sky_center.separation(sky_center2).arcsec > 0.01: - wcs_dict['FITS-WCS'] = ('APPROXIMATE', 'FITS WCS is approximate') - - if len(ndd.shape) == 1: - wcs_dict['CRPIX1'] = crpix - else: - # Comply with FITS standard, must define CRPIXj for "extra" axes - wcs_dict.update({f'CRPIX{j}': cpix for j, cpix in enumerate(np.concatenate([crpix, [1] * (nworld_axes-len(ndd.shape))]), start=1)}) - for i, unit in enumerate(wcs.output_frame.unit, start=1): - try: - wcs_dict[f'CUNIT{i}'] = unit.name - except AttributeError: - pass - - # To ensure an invertable CD matrix, we need to get nonexistent pixel axes - # "involved". - for j in range(len(ndd.shape), nworld_axes): - wcs_dict[f'CD{nworld_axes}_{j+1}'] = 1 - - return wcs_dict - -# ----------------------------------------------------------------------------- -# Helper functions -# ----------------------------------------------------------------------------- - -def model_is_affine(model): - """" - Test a Model for affinity. This is currently done by checking the - name of its class (or the class names of all its submodels) - - TODO: Is this the right thing to do? We could compute the affine - matrices *assuming* affinity, and then check that a number of random - points behave as expected. Is that better? - """ - if isinstance(model, dict): # handle fix_inputs() - return True - try: - return np.logical_and.reduce([model_is_affine(m) - for m in model]) - except TypeError: - # TODO: Delete "Const" one fix_inputs() broadcastingis fixed - return model.__class__.__name__[:5] in ('Affin', 'Rotat', 'Scale', - 'Shift', 'Ident', 'Mappi', - 'Const') - - -def calculate_affine_matrices(func, shape, origin=None): - """ - Compute the matrix and offset necessary of an affine transform that - represents the supplied function. This is done by computing the - linear matrix along all axes extending from the centre of the region, - and then calculating the offset such that the transformation is - accurate at the centre of the region. The matrix and offset are returned - in the standard python order (i.e., y-first for 2D). - - Parameters - ---------- - func : callable - function that maps input->output coordinates; these coordinates - are x-first, because "func" is usually an astropy.modeling.Model - shape : sequence - shape to use for fiducial points - origin : sequence/None - if a sequence, then use this as the opposite vertex (it must be - the same length as "shape") - - Returns - ------- - AffineMatrices(array, array) - affine matrix and offset - - """ - indim = len(shape) - try: - ndim = len(func(*shape)) # handle increase in number of axes - except TypeError: - ndim = 1 - if origin is None: - halfsize = [0.5 * length for length in shape] - else: - halfsize = [0.5 * (len1 + len2) - for len1, len2 in zip(origin, shape)] + [1.] - - points = np.array([halfsize] * (2 * indim + 1)).T - points[:, 1:indim + 1] += np.eye(indim) * points[:, 0] - points[:, indim + 1:] -= np.eye(indim) * points[:, 0] - if ndim > 1: - transformed = np.array(list(zip(*list(func(*point[indim-1::-1]) - for point in points.T)))).T - else: - transformed = np.array([func(*points)]).T - # Matrix of wcs derivatives wrt input coordinates in python order - matrix = np.array([[0.5 * (transformed[j + 1, i] - transformed[indim + j + 1, i]) / halfsize[j] - for j in range(indim-1, -1, -1)] for i in range(ndim)]) - offset = transformed[0] - np.dot(matrix, halfsize[::-1]) - - return AffineMatrices(matrix[::-1, ::-1], offset[::-1]) - - -# ------------------------------------------------------------------------- -# This stuff will hopefully all go into gwcs.utils -# ------------------------------------------------------------------------- -def read_wcs_from_header(header): - """ - Extract basic FITS WCS keywords from a FITS Header. - - Parameters - ---------- - header : `astropy.io.fits.Header` - FITS Header with WCS information. - - Returns - ------- - wcs_info : dict - A dictionary with WCS keywords. - """ - wcs_info = {} - - # NAXIS=0 if we're reading from a PHU - naxis = header.get('NAXIS') or max(int(k[5:]) for k in header['CRPIX*'].keys()) - wcs_info['NAXIS'] = naxis - try: - wcsaxes = header['WCSAXES'] - except KeyError: - wcsaxes = 0 - for kw in header["CTYPE*"]: - if re_ctype.match(kw): - wcsaxes = max(wcsaxes, int(re_ctype.match(kw).group(1)), naxis) - for kw in header["CD*_*"]: - if re_cd.match(kw): - wcsaxes = max(wcsaxes, int(re_cd.match(kw).group(1)), naxis) - wcs_info['WCSAXES'] = wcsaxes - # if not present call get_csystem - wcs_info['RADESYS'] = header.get('RADESYS', header.get('RADECSYS', 'FK5')) - wcs_info['VAFACTOR'] = header.get('VAFACTOR', 1) - # date keyword? - # wcs_info['DATEOBS'] = header.get('DATE-OBS', 'DATEOBS') - wcs_info['EQUINOX'] = header.get("EQUINOX", None) - wcs_info['EPOCH'] = header.get("EPOCH", None) - wcs_info['DATEOBS'] = header.get("MJD-OBS", header.get("DATE-OBS", None)) - - ctype = [] - cunit = [] - crpix = [] - crval = [] - cdelt = [] - # Handle more than 1 undefined (i.e., not CTYPEi) axis - untyped_axes = 0 - for i in range(1, wcsaxes + 1): - try: - this_ctype = header[f'CTYPE{i}'] - except KeyError: - this_ctype = f"LINEAR{untyped_axes+1 if untyped_axes else ''}" - untyped_axes += 1 - ctype.append(this_ctype) - cunit.append(header.get(f'CUNIT{i}', None)) - crpix.append(header.get(f'CRPIX{i}', 0.0)) - crval.append(header.get(f'CRVAL{i}', 0.0)) - cdelt.append(header.get(f'CDELT{i}', 1.0)) - - has_cd = len(header['CD?_?']) > 0 - cd = np.zeros((wcsaxes, naxis)) - for i in range(1, wcsaxes + 1): - for j in range(1, naxis + 1): - if has_cd: - cd[i - 1, j - 1] = header.get('CD{0}_{1}'.format(i, j), 0) - else: - cd[i - 1, j - 1] = cdelt[i - 1] * header.get('PC{0}_{1}'.format(i, j), - 1 if i == j else 0) - - # Hack to deal with non-FITS-compliant data where one axis is ignored - unspecified_pixel_axes = [axis for axis, unused in - enumerate(np.all(cd == 0, axis=0)) if unused] - if unspecified_pixel_axes: - unused_world_axes = [axis for axis, unused in - enumerate(np.all(cd == 0, axis=1)) if unused] - unused_world_axes += [wcsaxes - 1] * len(unspecified_pixel_axes) - for pixel_axis, world_axis in zip(unspecified_pixel_axes, unused_world_axes): - cd[world_axis, pixel_axis] = 1.0 - - wcs_info['CTYPE'] = ctype - wcs_info['CUNIT'] = cunit - wcs_info['CRPIX'] = crpix - wcs_info['CRVAL'] = crval - wcs_info['CD'] = cd - wcs_info.update({k: v for k, v in header.items() if k.startswith('PS')}) - return wcs_info - - -def get_axes(header): - """ - Matches input with spectral and sky coordinate axes. - - Parameters - ---------- - header : `astropy.io.fits.Header` or dict - FITS Header (or dict) with basic WCS information. - - Returns - ------- - sky_inmap, spectral_inmap, unknown : list - indices in the output representing sky and spectral coordinates. - - """ - if isinstance(header, fits.Header): - wcs_info = read_wcs_from_header(header) - elif isinstance(header, dict): - wcs_info = header - else: - raise TypeError("Expected a FITS Header or a dict.") - - # Split each CTYPE value at "-" and take the first part. - # This should represent the coordinate system. - ctype = [ax.split('-')[0].upper() for ax in wcs_info['CTYPE']] - sky_inmap = [] - spec_inmap = [] - unknown = [] - skysystems = np.array(list(sky_pairs.values())).flatten() - for ind, ax in enumerate(ctype): - if ax in specsystems: - spec_inmap.append(ind) - elif ax in skysystems: - sky_inmap.append(ind) - else: - unknown.append(ind) - - if len(sky_inmap) == 1: - unknown.append(sky_inmap.pop()) - - if sky_inmap: - _is_skysys_consistent(ctype, sky_inmap) - - return sky_inmap, spec_inmap, unknown - - -def _is_skysys_consistent(ctype, sky_inmap): - """ Determine if the sky axes in CTYPE match to form a standard celestial system.""" - if len(sky_inmap) != 2: - raise ValueError("{} sky coordinate axes found. " - "There must be exactly 2".format(len(sky_inmap))) - - for item in sky_pairs.values(): - if ctype[sky_inmap[0]] == item[0]: - if ctype[sky_inmap[1]] != item[1]: - raise ValueError( - "Inconsistent ctype for sky coordinates {0} and {1}".format(*ctype)) - break - elif ctype[sky_inmap[1]] == item[0]: - if ctype[sky_inmap[0]] != item[1]: - raise ValueError( - "Inconsistent ctype for sky coordinates {0} and {1}".format(*ctype)) - sky_inmap.reverse() - break - - -def _get_contributing_axes(wcs_info, world_axes): - """ - Returns a tuple indicating which axes in the pixel frame make a - contribution to an axis or axes in the output frame. - - Parameters - ---------- - wcs_info : dict - dict of WCS information - world_axes : int or iterable of int - axes in the world coordinate system - - Returns - ------- - axes : list - axes whose pixel coordinates affect the output axis/axes - """ - cd = wcs_info['CD'] - try: - return sorted(set(np.nonzero(cd[tuple(world_axes), :wcs_info['NAXIS']])[1])) - except TypeError: # world_axes is an int - return sorted(np.nonzero(cd[world_axes, :wcs_info['NAXIS']])[0]) - #return sorted(set(j for j in range(wcs_info['NAXIS']) - # for i in world_axes if cd[i, j] != 0)) - - -def make_fitswcs_transform(input): - """ - Create a basic FITS WCS transform. - It does not include distortions. - - Parameters - ---------- - header : `astropy.io.fits.Header` or dict - FITS Header (or dict) with basic WCS information - - """ - other = None - if isinstance(input, fits.Header): - wcs_info = read_wcs_from_header(input) - elif isinstance(input, dict): - wcs_info = input - else: - try: - wcs_info = read_wcs_from_header(input.meta['header']) - except AttributeError: - raise TypeError("Expected a FITS Header, dict, or NDData object") - else: - other = input.meta['other'] - - # If a pixel axis maps directly to an output axis, we want to have that - # model completely self-contained, so don't put all the CRPIXj shifts - # in a single CompoundModel at the beginning - transforms = [] - - # The tricky stuff! - sky_model = fitswcs_image(wcs_info) - other_models = fitswcs_other(wcs_info, other=other) - all_models = other_models - if sky_model: - i = -1 - for i, m in enumerate(all_models): - m.meta['output_axes'] = [i] - all_models.append(sky_model) - sky_model.meta['output_axes'] = [i+1, i+2] - - # Now arrange the models so the inputs and outputs are in the right places - all_models.sort(key=lambda m: m.meta['output_axes'][0]) - input_axes = [ax for m in all_models for ax in m.meta['input_axes']] - output_axes = [ax for m in all_models for ax in m.meta['output_axes']] - - if input_axes != list(range(len(input_axes))): - input_mapping = models.Mapping([max(x, 0) for x in input_axes]) - transforms.append(input_mapping) - - transforms.append(functools.reduce(core._model_oper('&'), all_models)) - - if output_axes != list(range(len(output_axes))): - output_mapping = models.Mapping(output_axes) - transforms.append(output_mapping) - - return functools.reduce(core._model_oper('|'), transforms) - - -def fitswcs_image(header): - """ - Make a complete transform from CRPIX-shifted pixels to - sky coordinates from FITS WCS keywords. A Mapping is inserted - at the beginning, which may be removed later - - Parameters - ---------- - header : `astropy.io.fits.Header` or dict - FITS Header or dict with basic FITS WCS keywords. - - """ - if isinstance(header, fits.Header): - wcs_info = read_wcs_from_header(header) - elif isinstance(header, dict): - wcs_info = header - else: - raise TypeError("Expected a FITS Header or a dict.") - - crpix = wcs_info['CRPIX'] - cd = wcs_info['CD'] - # get the part of the PC matrix corresponding to the imaging axes - sky_axes, spec_axes, unknown = get_axes(wcs_info) - if not sky_axes: - return - #if len(unknown) == 2: - # sky_axes = unknown - #else: # No sky here - # return - pixel_axes = _get_contributing_axes(wcs_info, sky_axes) - if len(pixel_axes) > 2: - raise ValueError("More than 2 pixel axes contribute to the sky coordinates") - - translation_models = [models.Shift(-(crpix[i] - 1), name='crpix' + str(i + 1)) - for i in pixel_axes] - translation = functools.reduce(lambda x, y: x & y, translation_models) - transforms = [translation] - - # If only one axis is contributing to the sky (e.g., slit spectrum) - # then it must be that there's an extra axis in the CD matrix, so we - # create a "ghost" orthogonal axis here so an inverse can be defined - # Modify the CD matrix in case we have to use a backup Matrix Model later - if len(pixel_axes) == 1: - sky_cd = np.array([[cd[sky_axes[0], pixel_axes[0]], -cd[sky_axes[1], pixel_axes[0]]], - [cd[sky_axes[1], pixel_axes[0]], cd[sky_axes[0], pixel_axes[0]]]]) - #cd[sky_axes[0], -1] = -cd[sky_axes[1], pixel_axes[0]] - #cd[sky_axes[1], -1] = cd[sky_axes[0], pixel_axes[0]] - #sky_cd = cd[np.ix_(sky_axes, pixel_axes + [-1])] - affine = models.AffineTransformation2D(matrix=sky_cd, name='cd_matrix') - # TODO: replace when PR#10362 is in astropy - #rotation = models.fix_inputs(affine, {'y': 0}) - rotation = models.Mapping((0, 0)) | models.Identity(1) & models.Const1D(0) | affine - rotation.inverse = affine.inverse | models.Mapping((0,), n_inputs=2) - else: - sky_cd = cd[np.ix_(sky_axes, pixel_axes)] - rotation = models.AffineTransformation2D(matrix=sky_cd, name='cd_matrix') - - # Do it this way so the whole CD matrix + projection is separable - projection = gwutils.fitswcs_nonlinear(wcs_info) - if projection: - rotation |= projection - transforms.append(rotation) - - sky_model = functools.reduce(lambda x, y: x | y, transforms) - sky_model.name = 'SKY' - sky_model.meta.update({'input_axes': pixel_axes, - 'output_axes': sky_axes}) - return sky_model - - -def fitswcs_other(header, other=None): - """ - Create WCS linear transforms for any axes not associated with - celestial coordinates. We require that each world axis aligns - precisely with only a single pixel axis. - - Parameters - ---------- - header : `astropy.io.fits.Header` or dict - FITS Header or dict with basic FITS WCS keywords. - - """ - # We *always* want the wavelength solution model to be called "WAVE" - # even if the CTYPE keyword is "AWAV" - model_name_mapping = {"AWAV": "WAVE"} - - if isinstance(header, fits.Header): - wcs_info = read_wcs_from_header(header) - elif isinstance(header, dict): - wcs_info = header - else: - raise TypeError("Expected a FITS Header or a dict.") - - cd = wcs_info['CD'] - crpix = wcs_info['CRPIX'] - crval = wcs_info['CRVAL'] - # get the part of the CD matrix corresponding to the imaging axes - sky_axes, spec_axes, unknown = get_axes(wcs_info) - #if not sky_axes and len(unknown) == 2: - # unknown = [] - - other_models = [] - for ax in spec_axes + unknown: - pixel_axes = _get_contributing_axes(wcs_info, ax) - ctype = wcs_info['CTYPE'][ax].upper() - if ctype.endswith("-TAB"): - table = None - if other is not None: - table_name = header.get(f'PS{ax + 1}_0') - table = other.get(table_name) - if table is None: - raise ValueError(f"Cannot read table for {ctype} for axis {ax}") - if isinstance(table, Table): - other_model = models.Tabular1D(lookup_table=table[header[f'PS{ax + 1}_1']]) - else: - other_model = models.Tabular2D(lookup_table=table.T) - other_model.name = model_name_mapping.get(ctype[:4], ctype[:4]) - del other[table_name] - elif len(pixel_axes) == 1: - pixel_axis = pixel_axes[0] - m1 = models.Shift(1 - crpix[pixel_axis], - name='crpix' + str(pixel_axis + 1)) - if ctype.endswith("-LOG"): - other_model = (m1 | models.Exponential1D( - amplitude=crval[ax], tau=crval[ax] / cd[ax, pixel_axis])) - ctype = ctype[:4] - else: - other_model = (m1 | models.Scale(cd[ax, pixel_axis]) | - models.Shift(crval[ax])) - other_model.name = model_name_mapping.get(ctype, ctype) - elif len(pixel_axes) == 0: - pixel_axes = [-1] - other_model = models.Const1D(crval[ax]) - other_model.inverse = models.Identity(1) - else: - raise ValueError(f"Axis {ax} depends on more than one input axis") - other_model.outputs = (ctype,) - other_model.meta.update({'input_axes': pixel_axes, - 'output_axes': [ax]}) - other_models.append(other_model) - - return other_models - - -def remove_axis_from_frame(frame, axis): - """ - Remove the numbered axis from a CoordinateFrame and return a modified - CoordinateFrame instance. - - Parameters - ---------- - frame: CoordinateFrame - The frame from which an axis is to be removed - axis: int - index of the axis to be removed - - Returns - ------- - CoordinateFrame: the modified frame - """ - if axis is None: - return frame - - if not isinstance(frame, cf.CompositeFrame): - if frame.name == "pixels" or frame.unit == (u.pix,) * frame.naxes: - return pixel_frame(frame.naxes - 1, name=frame.name) - else: - raise TypeError("Frame must be a CompositeFrame or pixel frame") - - new_frames = [] - for f in frame.frames: - if f.axes_order == (axis,): - continue - elif axis in f.axes_order: - new_frames.append(remove_axis_from_frame(f, axis)) - else: - nf = deepcopy(f) - nf._axes_order = tuple(x if x 1: - return cf.CompositeFrame(new_frames, name=frame.name) - raise ValueError("No frames left!") - - -def remove_axis_from_model(model, axis): - """ - Take a model where one output (axis) is no longer required and try to - construct a new model whether that output is removed. If the number of - inputs is reduced as a result, then report which input (axis) needs to - be removed. - - Parameters - ---------- - model: astropy.modeling.Model instance - model to modify - axis: int - Output axis number to be removed from the model - - Returns - ------- - tuple: Modified version of the model and input axis that is no longer - needed (input axis == None if completely removed) - """ - def is_identity(model): - """Determine whether a model does nothing and so can be removed""" - return (isinstance(model, models.Identity) or - isinstance(model, models.Mapping) and - tuple(model.mapping) == tuple(range(model.n_inputs))) - - if axis is None: - return model, None - - if isinstance(model, CompoundModel): - op = model.op - if op == "|": - new_right_model, input_axis = remove_axis_from_model(model.right, axis) - new_left_model, input_axis = remove_axis_from_model(model.left, input_axis) - if is_identity(new_left_model): - return new_right_model, input_axis - elif is_identity(new_right_model): - return new_left_model, input_axis - return (new_left_model | new_right_model), input_axis - elif op == "&": - nl_inputs = model.left.n_inputs - nr_inputs = model.right.n_inputs - if nl_inputs == 1 and axis == 0: - return model.right, 0 - elif nr_inputs == 1 and axis == nl_inputs: - return model.left, axis - elif axis < nl_inputs: - new_left_model, input_axis = remove_axis_from_model(model.left, axis) - return (new_left_model & model.right), input_axis - else: - new_right_model, input_axis = remove_axis_from_model(model.right, axis-nl_inputs) - return (model.left & new_right_model), (None if input_axis is None else input_axis+nl_inputs) - elif op in ("+", "-", "*", "/", "**"): - new_left_model, input_axis = remove_axis_from_model(model.left, axis) - new_right_model, input_axis2 = remove_axis_from_model(model.right, axis) - if input_axis != input_axis2: - raise ValueError("Different mappings on either side of an " - "arithmetic operator") - return functools.reduce(core._model_oper(op), - [new_left_model, new_right_model]), input_axis - elif op == "fix_inputs": - new_left_model, input_axis = remove_axis_from_model(model.left, axis) - fixed_inputs = model.right.copy() - if input_axis in fixed_inputs: - fixed_inputs.pop(input_axis) - input_axis = None - if fixed_inputs: - if input_axis is not None: - fixed_inputs = {(ax if ax < input_axis else ax-1): value - for ax, value in fixed_inputs.items()} - return core.fix_inputs(new_left_model, fixed_inputs), input_axis - else: - return new_left_model, input_axis - else: - raise ValueError(f"Cannot process operator {op}") - elif isinstance(model, models.Identity): - return models.Identity(model.n_inputs-1), axis - elif isinstance(model, models.Mapping): - mapping = model.mapping - input_axis = mapping[axis] - new_mapping = mapping[:axis] + mapping[axis+1:] - if input_axis not in new_mapping: - new_mapping = [ax if ax < input_axis else ax-1 for ax in new_mapping] - else: - input_axis = None - if new_mapping == list(range(len(new_mapping))): - return models.Identity(len(new_mapping)), input_axis - else: - return models.Mapping(tuple(new_mapping)), input_axis - - raise ValueError(f"Cannot process {model.__class__.__name__}") - - -def remove_unused_world_axis(ext): - """ - Remove a single axis from the output frame of the WCS if it has no - dependence on input pixel location. - - Parameters - ---------- - ext: single-slice AstroData object - """ - ndim = len(ext.shape) - if ext.wcs is None: - raise ValueError("The input has no WCS") - affine = calculate_affine_matrices(ext.wcs.forward_transform, ext.shape) - # Check whether there's a single output that isn't affected by the input - removable_axes = np.all(affine.matrix == 0, axis=1)[::-1] # xyz order - if removable_axes.sum() == 1: - output_axis = removable_axes.argmax() - else: - raise ValueError("No single degenerate output axis to remove") - - axis = output_axis - new_pipeline = [] - for step in reversed(ext.wcs.pipeline): - frame, transform = step.frame, step.transform - if transform is not None: - if axis < transform.n_outputs: - transform, axis = remove_axis_from_model(transform, axis) - if axis is not None and axis < frame.naxes: - frame = remove_axis_from_frame(frame, axis) - new_pipeline = [(frame, transform)] + new_pipeline - - if axis not in (ndim, None): - raise ValueError("Removed output axis does not trace back to removed" - " input axis") - - ext.wcs = gWCS(new_pipeline) - - -def create_new_image_projection(transform, new_center): - """ - Modifies a simple imaging transform - (Shift & Shift) | AffineTransformation2D | Pix2Sky | RotateNative2Celestial - so that the projection center is in a different sky location - - This works by rotating the AffineTransformation2D.matrix by the change in - angle (in Euclidean geometry) to the pole when moving from the original - projection center to the new one. The sign of this angle depends on whether - East is to the left or right when North is up. This works even when the - pole is on the image. - - This is accurate to <0.1 arcsec for shifts of up to 1 degree - - Parameters - ---------- - transform: Model - current forward imaging transform - new_center: tuple - (RA, DEC) coordinates of new projection center - - Returns - ------- - Model: a transform that is projected around the new center - """ - assert isinstance(transform[-1], models.RotateNative2Celestial) - assert isinstance(transform[-3], models.AffineTransformation2D) - current_center = transform[-1].lon.value, transform[-1].lat.value - xc, yc = transform.inverse(*current_center) - xcnew, ycnew = transform.inverse(*new_center) - xpole, ypole = transform.inverse(0, 90) - # astropy >=5.2 returns NaNs for a point not in the same hemisphere as - # the projection centre, so use the Celestial South Pole if required - if np.isnan(xpole) or np.isnan(ypole): - xpole, ypole = transform.inverse(0, -90) - angle1 = np.arctan2(xpole - xc, ypole - yc) - angle2 = np.arctan2(xpole - xcnew, ypole - ycnew) - rotation = (angle1 - angle2) * 180 / np.pi - matrix = transform[-3].matrix - # flipped means East is to the right when North is up - flipped = (matrix[0, 0] * matrix[1, 1] > 0 or - matrix[0, 1] * matrix[1, 0] < 0) - new_transform = deepcopy(transform[-3:]) - new_transform[0].matrix = models.Rotation2D(-rotation if flipped else rotation)(*matrix) - new_transform[-1].lon, new_transform[-1].lat = new_center - shifts = models.Shift(-xcnew, name='crpix1') & models.Shift(-ycnew, name='crpix2') - new_transform = shifts | new_transform - return new_transform diff --git a/gemini_instruments/bhros/__init__.py b/gemini_instruments/bhros/__init__.py index 7cf58aa60f..2591f01db1 100644 --- a/gemini_instruments/bhros/__init__.py +++ b/gemini_instruments/bhros/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataBhros -factory.addClass(AstroDataBhros) +factory.add_class(AstroDataBhros) diff --git a/gemini_instruments/cirpass/__init__.py b/gemini_instruments/cirpass/__init__.py index bb49382ed5..558c221689 100644 --- a/gemini_instruments/cirpass/__init__.py +++ b/gemini_instruments/cirpass/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataCirpass -factory.addClass(AstroDataCirpass) +factory.add_class(AstroDataCirpass) diff --git a/gemini_instruments/f2/__init__.py b/gemini_instruments/f2/__init__.py index 6e1ad5b906..b752fbb7db 100644 --- a/gemini_instruments/f2/__init__.py +++ b/gemini_instruments/f2/__init__.py @@ -5,5 +5,5 @@ from .adclass import AstroDataF2 from .lookup import filter_wavelengths -factory.addClass(AstroDataF2) +factory.add_class(AstroDataF2) addInstrumentFilterWavelengths('F2', filter_wavelengths) diff --git a/gemini_instruments/f2/tests/test_f2.py b/gemini_instruments/f2/tests/test_f2.py index 62f2b3fc0c..bd87788a92 100644 --- a/gemini_instruments/f2/tests/test_f2.py +++ b/gemini_instruments/f2/tests/test_f2.py @@ -31,7 +31,7 @@ def ad(request): filename = request.param file_path = astrodata.testing.download_from_archive(filename) - return astrodata.open(file_path) + return astrodata.from_file(file_path) @pytest.mark.dragons_remote_data diff --git a/gemini_instruments/flamingos/__init__.py b/gemini_instruments/flamingos/__init__.py index 9487c05a35..10ed3942c3 100644 --- a/gemini_instruments/flamingos/__init__.py +++ b/gemini_instruments/flamingos/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataFlamingos -factory.addClass(AstroDataFlamingos) +factory.add_class(AstroDataFlamingos) diff --git a/gemini_instruments/gemini/__init__.py b/gemini_instruments/gemini/__init__.py index 178bf79872..209d0a5afc 100644 --- a/gemini_instruments/gemini/__init__.py +++ b/gemini_instruments/gemini/__init__.py @@ -13,4 +13,4 @@ def addInstrumentFilterWavelengths(instrument, wl): } filter_wavelengths.update(update_dict) -factory.addClass(AstroDataGemini) +factory.add_class(AstroDataGemini) diff --git a/gemini_instruments/gemini/adclass.py b/gemini_instruments/gemini/adclass.py index 712480873b..8710b130a4 100644 --- a/gemini_instruments/gemini/adclass.py +++ b/gemini_instruments/gemini/adclass.py @@ -398,7 +398,7 @@ def _dec(self): def _parse_section(self, keyword, pretty): try: - value_filter = lambda x: (x.asIRAFsection() if pretty else x) + value_filter = lambda x: (x.as_iraf_section() if pretty else x) process_fn = lambda x: (None if x is None else value_filter(x)) # Dummy keyword FULLFRAME returns shape of full data array if keyword == 'FULLFRAME': diff --git a/gemini_instruments/gemini/tests/test_descriptors.py b/gemini_instruments/gemini/tests/test_descriptors.py index c7a96d0cba..8cff221246 100644 --- a/gemini_instruments/gemini/tests/test_descriptors.py +++ b/gemini_instruments/gemini/tests/test_descriptors.py @@ -116,7 +116,7 @@ def ad(request): filename = request.param path = astrodata.testing.download_from_archive(filename) - return astrodata.open(path) + return astrodata.from_file(path) @pytest.mark.dragons_remote_data diff --git a/gemini_instruments/ghost/__init__.py b/gemini_instruments/ghost/__init__.py index dc3e0abb7f..dc94e208e9 100644 --- a/gemini_instruments/ghost/__init__.py +++ b/gemini_instruments/ghost/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataGhost -factory.addClass(AstroDataGhost) +factory.add_class(AstroDataGhost) diff --git a/gemini_instruments/ghost/tests/test_ghost.py b/gemini_instruments/ghost/tests/test_ghost.py index 6272f5bd39..374aeb3fda 100644 --- a/gemini_instruments/ghost/tests/test_ghost.py +++ b/gemini_instruments/ghost/tests/test_ghost.py @@ -24,7 +24,7 @@ def ad(request): filename = request.param path = astrodata.testing.download_from_archive(filename) - return astrodata.open(path) + return astrodata.from_file(path) @pytest.mark.dragons_remote_data @@ -40,7 +40,7 @@ def test_can_return_ad_length(ad): @pytest.mark.dragons_remote_data def test_instrument(): path = astrodata.testing.download_from_archive("S20221209S0007.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) assert ad.phu['INSTRUME'] == 'GHOST' assert ad.instrument() == 'GHOST' @@ -48,7 +48,7 @@ def test_instrument(): @pytest.mark.dragons_remote_data def test_various_tags(): path = astrodata.testing.download_from_archive("S20221209S0007.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) # assert 'STD' in ad.tags #STD is no longer a tag assert 'GHOST' in ad.tags assert 'BUNDLE' in ad.tags @@ -57,7 +57,7 @@ def test_various_tags(): @pytest.mark.dragons_remote_data def test_detector_x_bin(): path = astrodata.testing.download_from_archive("S20221209S0007.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) xbin = ad.detector_x_bin() # should be a dict, since we are a bundle assert(isinstance(xbin, dict)) @@ -66,7 +66,7 @@ def test_detector_x_bin(): @pytest.mark.dragons_remote_data def test_ut_datetime(): path = astrodata.testing.download_from_archive("S20221209S0007.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) udt = ad.ut_datetime() # Check against expected UT Datetime, this descriptor also exercises the nascent PHU logic assert(abs(udt - datetime(2022, 12, 8, 20, 52, 22)) < timedelta(seconds=1)) @@ -75,7 +75,7 @@ def test_ut_datetime(): @pytest.mark.dragons_remote_data def test_data_label(): path = astrodata.testing.download_from_archive("S20221209S0007.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) dl = ad.data_label() # Check against expected UT Datetime, this descriptor also exercises the nascent PHU logic assert(dl == 'GS-ENG-GHOST-COM-3-123-001') @@ -84,21 +84,21 @@ def test_data_label(): @pytest.mark.dragons_remote_data def test_tab_bias(): path = astrodata.testing.download_from_archive("S20221208S0089.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) assert('BIAS' in ad.tags) @pytest.mark.dragons_remote_data def test_tab_flat(): path = astrodata.testing.download_from_archive("S20221209S0026.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) assert('FLAT' in ad.tags) @pytest.mark.dragons_remote_data def test_tab_arc(): path = astrodata.testing.download_from_archive("S20221208S0064.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) assert('ARC' in ad.tags) diff --git a/gemini_instruments/gmos/__init__.py b/gemini_instruments/gmos/__init__.py index 15d6322108..f858804cde 100644 --- a/gemini_instruments/gmos/__init__.py +++ b/gemini_instruments/gmos/__init__.py @@ -5,6 +5,6 @@ from .adclass import AstroDataGmos from .lookup import filter_wavelengths -factory.addClass(AstroDataGmos) +factory.add_class(AstroDataGmos) # Use the generic GMOS name for both GMOS-N and GMOS-S addInstrumentFilterWavelengths('GMOS', filter_wavelengths) diff --git a/gemini_instruments/gmos/tests/test_gmos.py b/gemini_instruments/gmos/tests/test_gmos.py index ba1befcd6d..2dc0c441df 100644 --- a/gemini_instruments/gmos/tests/test_gmos.py +++ b/gemini_instruments/gmos/tests/test_gmos.py @@ -30,7 +30,7 @@ def ad(request): filename = request.param path = astrodata.testing.download_from_archive(filename) - return astrodata.open(path) + return astrodata.from_file(path) @pytest.mark.dragons_remote_data @@ -71,7 +71,7 @@ def test_tag_as_standard_fake(astrofaker): @pytest.mark.dragons_remote_data def test_tag_as_standard_real(): path = astrodata.testing.download_from_archive("S20190215S0188.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) assert 'STANDARD' in ad.tags diff --git a/gemini_instruments/gnirs/__init__.py b/gemini_instruments/gnirs/__init__.py index da7fcd1c2b..2d904d21dc 100644 --- a/gemini_instruments/gnirs/__init__.py +++ b/gemini_instruments/gnirs/__init__.py @@ -5,5 +5,5 @@ from .adclass import AstroDataGnirs from .lookup import filter_wavelengths -factory.addClass(AstroDataGnirs) +factory.add_class(AstroDataGnirs) addInstrumentFilterWavelengths('GNIRS', filter_wavelengths) diff --git a/gemini_instruments/gnirs/tests/test_gnirs.py b/gemini_instruments/gnirs/tests/test_gnirs.py index 810f75acee..e747490809 100644 --- a/gemini_instruments/gnirs/tests/test_gnirs.py +++ b/gemini_instruments/gnirs/tests/test_gnirs.py @@ -44,7 +44,7 @@ def ad(request): """ filename = request.param path = astrodata.testing.download_from_archive(filename) - return astrodata.open(path) + return astrodata.from_file(path) @pytest.mark.xfail(reason="AstroFaker changes the AstroData factory") @@ -98,7 +98,7 @@ def test_slice_range(ad): # def test_read_a_keyword_from_phu(path_to_inputs): # -# ad = astrodata.open(os.path.join(path_to_inputs, filename)) +# ad = astrodata.from_file(os.path.join(path_to_inputs, filename)) # assert ad.phu['DETECTOR'] == 'GNIRS' @pytest.mark.dragons_remote_data @@ -182,7 +182,7 @@ def test_ra_dec_from_text(): @pytest.mark.parametrize("filename,expected_fpm", EXPECTED_FPMS) def test_ifu_fpm(filename, expected_fpm): path = astrodata.testing.download_from_archive(filename) - ad = astrodata.open(path) + ad = astrodata.from_file(path) assert("IFU" in ad.tags) assert(ad.focal_plane_mask(pretty=True) == expected_fpm) diff --git a/gemini_instruments/gpi/__init__.py b/gemini_instruments/gpi/__init__.py index eb32a258e5..b33664f49e 100644 --- a/gemini_instruments/gpi/__init__.py +++ b/gemini_instruments/gpi/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataGpi -factory.addClass(AstroDataGpi) +factory.add_class(AstroDataGpi) diff --git a/gemini_instruments/graces/__init__.py b/gemini_instruments/graces/__init__.py index c39a71804b..5599976f75 100644 --- a/gemini_instruments/graces/__init__.py +++ b/gemini_instruments/graces/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataGraces -factory.addClass(AstroDataGraces) +factory.add_class(AstroDataGraces) diff --git a/gemini_instruments/graces/tests/test_graces.py b/gemini_instruments/graces/tests/test_graces.py index f1c009c1bf..ce5c049507 100644 --- a/gemini_instruments/graces/tests/test_graces.py +++ b/gemini_instruments/graces/tests/test_graces.py @@ -10,7 +10,7 @@ @pytest.fixture def ad(): path = astrodata.testing.download_from_archive(filename) - return astrodata.open(path) + return astrodata.from_file(path) @pytest.mark.xfail(reason="AstroFaker changes the AstroData factory") diff --git a/gemini_instruments/gsaoi/__init__.py b/gemini_instruments/gsaoi/__init__.py index cd7014d400..247da99cf1 100644 --- a/gemini_instruments/gsaoi/__init__.py +++ b/gemini_instruments/gsaoi/__init__.py @@ -5,5 +5,5 @@ from .adclass import AstroDataGsaoi from .lookup import filter_wavelengths -factory.addClass(AstroDataGsaoi) +factory.add_class(AstroDataGsaoi) addInstrumentFilterWavelengths('GSAOI', filter_wavelengths) diff --git a/gemini_instruments/hokupaa_quirc/__init__.py b/gemini_instruments/hokupaa_quirc/__init__.py index f8dcbd523b..9821f08201 100644 --- a/gemini_instruments/hokupaa_quirc/__init__.py +++ b/gemini_instruments/hokupaa_quirc/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataHokupaaQUIRC -factory.addClass(AstroDataHokupaaQUIRC) +factory.add_class(AstroDataHokupaaQUIRC) diff --git a/gemini_instruments/hrwfs/__init__.py b/gemini_instruments/hrwfs/__init__.py index b2b7775900..3f23ad8217 100644 --- a/gemini_instruments/hrwfs/__init__.py +++ b/gemini_instruments/hrwfs/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataHrwfs -factory.addClass(AstroDataHrwfs) +factory.add_class(AstroDataHrwfs) diff --git a/gemini_instruments/igrins/__init__.py b/gemini_instruments/igrins/__init__.py index e827941abc..c43876316f 100644 --- a/gemini_instruments/igrins/__init__.py +++ b/gemini_instruments/igrins/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataIgrins -factory.addClass(AstroDataIgrins) +factory.add_class(AstroDataIgrins) diff --git a/gemini_instruments/michelle/__init__.py b/gemini_instruments/michelle/__init__.py index 848408e5c3..d102379627 100644 --- a/gemini_instruments/michelle/__init__.py +++ b/gemini_instruments/michelle/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataMichelle -factory.addClass(AstroDataMichelle) +factory.add_class(AstroDataMichelle) diff --git a/gemini_instruments/nici/__init__.py b/gemini_instruments/nici/__init__.py index 8ca39e4353..ba6b68d5cc 100644 --- a/gemini_instruments/nici/__init__.py +++ b/gemini_instruments/nici/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataNici -factory.addClass(AstroDataNici) +factory.add_class(AstroDataNici) diff --git a/gemini_instruments/nifs/__init__.py b/gemini_instruments/nifs/__init__.py index 0c2e67163a..54430e6595 100644 --- a/gemini_instruments/nifs/__init__.py +++ b/gemini_instruments/nifs/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataNifs -factory.addClass(AstroDataNifs) +factory.add_class(AstroDataNifs) diff --git a/gemini_instruments/nifs/tests/test_nifs.py b/gemini_instruments/nifs/tests/test_nifs.py index 3a7edf1080..10c720aa12 100644 --- a/gemini_instruments/nifs/tests/test_nifs.py +++ b/gemini_instruments/nifs/tests/test_nifs.py @@ -13,7 +13,7 @@ def ad(request): filename = request.param path = astrodata.testing.download_from_archive(filename) - return astrodata.open(path) + return astrodata.from_file(path) @pytest.mark.dragons_remote_data diff --git a/gemini_instruments/niri/__init__.py b/gemini_instruments/niri/__init__.py index 093a69140d..c81bf84052 100644 --- a/gemini_instruments/niri/__init__.py +++ b/gemini_instruments/niri/__init__.py @@ -5,5 +5,5 @@ from .adclass import AstroDataNiri from .lookup import filter_wavelengths -factory.addClass(AstroDataNiri) +factory.add_class(AstroDataNiri) addInstrumentFilterWavelengths('NIRI', filter_wavelengths) diff --git a/gemini_instruments/niri/tests/test_niri.py b/gemini_instruments/niri/tests/test_niri.py index 6eef03c991..2692a687c3 100644 --- a/gemini_instruments/niri/tests/test_niri.py +++ b/gemini_instruments/niri/tests/test_niri.py @@ -11,7 +11,7 @@ @pytest.fixture() def ad(): path = astrodata.testing.download_from_archive(filename) - return astrodata.open(path) + return astrodata.from_file(path) @pytest.mark.xfail(reason="AstroFaker changes the AstroData factory") diff --git a/gemini_instruments/oscir/__init__.py b/gemini_instruments/oscir/__init__.py index 23e0dbc7ce..c48695fd0f 100644 --- a/gemini_instruments/oscir/__init__.py +++ b/gemini_instruments/oscir/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataOscir -factory.addClass(AstroDataOscir) +factory.add_class(AstroDataOscir) diff --git a/gemini_instruments/phoenix/__init__.py b/gemini_instruments/phoenix/__init__.py index e4864d1b8b..fc890aa080 100644 --- a/gemini_instruments/phoenix/__init__.py +++ b/gemini_instruments/phoenix/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataPhoenix -factory.addClass(AstroDataPhoenix) +factory.add_class(AstroDataPhoenix) diff --git a/gemini_instruments/skycam/__init__.py b/gemini_instruments/skycam/__init__.py index d53c9d365e..95fce3c9c6 100644 --- a/gemini_instruments/skycam/__init__.py +++ b/gemini_instruments/skycam/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataSkyCam -factory.addClass(AstroDataSkyCam) +factory.add_class(AstroDataSkyCam) diff --git a/gemini_instruments/test/test_astrodata_descriptors.py b/gemini_instruments/test/test_astrodata_descriptors.py index 674b79f9f0..c0039a0b94 100644 --- a/gemini_instruments/test/test_astrodata_descriptors.py +++ b/gemini_instruments/test/test_astrodata_descriptors.py @@ -34,11 +34,11 @@ def test_descriptor(instr, filename, descriptor, value): if '_' in filename: filepath = os.path.join(path_to_test_data, filename) try: - ad = astrodata.open(filepath) + ad = astrodata.from_file(filepath) except FileNotFoundError: pytest.skip(f"{filename} not found") else: - ad = astrodata.open(astrodata.testing.download_from_archive(filename)) + ad = astrodata.from_file(astrodata.testing.download_from_archive(filename)) method = getattr(ad, descriptor) if value is None: diff --git a/gemini_instruments/test/test_astrodata_tags.py b/gemini_instruments/test/test_astrodata_tags.py index a337ab1c35..9db840976e 100644 --- a/gemini_instruments/test/test_astrodata_tags.py +++ b/gemini_instruments/test/test_astrodata_tags.py @@ -23,10 +23,10 @@ def test_descriptor(instr, filename, tag_set): if '_' in filename: filepath = os.path.join(path_to_test_data, filename) try: - ad = astrodata.open(filepath) + ad = astrodata.from_file(filepath) except FileNotFoundError: pytest.skip(f"{filename} not found") else: - ad = astrodata.open(astrodata.testing.download_from_archive(filename)) + ad = astrodata.from_file(astrodata.testing.download_from_archive(filename)) assert ad.tags == set(tag_set) diff --git a/gemini_instruments/texes/__init__.py b/gemini_instruments/texes/__init__.py index cad7685f80..55f9cb41f0 100644 --- a/gemini_instruments/texes/__init__.py +++ b/gemini_instruments/texes/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataTexes -factory.addClass(AstroDataTexes) +factory.add_class(AstroDataTexes) diff --git a/gemini_instruments/trecs/__init__.py b/gemini_instruments/trecs/__init__.py index 514af046d1..d7fcb9df35 100644 --- a/gemini_instruments/trecs/__init__.py +++ b/gemini_instruments/trecs/__init__.py @@ -3,4 +3,4 @@ from astrodata import factory from .adclass import AstroDataTrecs -factory.addClass(AstroDataTrecs) +factory.add_class(AstroDataTrecs) diff --git a/geminidr/core/primitives_bookkeeping.py b/geminidr/core/primitives_bookkeeping.py index 0daf952db0..91b4232c50 100644 --- a/geminidr/core/primitives_bookkeeping.py +++ b/geminidr/core/primitives_bookkeeping.py @@ -211,7 +211,7 @@ def is_lazy(ad): # the files to retain their orig_filename attributes, which # would otherwise change upon loading. orig_filename = ad.orig_filename - adinputs[i] = astrodata.open(ad.filename) + adinputs[i] = astrodata.from_file(ad.filename) adinputs[i].orig_filename = orig_filename return adinputs @@ -259,7 +259,7 @@ def getList(self, adinputs=None, **params): adinputs = [] for f in all_files: try: - adinputs.insert(0, astrodata.open(f)) + adinputs.insert(0, astrodata.from_file(f)) except astrodata.AstroDataError: log.stdinfo(" Cannot open {}".format(f)) if len(adinputs) >= max_frames: diff --git a/geminidr/core/primitives_crossdispersed.py b/geminidr/core/primitives_crossdispersed.py index 6825eb0555..6027abed47 100644 --- a/geminidr/core/primitives_crossdispersed.py +++ b/geminidr/core/primitives_crossdispersed.py @@ -220,7 +220,7 @@ def _cut_slits(self, ad, padding=0): else: cut_section = Section(x1=0, x2=ext.shape[1], y1=y1, y2=y2) log.stdinfo(f"Cutting slit {i+1} in extension {ext.id} " - f"from {cut_section.asIRAFsection()}") + f"from {cut_section.as_iraf_section()}") adout.append(deepcopy(ext.nddata[cut_section.asslice()])) adout[-1].SLITEDGE = slitedge[i*2:i*2+2] adout[-1].SLITEDGE["c0"] -= y1 @@ -280,7 +280,7 @@ def _cut_slits(self, ad, padding=0): # (e.g., MOS data from MOS), and will need to update # the array_section keyword then as well. adout[-1].hdr[detsec_kw] = ( - cut_section.asIRAFsection(binning=binnings), + cut_section.as_iraf_section(binning=binnings), self.keyword_comments.get(detsec_kw)) return adout diff --git a/geminidr/core/primitives_image.py b/geminidr/core/primitives_image.py index 86383f549a..03a3d216c9 100644 --- a/geminidr/core/primitives_image.py +++ b/geminidr/core/primitives_image.py @@ -412,7 +412,7 @@ def resampleToCommonFrame(self, adinputs=None, **params): raise OSError("All input images must have only one extension.") if isinstance(reference, str): - reference = astrodata.open(reference) + reference = astrodata.from_file(reference) elif reference is None and pixel_scale is None: # Reference image will be the first AD, so we need 2+ if len(adinputs) < 2: @@ -621,7 +621,7 @@ def transferObjectMask(self, adinputs=None, **params): source_stream = self.streams[source] except KeyError: try: - ad_source = astrodata.open(source) + ad_source = astrodata.from_file(source) except: log.warning(f"Cannot find stream or file named {source}. Continuing.") return adinputs diff --git a/geminidr/core/primitives_preprocess.py b/geminidr/core/primitives_preprocess.py index 46a42fba55..4f9391fb2a 100644 --- a/geminidr/core/primitives_preprocess.py +++ b/geminidr/core/primitives_preprocess.py @@ -269,7 +269,7 @@ def sky_coord(ad): # Produce a list of AD objects from the sky frame/list ad_skies = sky if isinstance(sky, list) else [sky] ad_skies = [ad if isinstance(ad, astrodata.AstroData) else - astrodata.open(ad) for ad in ad_skies] + astrodata.from_file(ad) for ad in ad_skies] else: # get from sky stream (put there by separateSky) ad_skies = self.streams.get('sky', []) @@ -1541,7 +1541,7 @@ def skyCorrect(self, adinputs=None, **params): break else: try: - sky = astrodata.open(filename) + sky = astrodata.from_file(filename) except astrodata.AstroDataError: log.warning(f"Cannot find a sky file named {filename}. " "Ignoring it.") diff --git a/geminidr/core/primitives_stack.py b/geminidr/core/primitives_stack.py index f77bbda928..edeeed0ddd 100644 --- a/geminidr/core/primitives_stack.py +++ b/geminidr/core/primitives_stack.py @@ -4,7 +4,7 @@ # primitives_stack.py # ------------------------------------------------------------------------------ import astrodata -from astrodata.fits import windowedOp +from astrodata.fits import windowed_operation import numpy as np from astropy import table @@ -332,7 +332,7 @@ def flatten(*args): with_uncertainty = True # Since all stacking methods return variance with_mask = apply_dq and not any(ad[index].nddata.window[:].mask is None for ad in adinputs) - result = windowedOp(stack_function, + result = windowed_operation(stack_function, [ad[index].nddata for ad in adinputs], scale=sfactors, zero=zfactors, diff --git a/geminidr/core/tests/test_bookkeeping.py b/geminidr/core/tests/test_bookkeeping.py index 7f6386758d..1fac7792e2 100644 --- a/geminidr/core/tests/test_bookkeeping.py +++ b/geminidr/core/tests/test_bookkeeping.py @@ -144,7 +144,7 @@ def test_addToList(self): filenames = ['N20070819S{:04d}_flatCorrected.fits'.format(i) for i in range(104, 109)] - adinputs = [astrodata.open(os.path.join(TESTDATAPATH, 'NIRI', f)) + adinputs = [astrodata.from_file(os.path.join(TESTDATAPATH, 'NIRI', f)) for f in filenames] # Add one image twice, just for laughs; it should appear only once @@ -180,7 +180,7 @@ def test_writeOutputs(self): filenames = ['N20070819S{:04d}_flatCorrected.fits'.format(i) for i in range(104, 106)] - adinputs = [astrodata.open(os.path.join(TESTDATAPATH, 'NIRI', f)) + adinputs = [astrodata.from_file(os.path.join(TESTDATAPATH, 'NIRI', f)) for f in filenames] p = NIRIImage(adinputs) diff --git a/geminidr/core/tests/test_ccd.py b/geminidr/core/tests/test_ccd.py index 8c5c42867e..6ab9611ea7 100644 --- a/geminidr/core/tests/test_ccd.py +++ b/geminidr/core/tests/test_ccd.py @@ -34,5 +34,5 @@ def test_saturation_level_modification_in_overscan_correct(raw_ad): @pytest.fixture(scope='function') def raw_ad(request): filename = request.param - raw_ad = astrodata.open(download_from_archive(filename)) + raw_ad = astrodata.from_file(download_from_archive(filename)) return raw_ad diff --git a/geminidr/core/tests/test_image.py b/geminidr/core/tests/test_image.py index e8b07b65e9..918906fe01 100644 --- a/geminidr/core/tests/test_image.py +++ b/geminidr/core/tests/test_image.py @@ -20,13 +20,13 @@ def test_transfer_object_mask(path_to_inputs, path_to_refs, dataset): """ Test the transferObjectMask primitive """ - ad_donor = astrodata.open(os.path.join(path_to_inputs, dataset)) - ad_target = astrodata.open(os.path.join(path_to_inputs, object_mask_datasets[dataset])) + ad_donor = astrodata.from_file(os.path.join(path_to_inputs, dataset)) + ad_target = astrodata.from_file(os.path.join(path_to_inputs, object_mask_datasets[dataset])) p = NIRIImage([ad_target]) p.streams['donor'] = [ad_donor] adout = p.transferObjectMask(source="donor", dq_threshold=0.01, dilation=1.5, interpolant="linear").pop() adout.write(overwrite=True) - adref = astrodata.open(os.path.join(path_to_refs, adout.filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, adout.filename)) assert_array_equal(adout[0].OBJMASK, adref[0].OBJMASK) diff --git a/geminidr/core/tests/test_nearIR.py b/geminidr/core/tests/test_nearIR.py index 35b1b47130..df82daa1f3 100644 --- a/geminidr/core/tests/test_nearIR.py +++ b/geminidr/core/tests/test_nearIR.py @@ -113,7 +113,7 @@ def test_remove_first_frame_by_filename(): "S20070131S0105", # GNIRS XD spectrum ]) def test_clean_readout(in_file, path_to_inputs, path_to_refs): - ad = astrodata.open(os.path.join(path_to_inputs, + ad = astrodata.from_file(os.path.join(path_to_inputs, in_file + '_skyCorrected.fits')) # Must use the correct default parameters, since this is a test that the @@ -124,7 +124,7 @@ def test_clean_readout(in_file, path_to_inputs, path_to_refs): p = pclass([ad]) ad_out = p.cleanReadout(clean="default")[0] - ref = astrodata.open(os.path.join(path_to_refs, ad.filename)) + ref = astrodata.from_file(os.path.join(path_to_refs, ad.filename)) assert ad_compare(ad_out, ref, atol=0.01) @@ -146,7 +146,7 @@ def test_clean_readout(in_file, path_to_inputs, path_to_refs): "N20231112S0136", # GNIRS LS ]) def test_clean_fftreadout(in_file, path_to_inputs, path_to_refs): - ad = astrodata.open(os.path.join(path_to_inputs, in_file + '_skyCorrected.fits')) + ad = astrodata.from_file(os.path.join(path_to_inputs, in_file + '_skyCorrected.fits')) # Must use the correct default parameters, since this is a test that the # defaults haven't changed pm = PrimitiveMapper(ad.tags, ad.instrument(generic=True).lower(), @@ -154,5 +154,5 @@ def test_clean_fftreadout(in_file, path_to_inputs, path_to_refs): pclass = pm.get_applicable_primitives() p = pclass([ad]) ad_out = p.cleanFFTReadout(clean="default")[0] - ref = astrodata.open(os.path.join(path_to_refs, in_file + '_readoutFFTCleaned.fits')) + ref = astrodata.from_file(os.path.join(path_to_refs, in_file + '_readoutFFTCleaned.fits')) assert ad_compare(ad_out, ref, atol=0.01) diff --git a/geminidr/core/tests/test_preprocess.py b/geminidr/core/tests/test_preprocess.py index 53ae544a60..1498c9f60a 100644 --- a/geminidr/core/tests/test_preprocess.py +++ b/geminidr/core/tests/test_preprocess.py @@ -38,7 +38,7 @@ def niri_images(niri_image): @pytest.fixture def niriprim(): file_path = download_from_archive("N20190120S0287.fits") - ad = astrodata.open(file_path) + ad = astrodata.from_file(file_path) p = NIRIImage([ad]) p.addDQ(static_bpm=download_from_archive("bpm_20010317_niri_niri_11_full_1amp.fits")) return p @@ -47,7 +47,7 @@ def niriprim(): @pytest.fixture def niriprim2(): file_path = download_from_archive("N20190120S0287.fits") - ad = astrodata.open(file_path) + ad = astrodata.from_file(file_path) ad.append(ad[0]) p = NIRIImage([ad]) p.addDQ() @@ -424,10 +424,10 @@ def test_fixpixels_multiple_ext(niriprim2): #def test_nonlinearity_correct(path_to_inputs, path_to_refs, dataset): def test_nonlinearity_correct(path_to_inputs, path_to_refs, dataset): """Only GSAOI uses the core primitive with real coefficients""" - ad = astrodata.open(os.path.join(path_to_inputs, dataset[0])) + ad = astrodata.from_file(os.path.join(path_to_inputs, dataset[0])) p = GSAOIImage([ad]) ad_out = p.nonlinearityCorrect().pop() - ad_ref = astrodata.open(os.path.join(path_to_refs, dataset[1])) + ad_ref = astrodata.from_file(os.path.join(path_to_refs, dataset[1])) assert ad_compare(ad_out, ad_ref, ignore=['filename']) @@ -456,7 +456,7 @@ def test_scale_by_exposure_time(niri_images): # def test_add_object_mask_to_dq(astrofaker): # ad_orig = astrofaker.create('F2', 'IMAGE') -# # astrodata.open(os.path.join(TESTDATAPATH, 'GMOS', 'N20150624S0106_refcatAdded.fits')) +# # astrodata.from_file(os.path.join(TESTDATAPATH, 'GMOS', 'N20150624S0106_refcatAdded.fits')) # p = GMOSImage([deepcopy(ad_orig)]) # ad = p.addObjectMaskToDQ()[0] @@ -468,7 +468,7 @@ def test_scale_by_exposure_time(niri_images): # @pytest.mark.xfail(reason="Test needs revision", run=False) # def test_adu_to_electrons(astrofaker): # ad = astrofaker.create("NIRI", "IMAGE") -# # astrodata.open(os.path.join(TESTDATAPATH, 'NIRI', 'N20070819S0104_dqAdded.fits')) +# # astrodata.from_file(os.path.join(TESTDATAPATH, 'NIRI', 'N20070819S0104_dqAdded.fits')) # p = NIRIImage([ad]) # ad = p.ADUToElectrons()[0] # assert ad_compare(ad, os.path.join(TESTDATAPATH, 'NIRI', @@ -563,7 +563,7 @@ def test_associate_sky_exclude_some(niri_image, niri_sequence): # pass # def test_darkCorrect(self): -# ad = astrodata.open(os.path.join(TESTDATAPATH, 'NIRI', +# ad = astrodata.from_file(os.path.join(TESTDATAPATH, 'NIRI', # 'N20070819S0104_nonlinearityCorrected.fits')) # p = NIRIImage([ad]) # ad = p.darkCorrect()[0] @@ -584,7 +584,7 @@ def test_darkCorrect_with_af(astrofaker): # af.init_default_extensions() # af[0].mask = np.zeros_like(af[0].data, dtype=np.uint16) # def test_flatCorrect(self): -# ad = astrodata.open(os.path.join(TESTDATAPATH, 'NIRI', +# ad = astrodata.from_file(os.path.join(TESTDATAPATH, 'NIRI', # 'N20070819S0104_darkCorrected.fits')) # p = NIRIImage([ad]) # ad = p.flatCorrect()[0] @@ -597,7 +597,7 @@ def test_darkCorrect_with_af(astrofaker): # def test_normalizeFlat(self): # flat_file = os.path.join(TESTDATAPATH, 'NIRI', # 'N20070913S0220_flat.fits') -# ad = astrodata.open(flat_file) +# ad = astrodata.from_file(flat_file) # ad.multiply(10.0) # del ad.phu['NORMLIZE'] # Delete timestamp of previous processing # p = NIRIImage([ad]) @@ -814,7 +814,7 @@ def test_separate_sky_proximity(groups, niri_sequence): # pass # # def test_subtractSkyBackground(self): -# ad = astrodata.open(os.path.join(TESTDATAPATH, 'NIRI', +# ad = astrodata.from_file(os.path.join(TESTDATAPATH, 'NIRI', # 'N20070819S0104_flatCorrected.fits')) # ad.hdr['SKYLEVEL'] = 1000.0 # orig_data = ad[0].data.copy() @@ -824,7 +824,7 @@ def test_separate_sky_proximity(groups, niri_sequence): # assert (orig_data - ad[0].data).max() < 1000.01 # # def test_thresholdFlatfield(self): -# ad = astrodata.open(os.path.join(TESTDATAPATH, 'NIRI', +# ad = astrodata.from_file(os.path.join(TESTDATAPATH, 'NIRI', # 'N20070913S0220_flat.fits')) # del ad.phu['TRHFLAT'] # Delete timestamp of previous processing # ad[0].data[100, 100] = 20.0 diff --git a/geminidr/core/tests/test_spect.py b/geminidr/core/tests/test_spect.py index f1fa3a6848..4f468f675d 100644 --- a/geminidr/core/tests/test_spect.py +++ b/geminidr/core/tests/test_spect.py @@ -138,7 +138,7 @@ def test_find_apertures(filename, path_to_inputs, change_working_dir): 'S20210709S0035_stack.fits': {'min_snr': 10}} with change_working_dir(path_to_inputs): - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) try: # Check for custom parameter values for individual tests params.update(extra_params[filename]) @@ -158,7 +158,7 @@ def test_find_apertures(filename, path_to_inputs, change_working_dir): @pytest.mark.preprocessed_data def test_create_new_aperture(path_to_inputs): - ad = astrodata.open(os.path.join(path_to_inputs, 'S20060826S0305_2D.fits')) + ad = astrodata.from_file(os.path.join(path_to_inputs, 'S20060826S0305_2D.fits')) p = GNIRSLongslit([ad]) # Test creating a new aperture @@ -181,7 +181,7 @@ def test_create_new_aperture(path_to_inputs): @pytest.mark.preprocessed_data def test_create_new_aperture_warnings_and_errors(path_to_inputs, caplog): - ad = astrodata.open(os.path.join(path_to_inputs, 'S20060826S0305_2D.fits')) + ad = astrodata.from_file(os.path.join(path_to_inputs, 'S20060826S0305_2D.fits')) p = GNIRSLongslit([ad]) # Check that only passing one 'aper' parameter raises a ValueError @@ -394,7 +394,7 @@ def test_adjust_wavelength_zero_point_shift(in_shift, change_working_dir, path_to_inputs): """Apply a shift and confirm that the WCS has changed correctly""" with change_working_dir(path_to_inputs): - ad = astrodata.open('N20220706S0337_wavelengthSolutionAttached.fits') + ad = astrodata.from_file('N20220706S0337_wavelengthSolutionAttached.fits') dispaxis = 2 - ad.dispersion_axis()[0] # python sense center = ad[0].shape[1 - dispaxis] // 2 @@ -413,7 +413,7 @@ def test_adjust_wavelength_zero_point_overlarge_shift(in_shift, change_working_dir, path_to_inputs): with change_working_dir(path_to_inputs): - ad = astrodata.open('N20220706S0337_wavelengthSolutionAttached.fits') + ad = astrodata.from_file('N20220706S0337_wavelengthSolutionAttached.fits') p = GNIRSLongslit([ad]) with pytest.raises(ValueError): @@ -454,7 +454,7 @@ def test_adjust_wavelength_zero_point_auto_shift(filename, result, center = centers.get(filename) with change_working_dir(path_to_inputs): - ad = astrodata.open(filename + '_wavelengthSolutionAttached.fits') + ad = astrodata.from_file(filename + '_wavelengthSolutionAttached.fits') instrument = ad.instrument() p = classes_dict[instrument]([ad]) @@ -484,7 +484,7 @@ def test_adjust_wavelength_zero_point_controlled(filename, center, shift, 'F2': F2Longslit, 'NIRI': NIRILongslit} - ad = astrodata.open(os.path.join(path_to_inputs, + ad = astrodata.from_file(os.path.join(path_to_inputs, filename + '_wavelengthSolutionAttached.fits')) p = classes_dict[ad.instrument()]([ad]) @@ -523,11 +523,11 @@ def test_mask_beyond_slit(in_file, instrument, change_working_dir, 'F2': F2Longslit, 'NIRI': NIRILongslit} - ad = astrodata.open(os.path.join(path_to_inputs, + ad = astrodata.from_file(os.path.join(path_to_inputs, in_file + '_slitEdgesDetermined.fits')) p = classes_dict[instrument]([ad]) ad_out = p.maskBeyondSlit().pop() - ref = astrodata.open(os.path.join(path_to_refs, + ref = astrodata.from_file(os.path.join(path_to_refs, in_file + '_maskedBeyondSlit.fits')) # Find the size of the smallest extension in the file; we don't need the # mask to match *exactly*, so as long as the mismatch isn't more than 0.1 of @@ -564,7 +564,7 @@ def test_slit_rectification(filename, instrument, change_working_dir, 'NIRI': NIRILongslit} with change_working_dir(path_to_inputs): - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) p = classes_dict[instrument]([ad]) @@ -644,7 +644,7 @@ def test_get_sky_spectrum(path_to_inputs, path_to_refs): # is a Chebyshev1D, as required. (In normal reduction, a Cheb1D will be # provided bto _get_sky_spectrum() y determineWavelengthSolution, # regardless of the state of the input file.) - ad_f2 = astrodata.open(os.path.join( + ad_f2 = astrodata.from_file(os.path.join( path_to_inputs, 'S20180114S0104_wavelengthSolutionDetermined.fits')) wave_model = am.get_named_submodel(ad_f2[0].wcs.forward_transform, 'WAVE') @@ -706,15 +706,15 @@ def test_transfer_distortion_model(change_working_dir, path_to_inputs, path_to_r p.findApertures() p.determineWavelengthSolution(absorption=True) """ - ad_no_dist_model = astrodata.open(os.path.join(path_to_inputs, 'N20121221S0199_wavelengthSolutionDetermined.fits')) - ad_with_dist_model = astrodata.open(os.path.join(path_to_inputs, 'N20121221S0199_wavelengthSolutionAttached.fits')) + ad_no_dist_model = astrodata.from_file(os.path.join(path_to_inputs, 'N20121221S0199_wavelengthSolutionDetermined.fits')) + ad_with_dist_model = astrodata.from_file(os.path.join(path_to_inputs, 'N20121221S0199_wavelengthSolutionAttached.fits')) p = primitives_gnirs_longslit.GNIRSLongslit([ad_no_dist_model]) p.streams["with_distortion_model"] = ad_with_dist_model ad_with_dist_model_transferred = p.transferDistortionModel(source="with_distortion_model") p.writeOutputs() with change_working_dir(path_to_refs): ref_with_dist_model_transferred = \ - astrodata.open(os.path.join(path_to_refs, "N20121221S0199_distortionModelTransferred.fits")) + astrodata.from_file(os.path.join(path_to_refs, "N20121221S0199_distortionModelTransferred.fits")) # Compare output WCS as well as pixel values (by evaluating it at the # ends of the ranges, since there are multiple ways of constructing an diff --git a/geminidr/core/tests/test_standardize.py b/geminidr/core/tests/test_standardize.py index 5f61c185db..63e65ab000 100644 --- a/geminidr/core/tests/test_standardize.py +++ b/geminidr/core/tests/test_standardize.py @@ -61,14 +61,14 @@ def test_addDQ(self, change_working_dir, path_to_refs, path_to_common_inputs): with change_working_dir(): - ad = astrodata.open(os.path.join(path_to_refs, + ad = astrodata.from_file(os.path.join(path_to_refs, 'N20070819S0104_prepared.fits')) bpmfile = os.path.join(path_to_common_inputs, 'bpm_20010317_niri_niri_11_full_1amp.fits') p = NIRIImage([ad]) adout = p.addDQ(static_bpm=bpmfile)[0] assert ad_compare(adout, - astrodata.open(os.path.join(path_to_refs, + astrodata.from_file(os.path.join(path_to_refs, 'N20070819S0104_dqAdded.fits'))) @pytest.mark.niri @@ -78,14 +78,14 @@ def test_addIllumMaskToDQ(self, change_working_dir, path_to_inputs, path_to_refs): with change_working_dir(): - ad = astrodata.open(os.path.join(path_to_refs, + ad = astrodata.from_file(os.path.join(path_to_refs, 'N20070819S0104_dqAdded.fits')) p = NIRIImage([ad]) adout = p.addIllumMaskToDQ()[0] assert ad_compare(adout, - astrodata.open(os.path.join( + astrodata.from_file(os.path.join( path_to_refs, 'N20070819S0104_illumMaskAdded.fits'))) @@ -95,11 +95,11 @@ def test_addIllumMaskToDQ(self, change_working_dir, path_to_inputs, def test_addVAR(self, change_working_dir, path_to_inputs, path_to_refs): with change_working_dir(): - ad = astrodata.open(os.path.join(path_to_inputs, + ad = astrodata.from_file(os.path.join(path_to_inputs, 'N20070819S0104_ADUToElectrons.fits')) p = NIRIImage([ad]) adout = p.addVAR(read_noise=True, poisson_noise=True)[0] - assert ad_compare(adout, astrodata.open(os.path.join(path_to_refs, + assert ad_compare(adout, astrodata.from_file(os.path.join(path_to_refs, 'N20070819S0104_varAdded.fits'))) @pytest.mark.dragons_remote_data @@ -112,7 +112,7 @@ def test_makeIRAFCompatible(self, filename, instrument, inst_class): GMOS_keywords = ('GPREPARE', 'GGAIN', 'GAINMULT', 'CCDSUM') - p = inst_class([astrodata.open(download_from_archive(filename))]) + p = inst_class([astrodata.from_file(download_from_archive(filename))]) p.prepare() p.ADUToElectrons() ad = p.makeIRAFCompatible()[0] @@ -133,7 +133,7 @@ def test_makeIRAFCompatible(self, filename, instrument, inst_class): def test_prepare(self, change_working_dir, path_to_inputs, path_to_refs): - ad = astrodata.open(os.path.join(path_to_inputs, + ad = astrodata.from_file(os.path.join(path_to_inputs, 'N20070819S0104.fits')) with change_working_dir(): logutils.config(file_name=f'log_regression_{ad.data_label()}.txt') @@ -143,7 +143,7 @@ def test_prepare(self, change_working_dir, path_to_inputs, outfilename='N20070819S0104_prepared.fits').pop() del prepared_ad.phu['SDZWCS'] # temporary fix - ref_ad = astrodata.open( + ref_ad = astrodata.from_file( os.path.join(path_to_refs, 'N20070819S0104_prepared.fits')) assert ad_compare(prepared_ad, ref_ad) @@ -153,7 +153,7 @@ def test_prepare(self, change_working_dir, path_to_inputs, @pytest.mark.preprocessed_data def test_standardizeHeaders(self, change_working_dir, path_to_inputs): - ad = astrodata.open(os.path.join(path_to_inputs, + ad = astrodata.from_file(os.path.join(path_to_inputs, 'N20070819S0104.fits')) with change_working_dir(): diff --git a/geminidr/core/tests/test_telluric.py b/geminidr/core/tests/test_telluric.py index d8adae2272..781ab8d2c0 100644 --- a/geminidr/core/tests/test_telluric.py +++ b/geminidr/core/tests/test_telluric.py @@ -17,7 +17,7 @@ @pytest.mark.preprocessed_data @pytest.mark.parametrize("filename", ["hip93667_109_ad.fits"]) def test_fit_telluric(path_to_inputs, path_to_refs, filename): - ad = astrodata.open(os.path.join(path_to_inputs, filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, filename)) pm = PrimitiveMapper(ad.tags, ad.instrument(generic=True).lower(), mode='sq', drpkg='geminidr') @@ -26,7 +26,7 @@ def test_fit_telluric(path_to_inputs, path_to_refs, filename): adout = p.fitTelluric(magnitude="K=5.241", bbtemp=9650, shift_tolerance=None).pop() - adref = astrodata.open(os.path.join(path_to_refs, adout.filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, adout.filename)) assert ad_compare(adout, adref) assert np.allclose(adout[0].TELLFIT['PCA coefficients'].data, adref[0].TELLFIT['PCA coefficients'].data) @@ -44,7 +44,7 @@ def test_fit_telluric(path_to_inputs, path_to_refs, filename): ]) def test_get_atran_linelist(filename, model_params, change_working_dir, path_to_inputs, path_to_refs): - ad = astrodata.open(os.path.join(path_to_inputs, filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, filename)) p = GNIRSLongslit([]) wave_model = am.get_named_submodel(ad[0].wcs.forward_transform, 'WAVE') linelist = p._get_atran_linelist(wave_model=wave_model, ext=ad[0], diff --git a/geminidr/core/tests/test_visualize.py b/geminidr/core/tests/test_visualize.py index 006611b19b..21fcb7b84a 100644 --- a/geminidr/core/tests/test_visualize.py +++ b/geminidr/core/tests/test_visualize.py @@ -158,7 +158,7 @@ def input_ads(path_to_inputs, request): input_data_list = [] for p in input_paths: if os.path.exists(p): - input_data_list.append(astrodata.open(p)) + input_data_list.append(astrodata.from_file(p)) else: raise FileNotFoundError(p) @@ -202,7 +202,7 @@ def create_inputs(): arc_paths = [download_from_archive(f) for f in arc_list] cals = [] - raw_ads = [astrodata.open(p) for p in raw_paths] + raw_ads = [astrodata.from_file(p) for p in raw_paths] data_label = raw_ads[0].data_label() print('Current working directory:\n {:s}'.format(os.getcwd())) diff --git a/geminidr/core/tests/test_wcs_creation_and_stability.py b/geminidr/core/tests/test_wcs_creation_and_stability.py index 42f951cc7a..182c231d56 100644 --- a/geminidr/core/tests/test_wcs_creation_and_stability.py +++ b/geminidr/core/tests/test_wcs_creation_and_stability.py @@ -55,7 +55,7 @@ def tile_all(request): return request.param def test_gmos_wcs_stability(raw_ad_path, do_prepare, do_overscan_correct, tile_all): - raw_ad = astrodata.open(raw_ad_path) + raw_ad = astrodata.from_file(raw_ad_path) # Ensure it's tagged IMAGE so we can get an imaging WCS and can use SkyCoord raw_ad.phu['GRATING'] = 'MIRROR' @@ -101,7 +101,7 @@ def test_gmos_wcs_stability(raw_ad_path, do_prepare, do_overscan_correct, tile_a # Now write the file to disk and read it back in and check WCS stability ad.write(TEMPFILE, overwrite=True) - ad = astrodata.open(TEMPFILE) + ad = astrodata.from_file(TEMPFILE) c = SkyCoord(*ad[new_ref_index].wcs(x, y), unit="deg") assert c0.separation(c) < 1e-9 * u.arcsec diff --git a/geminidr/doc/tutorials/F2Img-DRTutorial/savefig/display_flamingos2_stack.py b/geminidr/doc/tutorials/F2Img-DRTutorial/savefig/display_flamingos2_stack.py index b68e121957..73cceef548 100755 --- a/geminidr/doc/tutorials/F2Img-DRTutorial/savefig/display_flamingos2_stack.py +++ b/geminidr/doc/tutorials/F2Img-DRTutorial/savefig/display_flamingos2_stack.py @@ -18,7 +18,7 @@ def main(): filename = get_stack_filename() - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) data = ad[0].data mask = ad[0].mask diff --git a/geminidr/doc/tutorials/GMOSImg-DRTutorial/savefig/display_gmos_single.py b/geminidr/doc/tutorials/GMOSImg-DRTutorial/savefig/display_gmos_single.py index 6c0d1c33c0..b5bbbec06b 100755 --- a/geminidr/doc/tutorials/GMOSImg-DRTutorial/savefig/display_gmos_single.py +++ b/geminidr/doc/tutorials/GMOSImg-DRTutorial/savefig/display_gmos_single.py @@ -19,7 +19,7 @@ def main(): filename = get_stack_filename() - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) fig = plt.figure(num=filename, figsize=(7, 4.5)) fig.suptitle(os.path.basename(filename), y=0.97) diff --git a/geminidr/doc/tutorials/GMOSImg-DRTutorial/savefig/display_gmos_stack.py b/geminidr/doc/tutorials/GMOSImg-DRTutorial/savefig/display_gmos_stack.py index a1c4196221..be8f91cad6 100755 --- a/geminidr/doc/tutorials/GMOSImg-DRTutorial/savefig/display_gmos_stack.py +++ b/geminidr/doc/tutorials/GMOSImg-DRTutorial/savefig/display_gmos_stack.py @@ -19,7 +19,7 @@ def main(): args = _parse_args() filename = args.filename - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) data = ad[0].data mask = ad[0].mask diff --git a/geminidr/doc/tutorials/GSAOIImg-DRTutorial/scripts/show_discostu_output.py b/geminidr/doc/tutorials/GSAOIImg-DRTutorial/scripts/show_discostu_output.py index 64f88cfdd7..58a0757e1f 100755 --- a/geminidr/doc/tutorials/GSAOIImg-DRTutorial/scripts/show_discostu_output.py +++ b/geminidr/doc/tutorials/GSAOIImg-DRTutorial/scripts/show_discostu_output.py @@ -19,7 +19,7 @@ def main(): filename = get_filename() - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) print(ad.info()) fig = plt.figure(num=filename, figsize=(8, 8)) diff --git a/geminidr/doc/tutorials/GSAOIImg-DRTutorial/scripts/show_flat_corrected_image.py b/geminidr/doc/tutorials/GSAOIImg-DRTutorial/scripts/show_flat_corrected_image.py index 98a605abe0..72f7375f32 100755 --- a/geminidr/doc/tutorials/GSAOIImg-DRTutorial/scripts/show_flat_corrected_image.py +++ b/geminidr/doc/tutorials/GSAOIImg-DRTutorial/scripts/show_flat_corrected_image.py @@ -19,7 +19,7 @@ def main(): # filename = 'S20170505S0102_flatCorrected.fits' filename = get_filename() - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) print(ad.info()) fig = plt.figure(num=filename, figsize=(8, 8)) diff --git a/geminidr/f2/tests/longslit/test_determine_distortion.py b/geminidr/f2/tests/longslit/test_determine_distortion.py index 2d1a0fefcd..f2470bd1f0 100644 --- a/geminidr/f2/tests/longslit/test_determine_distortion.py +++ b/geminidr/f2/tests/longslit/test_determine_distortion.py @@ -287,7 +287,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -501,7 +501,7 @@ def create_inputs_recipe(): flat_darks_paths = [download_from_archive(f) for f in cals['flat_darks']] flat_path = [download_from_archive(f) for f in cals['flat']] - arc_ad = astrodata.open(arc_path) + arc_ad = astrodata.from_file(arc_path) data_label = arc_ad.data_label() logutils.config(file_name='log_arc_darks_{}.txt'.format(data_label)) @@ -556,7 +556,7 @@ def create_refs_recipe(): print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, params in input_pars: - ad = astrodata.open(os.path.join('inputs', filename)) + ad = astrodata.from_file(os.path.join('inputs', filename)) p = F2Longslit([ad]) p.determineDistortion(**{**fixed_parameters_for_determine_distortion, **params}) diff --git a/geminidr/f2/tests/longslit/test_determine_wavelength_solution.py b/geminidr/f2/tests/longslit/test_determine_wavelength_solution.py index 1aeeeefe4c..2864c4bd98 100644 --- a/geminidr/f2/tests/longslit/test_determine_wavelength_solution.py +++ b/geminidr/f2/tests/longslit/test_determine_wavelength_solution.py @@ -349,7 +349,7 @@ def test_regression_determine_wavelength_solution( if record.levelname == "WARNING": assert "No acceptable wavelength solution found" not in record.message - ref_ad = astrodata.open(os.path.join(path_to_refs, wcalibrated_ad.filename)) + ref_ad = astrodata.from_file(os.path.join(path_to_refs, wcalibrated_ad.filename)) model = am.get_named_submodel(wcalibrated_ad[0].wcs.forward_transform, "WAVE") ref_model = am.get_named_submodel(ref_ad[0].wcs.forward_transform, "WAVE") @@ -407,7 +407,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -644,7 +644,7 @@ def create_inputs_recipe(): flat_darks_paths = [download_from_archive(f) for f in cals['flat_darks']] flat_path = [download_from_archive(f) for f in cals['flat']] - arc_ad = astrodata.open(arc_path) + arc_ad = astrodata.from_file(arc_path) data_label = arc_ad.data_label() logutils.config(file_name='log_arc_darks_{}.txt'.format(data_label)) @@ -699,7 +699,7 @@ def create_refs_recipe(): print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, params in input_pars: - ad = astrodata.open(os.path.join('inputs', filename)) + ad = astrodata.from_file(os.path.join('inputs', filename)) p = F2Longslit([ad]) p.determineWavelengthSolution(**{**determine_wavelength_solution_parameters, **params}) diff --git a/geminidr/f2/tests/longslit/test_f2_longslit.py b/geminidr/f2/tests/longslit/test_f2_longslit.py index 7892638c87..3f54edc4ac 100644 --- a/geminidr/f2/tests/longslit/test_f2_longslit.py +++ b/geminidr/f2/tests/longslit/test_f2_longslit.py @@ -15,7 +15,7 @@ @pytest.mark.dragons_remote_data def test_addMDF(): - p = F2Longslit([astrodata.open( + p = F2Longslit([astrodata.from_file( download_from_archive('S20140605S0101.fits'))]) ad = p.prepare()[0] # Includes addMDF() as a step. diff --git a/geminidr/f2/tests/longslit/test_flat_correct.py b/geminidr/f2/tests/longslit/test_flat_correct.py index d979d3ac18..c85f7acd41 100644 --- a/geminidr/f2/tests/longslit/test_flat_correct.py +++ b/geminidr/f2/tests/longslit/test_flat_correct.py @@ -38,7 +38,7 @@ def ad(path_to_inputs, request): """Return AD object in input directory""" path = os.path.join(path_to_inputs, request.param) if os.path.exists(path): - return astrodata.open(path) + return astrodata.from_file(path) raise FileNotFoundError(path) diff --git a/geminidr/f2/tests/longslit/test_sky_stacking.py b/geminidr/f2/tests/longslit/test_sky_stacking.py index cfec4434f7..76dc4f229c 100644 --- a/geminidr/f2/tests/longslit/test_sky_stacking.py +++ b/geminidr/f2/tests/longslit/test_sky_stacking.py @@ -13,7 +13,7 @@ # ---- Fixtures --------------------------------------------------------------- @pytest.fixture def f2_abba(): - return [astrodata.open(download_from_archive(f)) for f in + return [astrodata.from_file(download_from_archive(f)) for f in ('S20200301S0071.fits', 'S20200301S0072.fits', 'S20200301S0073.fits', 'S20200301S0074.fits')] @@ -90,7 +90,7 @@ def test_associate_sky_quasi_abcde(): 'S20210515S0203.fits', 'S20210515S0206.fits', 'S20210515S0208.fits'] - data = [astrodata.open(download_from_archive(f)) for f in files] + data = [astrodata.from_file(download_from_archive(f)) for f in files] p = F2Longslit(data) p.prepare() diff --git a/geminidr/f2/tests/spect/test_trace_apertures.py b/geminidr/f2/tests/spect/test_trace_apertures.py index 4d169a8982..c467e189be 100644 --- a/geminidr/f2/tests/spect/test_trace_apertures.py +++ b/geminidr/f2/tests/spect/test_trace_apertures.py @@ -101,7 +101,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) diff --git a/geminidr/f2/tests/test_determine_slit_edges.py b/geminidr/f2/tests/test_determine_slit_edges.py index cda3c8231c..61f88a4f34 100644 --- a/geminidr/f2/tests/test_determine_slit_edges.py +++ b/geminidr/f2/tests/test_determine_slit_edges.py @@ -76,7 +76,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) diff --git a/geminidr/f2/tests/test_select_from_inputs.py b/geminidr/f2/tests/test_select_from_inputs.py index 4472cb2b55..fba0edeb80 100644 --- a/geminidr/f2/tests/test_select_from_inputs.py +++ b/geminidr/f2/tests/test_select_from_inputs.py @@ -21,7 +21,7 @@ def input_ad(request): filename = request.param path = download_from_archive(filename) - ad = astrodata.open(path) + ad = astrodata.from_file(path) return ad diff --git a/geminidr/gemini/tests/test_gemini.py b/geminidr/gemini/tests/test_gemini.py index 244664b28f..5c5ee2e936 100644 --- a/geminidr/gemini/tests/test_gemini.py +++ b/geminidr/gemini/tests/test_gemini.py @@ -16,7 +16,7 @@ # @pytest.fixture(scope='module') # def ad(path_to_inputs): # -# return astrodata.open( +# return astrodata.from_file( # os.path.join(path_to_inputs, 'N20020829S0026.fits')) # --- Delete me? --- @@ -140,7 +140,7 @@ def test_standardize_wcs_create_new(dataset): filenames = [f"{dataset[0][:10]}{{:04d}}.fits".format(i) for i in range(start, start+dataset[1])] files = [download_from_archive(f) for f in filenames] - adinputs = [astrodata.open(f) for f in files] + adinputs = [astrodata.from_file(f) for f in files] # Remove third dimension if adinputs[0].instrument() == "F2": diff --git a/geminidr/gemini/tests/test_qa.py b/geminidr/gemini/tests/test_qa.py index 648a578a39..f26f8701bc 100644 --- a/geminidr/gemini/tests/test_qa.py +++ b/geminidr/gemini/tests/test_qa.py @@ -26,7 +26,7 @@ def ad(path_to_inputs): """Has been run through prepare, addDQ, overscanCorrect, detectSources, addReferenceCatalog, determineAstrometricSolution""" - ad = astrodata.open(os.path.join(path_to_inputs, "N20150624S0106_astrometryCorrected.fits")) + ad = astrodata.from_file(os.path.join(path_to_inputs, "N20150624S0106_astrometryCorrected.fits")) return ad @@ -148,7 +148,7 @@ def test_measureIQ(caplog, ad): def test_measureIQ_no_objcat_AO(caplog): """Confirm we get a report with AO seeing if no OBJCAT""" caplog.set_level(logging.DEBUG) - ad = astrodata.open(download_from_archive("N20131215S0156.fits")) + ad = astrodata.from_file(download_from_archive("N20131215S0156.fits")) p = NIRIImage([ad]) p.measureIQ() @@ -167,7 +167,7 @@ def test_measureIQ_no_objcat_AO(caplog): def test_measure_IQ_GMOS_thru_slit(caplog): """Measure on a GMOS thru-slit LS observation""" caplog.set_level(logging.DEBUG) - ad = astrodata.open(download_from_archive("N20180521S0099.fits")) + ad = astrodata.from_file(download_from_archive("N20180521S0099.fits")) p = GMOSImage([ad]) p.prepare(attach_mdf=True) p.addDQ() @@ -185,7 +185,7 @@ def test_measure_IQ_GMOS_thru_slit(caplog): @pytest.mark.dragons_remote_data def test_measureIQ_no_objcat(): """Confirm the primitive doesn't crash with no OBJCAT""" - ad = astrodata.open(download_from_archive("N20180105S0064.fits")) + ad = astrodata.from_file(download_from_archive("N20180105S0064.fits")) p = GMOSImage([ad]) p.measureIQ()[0] @@ -193,7 +193,7 @@ def test_measureIQ_no_objcat(): @pytest.mark.dragons_remote_data def test_measureIQ_no_objcat_GSAOI(): """Confirm the primitive doesn't for GSAOI with no OBJCAT""" - ad = astrodata.open(download_from_archive("S20150528S0112.fits")) + ad = astrodata.from_file(download_from_archive("S20150528S0112.fits")) p = NIRIImage([ad]) p.measureBG()[0] @@ -201,6 +201,6 @@ def test_measureIQ_no_objcat_GSAOI(): @pytest.mark.dragons_remote_data def test_measureBG_no_zeropoint(caplog): """Confirm the primitive doesn't crash with no nominal_photometric_zeropoint""" - ad = astrodata.open(download_from_archive("N20131215S0152.fits")) + ad = astrodata.from_file(download_from_archive("N20131215S0152.fits")) p = NIRIImage([ad]) p.measureBG()[0] diff --git a/geminidr/ghost/polyfit/test/test_extract.py b/geminidr/ghost/polyfit/test/test_extract.py index f3d7da7d55..6f673e2909 100644 --- a/geminidr/ghost/polyfit/test/test_extract.py +++ b/geminidr/ghost/polyfit/test/test_extract.py @@ -45,7 +45,7 @@ def make_extractor(self, request): datetime.date(2016, 11, 20), None, caltype) for caltype in ('xmod', 'wavemod', 'spatmod', 'specmod', 'rotmod')] - ga.spectral_format_with_matrix(*[astrodata.open(fn)[0].data + ga.spectral_format_with_matrix(*[astrodata.from_file(fn)[0].data for fn in fnames]) # Stand up the Slitview, and run necessary functions @@ -54,7 +54,7 @@ def make_extractor(self, request): sv = slitview.SlitView( slit_image=np.loadtxt(os.path.join(TEST_DATA_DIR, 'slitimage.dat')), flat_image=np.loadtxt(os.path.join(TEST_DATA_DIR, 'slitflat.dat')), - slitvpars=astrodata.open(slitv_fn).TABLE[0], mode=res) + slitvpars=astrodata.from_file(slitv_fn).TABLE[0], mode=res) ext = extract.Extractor(ga, sv) diff --git a/geminidr/ghost/polyfit/test/test_slitview.py b/geminidr/ghost/polyfit/test/test_slitview.py index 2c1d2cfe2f..9abe6b881a 100644 --- a/geminidr/ghost/polyfit/test/test_slitview.py +++ b/geminidr/ghost/polyfit/test/test_slitview.py @@ -26,7 +26,7 @@ def get_slitview_obj(self, request): flat_arr = np.zeros((160, 160)) slitv_fn = get_polyfit_filename( None, 'slitv', res, datetime.date(2016, 11, 20), None, 'slitvmod') - slitvpars = astrodata.open(slitv_fn).TABLE[0] + slitvpars = astrodata.from_file(slitv_fn).TABLE[0] sv = polyfit.slitview.SlitView( data_arr, flat_arr, slitvpars=slitvpars, mode=res) # import pdb; pdb.set_trace() @@ -130,11 +130,11 @@ def test_slitview_slit_profile_shape(self, arm, use_flat, get_slitview_obj): @pytest.mark.ghostslit @pytest.mark.parametrize("filename, results", SEEING_ESTIMATES) def test_seeing_estimate(filename, results, path_to_inputs): - ad = astrodata.open(os.path.join(path_to_inputs, filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, filename)) slitv_fn = get_polyfit_filename( None, 'slitv', ad.res_mode(), ad.ut_date(), None, 'slitvmod') sv = polyfit.slitview.SlitView( - ad[0].data, None, slitvpars=astrodata.open(slitv_fn).TABLE[0], + ad[0].data, None, slitvpars=astrodata.from_file(slitv_fn).TABLE[0], mode=ad.res_mode()) m = sv.model_profile(ad[0].data) for k, v in m.items(): diff --git a/geminidr/ghost/primitives_ghost_slit.py b/geminidr/ghost/primitives_ghost_slit.py index 64d8e2f568..f4434e2d66 100644 --- a/geminidr/ghost/primitives_ghost_slit.py +++ b/geminidr/ghost/primitives_ghost_slit.py @@ -641,7 +641,7 @@ def _total_obj_flux(log, res, ut_date, filename, data, flat_data=None, binning=2 slitv_fn = polyfit_lookup.get_polyfit_filename(log, 'slitv', res, ut_date, filename, 'slitvmod') - slitvpars = astrodata.open(slitv_fn) + slitvpars = astrodata.from_file(slitv_fn) svobj = SlitView(data, flat_data, slitvpars.TABLE[0], mode=res, microns_pix=4.54*180/50, binning=binning) # OK to pass None for flat reds = svobj.object_slit_profiles( diff --git a/geminidr/ghost/primitives_ghost_spect.py b/geminidr/ghost/primitives_ghost_spect.py index 8c89dd1ff1..212ab34863 100644 --- a/geminidr/ghost/primitives_ghost_spect.py +++ b/geminidr/ghost/primitives_ghost_spect.py @@ -155,9 +155,9 @@ def attachWavelengthSolution(self, adinputs=None, **params): arc_before_file = params["arc_before"] arc_after_file = params["arc_after"] if arc_before_file: - arc_before = astrodata.open(arc_before_file) + arc_before = astrodata.from_file(arc_before_file) if arc_after_file: - arc_after = astrodata.open(arc_after_file) + arc_after = astrodata.from_file(arc_after_file) input_frame = adwcs.pixel_frame(2) output_frame = cf.SpectralFrame(axes_order=(0,), unit=u.nm, @@ -1003,10 +1003,10 @@ def determineWavelengthSolution(self, adinputs=None, **params): poly_spat = self._get_polyfit_filename(ad, 'spatmod') poly_spec = self._get_polyfit_filename(ad, 'specmod') poly_rot = self._get_polyfit_filename(ad, 'rotmod') - wpars = astrodata.open(poly_wave) - spatpars = astrodata.open(poly_spat) - specpars = astrodata.open(poly_spec) - rotpars = astrodata.open(poly_rot) + wpars = astrodata.from_file(poly_wave) + spatpars = astrodata.from_file(poly_spat) + specpars = astrodata.from_file(poly_spec) + rotpars = astrodata.from_file(poly_rot) except IOError: log.warning("Cannot open required initial model files; " "skipping") @@ -1355,11 +1355,11 @@ def extractSpectra(self, adinputs=None, **params): poly_spec = self._get_polyfit_filename(ad, 'specmod') poly_rot = self._get_polyfit_filename(ad, 'rotmod') slitv_fn = self._get_slitv_polyfit_filename(ad) - wpars = astrodata.open(poly_wave) - spatpars = astrodata.open(poly_spat) - specpars = astrodata.open(poly_spec) - rotpars = astrodata.open(poly_rot) - slitvpars = astrodata.open(slitv_fn) + wpars = astrodata.from_file(poly_wave) + spatpars = astrodata.from_file(poly_spat) + specpars = astrodata.from_file(poly_spec) + rotpars = astrodata.from_file(poly_rot) + slitvpars = astrodata.from_file(slitv_fn) except IOError: log.warning("Cannot open required initial model files for {};" " skipping".format(ad.filename)) @@ -1728,11 +1728,11 @@ def measureBlaze(self, adinputs=None, **params): poly_spec = self._get_polyfit_filename(ad, 'specmod') poly_rot = self._get_polyfit_filename(ad, 'rotmod') slitv_fn = self._get_slitv_polyfit_filename(ad) - wpars = astrodata.open(poly_wave) - spatpars = astrodata.open(poly_spat) - specpars = astrodata.open(poly_spec) - rotpars = astrodata.open(poly_rot) - slitvpars = astrodata.open(slitv_fn) + wpars = astrodata.from_file(poly_wave) + spatpars = astrodata.from_file(poly_spat) + specpars = astrodata.from_file(poly_spec) + rotpars = astrodata.from_file(poly_rot) + slitvpars = astrodata.from_file(slitv_fn) except IOError: raise RuntimeError("Cannot open required initial model files; " "skipping") @@ -1853,11 +1853,11 @@ def removeScatteredLight(self, adinputs=None, **params): poly_spec = self._get_polyfit_filename(ad, 'specmod') poly_rot = self._get_polyfit_filename(ad, 'rotmod') slitv_fn = self._get_slitv_polyfit_filename(ad) - wpars = astrodata.open(poly_wave) - spatpars = astrodata.open(poly_spat) - specpars = astrodata.open(poly_spec) - rotpars = astrodata.open(poly_rot) - slitvpars = astrodata.open(slitv_fn) + wpars = astrodata.from_file(poly_wave) + spatpars = astrodata.from_file(poly_spat) + specpars = astrodata.from_file(poly_spec) + rotpars = astrodata.from_file(poly_rot) + slitvpars = astrodata.from_file(slitv_fn) except IOError: raise RuntimeError("Cannot open required initial model files; " "skipping") @@ -2292,9 +2292,9 @@ def traceFibers(self, adinputs=None, **params): log.stdinfo(f'Found spatmod: {poly_spat}') slitv_fn = self._get_slitv_polyfit_filename(ad) log.stdinfo(f'Found slitvmod: {slitv_fn}') - xpars = astrodata.open(poly_xmod) - spatpars = astrodata.open(poly_spat) - slitvpars = astrodata.open(slitv_fn) + xpars = astrodata.from_file(poly_xmod) + spatpars = astrodata.from_file(poly_spat) + slitvpars = astrodata.from_file(slitv_fn) except IOError: log.warning("Cannot open required initial model files; " "skipping") @@ -2341,9 +2341,9 @@ def traceFibers(self, adinputs=None, **params): poly_wave = self._get_polyfit_filename(ad, 'wavemod') poly_spec = self._get_polyfit_filename(ad, 'specmod') poly_rot = self._get_polyfit_filename(ad, 'rotmod') - wpars = astrodata.open(poly_wave) - specpars = astrodata.open(poly_spec) - rotpars = astrodata.open(poly_rot) + wpars = astrodata.from_file(poly_wave) + specpars = astrodata.from_file(poly_spec) + rotpars = astrodata.from_file(poly_rot) except IOError: log.warning("Cannot open required initial model files " "for PIXELMODEL; skipping") @@ -2493,7 +2493,7 @@ def _request_bracket_arc(self, ad, before=None): # If the arc is retrieved from user_cals then it will ignore 'ARCBEFOR' # and the same file will be returned twice, but we don't want that if arc_ad: - arc_ad = astrodata.open(arc_ad) + arc_ad = astrodata.from_file(arc_ad) correct_timing = before == (arc_ad.ut_datetime() < ad.ut_datetime()) return arc_ad if correct_timing else None return None diff --git a/geminidr/ghost/recipes/sq/tests/test_reduce_arc.py b/geminidr/ghost/recipes/sq/tests/test_reduce_arc.py index 75f965df4f..358798ac04 100644 --- a/geminidr/ghost/recipes/sq/tests/test_reduce_arc.py +++ b/geminidr/ghost/recipes/sq/tests/test_reduce_arc.py @@ -21,7 +21,7 @@ @pytest.fixture def input_filename(change_working_dir, request): with change_working_dir(): - ad = astrodata.open(download_from_archive(request.param)) + ad = astrodata.from_file(download_from_archive(request.param)) p = GHOSTBundle([ad]) adoutputs = p.splitBundle() return_dict = {} @@ -59,8 +59,8 @@ def test_reduce_arc(input_filename, caldict, arm, path_to_inputs, path_to_refs): makeProcessedArc(p) assert len(p.streams['main']) == 1 output_filename = p.streams['main'][0].filename - adout = astrodata.open(os.path.join("calibrations", "processed_arc", output_filename)) - adref = astrodata.open(os.path.join(path_to_refs, output_filename)) + adout = astrodata.from_file(os.path.join("calibrations", "processed_arc", output_filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, output_filename)) # Changed timestamp kw from STCKARCS -> STACKARC and don't have time to # re-upload reference, so just add these to the "ignore" list assert ad_compare(adref, adout, ignore_kw=['PROCARC', 'STACKARC', 'STCKARCS']) diff --git a/geminidr/ghost/recipes/sq/tests/test_reduce_bias.py b/geminidr/ghost/recipes/sq/tests/test_reduce_bias.py index 47c1841197..5fa1981544 100644 --- a/geminidr/ghost/recipes/sq/tests/test_reduce_bias.py +++ b/geminidr/ghost/recipes/sq/tests/test_reduce_bias.py @@ -17,7 +17,7 @@ @pytest.fixture def input_filename(change_working_dir, request): with change_working_dir(): - ad = astrodata.open(download_from_archive(request.param)) + ad = astrodata.from_file(download_from_archive(request.param)) p = GHOSTBundle([ad]) adoutputs = p.splitBundle() return_dict = {} @@ -42,6 +42,6 @@ def test_reduce_bias(change_working_dir, path_to_inputs, input_filename, arm, pa makeProcessedBias(p) assert len(p.streams['main']) == 1 output_filename = p.streams['main'][0].filename - adout = astrodata.open(os.path.join("calibrations", "processed_bias", output_filename)) - adref = astrodata.open(os.path.join(path_to_refs, output_filename)) + adout = astrodata.from_file(os.path.join("calibrations", "processed_bias", output_filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, output_filename)) assert ad_compare(adref, adout, ignore_kw=['PROCBIAS']) diff --git a/geminidr/ghost/recipes/sq/tests/test_reduce_flat.py b/geminidr/ghost/recipes/sq/tests/test_reduce_flat.py index b4386fd51f..b198096f59 100644 --- a/geminidr/ghost/recipes/sq/tests/test_reduce_flat.py +++ b/geminidr/ghost/recipes/sq/tests/test_reduce_flat.py @@ -20,7 +20,7 @@ @pytest.fixture def input_filename(change_working_dir, request): with change_working_dir(): - ad = astrodata.open(download_from_archive(request.param)) + ad = astrodata.from_file(download_from_archive(request.param)) p = GHOSTBundle([ad]) adoutputs = p.splitBundle() return_dict = {} @@ -55,8 +55,8 @@ def test_reduce_flat(change_working_dir, input_filename, bias, arm, makeProcessedFlat(p) assert len(p.streams['main']) == 1 output_filename = p.streams['main'][0].filename - adout = astrodata.open(os.path.join("calibrations", "processed_flat", output_filename)) - adref = astrodata.open(os.path.join(path_to_refs, output_filename)) + adout = astrodata.from_file(os.path.join("calibrations", "processed_flat", output_filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, output_filename)) assert ad_compare(adref, adout, ignore_kw=['PROCFLAT']) # Comparison doesn't include "exotic" extensions diff --git a/geminidr/ghost/recipes/sq/tests/test_reduce_sci.py b/geminidr/ghost/recipes/sq/tests/test_reduce_sci.py index 9e0f21b5f2..555485828d 100644 --- a/geminidr/ghost/recipes/sq/tests/test_reduce_sci.py +++ b/geminidr/ghost/recipes/sq/tests/test_reduce_sci.py @@ -22,7 +22,7 @@ @pytest.fixture def input_filename(change_working_dir, request): with change_working_dir(): - ad = astrodata.open(download_from_archive(request.param)) + ad = astrodata.from_file(download_from_archive(request.param)) p = GHOSTBundle([ad]) adoutputs = p.splitBundle() return_dict = {} @@ -80,16 +80,16 @@ def test_reduce_science(input_filename, caldict, arm, skysub, path_to_inputs, assert len(p.streams['main']) == 1 p.writeOutputs() output_filename = p.streams['main'][0].filename - adout = astrodata.open(output_filename) - adref = astrodata.open(os.path.join( + adout = astrodata.from_file(output_filename) + adref = astrodata.from_file(os.path.join( path_to_refs, f"skysub_{skysub}", output_filename)) assert ad_compare(adref, adout, ignore_kw=['ARCIM_A', 'ARCIM_B', 'PROCSCI'], atol=1e-14, max_miss=1) # Now compare the _calibrated.fits files (not order-combined) intermediate_filename = output_filename.replace("_dragons", "_calibrated") - adout = astrodata.open(os.path.join(path_to_outputs, "outputs", intermediate_filename)) - adref = astrodata.open(os.path.join( + adout = astrodata.from_file(os.path.join(path_to_outputs, "outputs", intermediate_filename)) + adref = astrodata.from_file(os.path.join( path_to_refs, f"skysub_{skysub}", intermediate_filename)) assert ad_compare(adref, adout, ignore_kw=['ARCIM_A', 'ARCIM_B', 'PROCSCI'], atol=1e-14, max_miss=1) diff --git a/geminidr/ghost/recipes/sq/tests/test_reduce_slit.py b/geminidr/ghost/recipes/sq/tests/test_reduce_slit.py index 3323d84012..94e8c87426 100644 --- a/geminidr/ghost/recipes/sq/tests/test_reduce_slit.py +++ b/geminidr/ghost/recipes/sq/tests/test_reduce_slit.py @@ -27,13 +27,13 @@ @pytest.mark.parametrize("input_filename", bias_datasets) def test_reduce_slit_bias(input_filename, path_to_inputs, path_to_refs, change_working_dir): """Test the complete reduction of slitviewer bias frames""" - ad = astrodata.open(os.path.join(path_to_inputs, input_filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, input_filename)) p = GHOSTSlit([ad]) with change_working_dir(): makeProcessedSlitBias(p) output_filename = p.streams['main'][0].filename - adout = astrodata.open(os.path.join("calibrations", "processed_bias", output_filename)) - adref = astrodata.open(os.path.join(path_to_refs, output_filename)) + adout = astrodata.from_file(os.path.join("calibrations", "processed_bias", output_filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, output_filename)) assert ad_compare(adref, adout, ignore_kw=['PROCBIAS']) @@ -43,15 +43,15 @@ def test_reduce_slit_bias(input_filename, path_to_inputs, path_to_refs, change_w def test_reduce_slit_flat(input_filename, processed_bias, path_to_inputs, path_to_refs, change_working_dir): """Test the complete reduction of slitviewer flat frames""" - ad = astrodata.open(os.path.join(path_to_inputs, input_filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, input_filename)) processed_bias = os.path.join(path_to_inputs, processed_bias) ucals = {"processed_bias": processed_bias} p = GHOSTSlit([ad], ucals=ucals) with change_working_dir(): makeProcessedSlitFlat(p) output_filename = p.streams['main'][0].filename - adout = astrodata.open(os.path.join("calibrations", "processed_slitflat", output_filename)) - adref = astrodata.open(os.path.join(path_to_refs, output_filename)) + adout = astrodata.from_file(os.path.join("calibrations", "processed_slitflat", output_filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, output_filename)) assert ad_compare(adref, adout, ignore_kw=['PRSLITFL']) @@ -61,15 +61,15 @@ def test_reduce_slit_flat(input_filename, processed_bias, path_to_inputs, def test_reduce_slit_arc(input_filename, caldict, path_to_inputs, path_to_refs, change_working_dir): """Test the complete reduction of slitviewer arc frames""" - ad = astrodata.open(os.path.join(path_to_inputs, input_filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, input_filename)) ucals = {k: os.path.join(path_to_inputs, v) for k, v in caldict.items()} p = GHOSTSlit([ad], ucals=ucals) with change_working_dir(): makeProcessedSlitArc(p) output_filename = p.streams['main'][0].filename - adout = astrodata.open(os.path.join("calibrations", "processed_slit", output_filename)) - adref = astrodata.open(os.path.join(path_to_refs, output_filename)) + adout = astrodata.from_file(os.path.join("calibrations", "processed_slit", output_filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, output_filename)) assert ad_compare(adref, adout, ignore_kw=['PRSLITIM']) @@ -79,13 +79,13 @@ def test_reduce_slit_arc(input_filename, caldict, path_to_inputs, def test_reduce_slit_science(input_filename, caldict, path_to_inputs, path_to_refs, change_working_dir): """Test the complete reduction of slitviewer science frames""" - ad = astrodata.open(os.path.join(path_to_inputs, input_filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, input_filename)) ucals = {k: os.path.join(path_to_inputs, v) for k, v in caldict.items()} p = GHOSTSlit([ad], ucals=ucals) with change_working_dir(): makeProcessedSlit(p) for output_filename in [ad.filename for ad in p.streams['main']]: - adout = astrodata.open(os.path.join("calibrations", "processed_slit", output_filename)) - adref = astrodata.open(os.path.join(path_to_refs, output_filename)) + adout = astrodata.from_file(os.path.join("calibrations", "processed_slit", output_filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, output_filename)) assert ad_compare(adref, adout, ignore_kw=['PRSLITIM']) diff --git a/geminidr/ghost/tests/bundle/test_split_bundle.py b/geminidr/ghost/tests/bundle/test_split_bundle.py index 976aa80e1f..83865b6f7e 100644 --- a/geminidr/ghost/tests/bundle/test_split_bundle.py +++ b/geminidr/ghost/tests/bundle/test_split_bundle.py @@ -24,7 +24,7 @@ def test_split_bundle(change_working_dir, path_to_refs): S20230214S0025 has 1 blue, 3 red, and 5 slit images """ with change_working_dir(): - ad = astrodata.open(download_from_archive("S20230214S0025.fits")) + ad = astrodata.from_file(download_from_archive("S20230214S0025.fits")) p = GHOSTBundle([ad]) p.splitBundle() @@ -47,5 +47,5 @@ def test_split_bundle(change_working_dir, path_to_refs): assert len(sciexp) == 4 for adout in blue_files + red_files + slit_files: - adref = astrodata.open(os.path.join(path_to_refs, adout.filename)) + adref = astrodata.from_file(os.path.join(path_to_refs, adout.filename)) assert ad_compare(adref, adout, ignore_kw=['GHOSTDR']) diff --git a/geminidr/ghost/tests/slit/__init__.py b/geminidr/ghost/tests/slit/__init__.py index c774ab7552..bc77fc5341 100644 --- a/geminidr/ghost/tests/slit/__init__.py +++ b/geminidr/ghost/tests/slit/__init__.py @@ -66,7 +66,7 @@ def ad_slit(): # scale by fluxes slitv_fn = polyfit_lookup.get_polyfit_filename( None, 'slitv', 'std', ad.ut_date(), ad.filename, 'slitvmod') - slitvpars = astrodata.open(slitv_fn) + slitvpars = astrodata.from_file(slitv_fn) sview = SlitView(None, None, slitvpars.TABLE[0], mode=ad.res_mode()) slit_data = sview.fake_slitimage(seeing=0.7) for ext in ad: diff --git a/geminidr/ghost/tests/spect/test_combine_orders.py b/geminidr/ghost/tests/spect/test_combine_orders.py index dbe6b05fc0..eac0033a03 100644 --- a/geminidr/ghost/tests/spect/test_combine_orders.py +++ b/geminidr/ghost/tests/spect/test_combine_orders.py @@ -19,7 +19,7 @@ @pytest.mark.parametrize("filename", FILENAMES) def test_combine_orders_single_file(change_working_dir, path_to_inputs, filename): """Check that combineOrders() works on a single file""" - ad = astrodata.open(os.path.join(path_to_inputs, filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, filename)) orig_wavl = make_wavelength_table(ad[0]) p = GHOSTSpect([ad]) ad_out = p.combineOrders().pop() @@ -31,14 +31,14 @@ def test_combine_orders_single_file(change_working_dir, path_to_inputs, filename # Check that we can write and read back the _ordersCombined file with change_working_dir(): ad_out.write("test.fits", overwrite=True) - ad2 = astrodata.open("test.fits") + ad2 = astrodata.from_file("test.fits") np.testing.assert_allclose(ad_out[0].wcs(pixels), ad2[0].wcs(pixels)) @pytest.mark.ghostspect def test_combine_orders_red_and_blue_no_stacking(path_to_inputs): """Check that combineOrders() works on red and blue files with stacking""" - adinputs = [astrodata.open(os.path.join(path_to_inputs, filename)) + adinputs = [astrodata.from_file(os.path.join(path_to_inputs, filename)) for filename in FILENAMES] p = GHOSTSpect(adinputs) adoutputs = p.combineOrders(stacking_mode="none") @@ -54,7 +54,7 @@ def test_combine_orders_red_and_blue_no_stacking(path_to_inputs): @pytest.mark.parametrize("stacking_mode", ("scaled", "unscaled")) def test_combine_orders_red_and_blue_stacking(path_to_inputs, stacking_mode): """Check that combineOrders() works on red and blue files with stacking""" - adinputs = [astrodata.open(os.path.join(path_to_inputs, filename)) + adinputs = [astrodata.from_file(os.path.join(path_to_inputs, filename)) for filename in FILENAMES] orig_wavl = np.array([make_wavelength_table(ad[0]).min() for ad in adinputs]) @@ -74,7 +74,7 @@ def test_combine_orders_red_and_blue_stacking(path_to_inputs, stacking_mode): @pytest.mark.parametrize("stacking_mode", ("scaled", "unscaled")) def test_combine_orders_one_arm_stacking(path_to_inputs, stacking_mode): """Check that combineOrders() stacks files from one arm""" - adinputs = [astrodata.open(os.path.join(path_to_inputs, FILENAMES[0])) * 2] + adinputs = [astrodata.from_file(os.path.join(path_to_inputs, FILENAMES[0])) * 2] orig_wavl = np.array([make_wavelength_table(ad[0]).min() for ad in adinputs]) p = GHOSTSpect(adinputs) diff --git a/geminidr/ghost/tests/spect/test_extract_spectra.py b/geminidr/ghost/tests/spect/test_extract_spectra.py index d5573d9770..83b90d526f 100644 --- a/geminidr/ghost/tests/spect/test_extract_spectra.py +++ b/geminidr/ghost/tests/spect/test_extract_spectra.py @@ -13,7 +13,7 @@ def test_synthetic_slit_profile(path_to_inputs): with a synthetic slit profile. The output is not checked""" sci_filename = "S20230513S0229_red001_arraysTiled.fits" raw_flat_filename = "S20230511S0035.fits" - ad = astrodata.open(os.path.join(path_to_inputs, sci_filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, sci_filename)) arm = ad.arm() processed_flat = os.path.join( path_to_inputs, diff --git a/geminidr/ghost/tests/spect/test_flux_calibrate.py b/geminidr/ghost/tests/spect/test_flux_calibrate.py index 7e1e698c46..ac96e06398 100644 --- a/geminidr/ghost/tests/spect/test_flux_calibrate.py +++ b/geminidr/ghost/tests/spect/test_flux_calibrate.py @@ -31,11 +31,11 @@ def test_flux_calibrate_binning(path_to_inputs, std_filenames): for sci_filename in std_filenames: outputs = {} for std_filename in std_filenames: - ad_sci = astrodata.open(os.path.join(path_to_inputs, sci_filename)) + ad_sci = astrodata.from_file(os.path.join(path_to_inputs, sci_filename)) arm = ad_sci.arm() sci_xbin, sci_ybin = ad_sci.detector_x_bin(), ad_sci.detector_y_bin() p = GHOSTSpect([ad_sci]) - ad_std = astrodata.open(os.path.join(path_to_inputs, std_filename)) + ad_std = astrodata.from_file(os.path.join(path_to_inputs, std_filename)) xbin, ybin = ad_std.detector_x_bin(), ad_std.detector_y_bin() outputs[(xbin, ybin)] = p.fluxCalibrate(standard=ad_std).pop() diff --git a/geminidr/ghost/utils/mkspatmod.py b/geminidr/ghost/utils/mkspatmod.py index b393984462..42581a6a84 100644 --- a/geminidr/ghost/utils/mkspatmod.py +++ b/geminidr/ghost/utils/mkspatmod.py @@ -19,9 +19,9 @@ def get_fibre_separation(p): ad, ad_slitflat = p.streams['main'] #p.getProcessedSlitFlat(refresh=False) #slitflat = p._get_cal(ad, 'processed_slitflat') - #ad_slitflat = astrodata.open(slitflat) + #ad_slitflat = astrodata.from_file(slitflat) slitv_fn = p._get_slitv_polyfit_filename(ad) - slitvpars = astrodata.open(slitv_fn) + slitvpars = astrodata.from_file(slitv_fn) sv = SlitView(None, ad_slitflat[0].data, slitvpars.TABLE[0], mode=ad.res_mode()) slit_models = sv.model_profile(flat_image=ad_slitflat[0].data) @@ -36,7 +36,7 @@ def get_xpars(p): ad = p.streams['main'][0] poly_xmod = p._get_polyfit_filename(ad, 'xmod') print(f"Using XMOD {poly_xmod}") - xpars = astrodata.open(poly_xmod) + xpars = astrodata.from_file(poly_xmod) return xpars[0].data @@ -45,7 +45,7 @@ def get_initial_spatmod(p): ad = p.streams['main'][0] poly_spat = p._get_polyfit_filename(ad, 'spatmod') print(f"Using SPATMOD {poly_spat}") - spatpars = astrodata.open(poly_spat) + spatpars = astrodata.from_file(poly_spat) return spatpars[0].data @@ -229,8 +229,8 @@ def main(ad, ad_slitflat, flat_bin=8): slitflat_filename = sys.argv[2] except IndexError: print("Usage: mkspatmod.py ") - flat = astrodata.open(flat_filename) - slitflat = astrodata.open(slitflat_filename) + flat = astrodata.from_file(flat_filename) + slitflat = astrodata.from_file(slitflat_filename) assert ({'FLAT', 'PROCESSED'}.issubset(flat.tags) and 'SLIT' not in flat.tags), f"{flat.filename} is not a FLAT" assert flat.res_mode() == "std", "Must be run on SR data" diff --git a/geminidr/gmos/lookups/bpmtab.py b/geminidr/gmos/lookups/bpmtab.py index 3d5393ad93..6f3d8dd31d 100644 --- a/geminidr/gmos/lookups/bpmtab.py +++ b/geminidr/gmos/lookups/bpmtab.py @@ -138,7 +138,7 @@ def tabl(ffiles): print(header) ffiles.sort() for ff in ffiles: - ad = astrodata.open(ff) + ad = astrodata.from_file(ff) fname = os.path.split(ff)[-1] if len(fname) < 25: print(rows.format( diff --git a/geminidr/gmos/primitives_gmos.py b/geminidr/gmos/primitives_gmos.py index 7e987b3126..46d3bc322b 100644 --- a/geminidr/gmos/primitives_gmos.py +++ b/geminidr/gmos/primitives_gmos.py @@ -178,7 +178,7 @@ def standardizeInstrumentHeaders(self, adinputs=None, suffix=None): # # When we create the new AD object, it needs to retain the # # filename information # orig_path = ad.path - # ad = astrodata.open(hdulist) + # ad = astrodata.from_file(hdulist) # ad.path = orig_path # KL Commissioning GMOS-N Hamamatsu. Headers are not fully diff --git a/geminidr/gmos/recipes/qa/tests/test_flat_image.py b/geminidr/gmos/recipes/qa/tests/test_flat_image.py index 2d7822745d..69904a260f 100755 --- a/geminidr/gmos/recipes/qa/tests/test_flat_image.py +++ b/geminidr/gmos/recipes/qa/tests/test_flat_image.py @@ -117,7 +117,7 @@ def mock_get_processed_bpm(orig, self, adinputs, caltype, *args, **kwargs): shutil.rmtree('calibrations/') [os.remove(f) for f in glob.glob('*_forStack.fits')] - ad = astrodata.open(r.output_filenames[0]) + ad = astrodata.from_file(r.output_filenames[0]) for ext in ad: data = np.ma.masked_array(ext.data, mask=ext.mask) @@ -198,7 +198,7 @@ def create_master_bias_for_tests(): paths = [download_from_archive(f) for f in filenames] f = paths[0] - ad = astrodata.open(f) + ad = astrodata.from_file(f) if not os.path.exists(f.replace('.fits', '_bias.fits')): print(f" Creating input file:\n") diff --git a/geminidr/gmos/recipes/ql/tests/test_flat_ls_spect.py b/geminidr/gmos/recipes/ql/tests/test_flat_ls_spect.py index b3a10afe71..5ab8bcc696 100644 --- a/geminidr/gmos/recipes/ql/tests/test_flat_ls_spect.py +++ b/geminidr/gmos/recipes/ql/tests/test_flat_ls_spect.py @@ -181,7 +181,7 @@ def mock_get_processed_bpm(orig, self, adinputs, caltype, *args, **kwargs): flat_filename = request.param flat_path = download_from_archive(flat_filename) - flat_raw = astrodata.open(flat_path) + flat_raw = astrodata.from_file(flat_path) master_bias = os.path.join(path_to_inputs, associated_calibrations[flat_filename]) calibration_files = ['processed_bias:{}'.format(master_bias)] @@ -201,7 +201,7 @@ def mock_get_processed_bpm(orig, self, adinputs, caltype, *args, **kwargs): shutil.rmtree('calibrations/') _processed_flat_filename = reduce.output_filenames.pop() - _processed_flat = astrodata.open(_processed_flat_filename) + _processed_flat = astrodata.from_file(_processed_flat_filename) return _processed_flat @@ -261,7 +261,7 @@ def create_master_bias_for_tests(): print('Downloading files...') sci_path = download_from_archive(filename) - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() bias_paths = [download_from_archive(f) for f in bias_files] diff --git a/geminidr/gmos/recipes/sq/tests/test_make_processed_slit_illum.py b/geminidr/gmos/recipes/sq/tests/test_make_processed_slit_illum.py index f03e670697..0b1505e92b 100644 --- a/geminidr/gmos/recipes/sq/tests/test_make_processed_slit_illum.py +++ b/geminidr/gmos/recipes/sq/tests/test_make_processed_slit_illum.py @@ -62,7 +62,7 @@ def processed_slit_illum(change_working_dir, path_to_inputs, request): """ twi_filename = request.param twi_path = download_from_archive(twi_filename) - twi_ad = astrodata.open(twi_path) + twi_ad = astrodata.from_file(twi_path) print(twi_ad.tags) @@ -86,7 +86,7 @@ def processed_slit_illum(change_working_dir, path_to_inputs, request): reduce.runr() _processed_twi_filename = reduce.output_filenames.pop() - _processed_twi = astrodata.open(_processed_twi_filename) + _processed_twi = astrodata.from_file(_processed_twi_filename) return _processed_twi diff --git a/geminidr/gmos/recipes/sq/tests/test_separate_ccd_reduction.py b/geminidr/gmos/recipes/sq/tests/test_separate_ccd_reduction.py index ec506d13a8..779c00d8cc 100644 --- a/geminidr/gmos/recipes/sq/tests/test_separate_ccd_reduction.py +++ b/geminidr/gmos/recipes/sq/tests/test_separate_ccd_reduction.py @@ -37,7 +37,7 @@ def test_separate_ccd_reduction_astrometry(change_working_dir): r.recipename = recipe_name r.suffix = f"_{recipe_name}" r.runr() - adoutputs.append(astrodata.open(r._output_filenames[0])) + adoutputs.append(astrodata.from_file(r._output_filenames[0])) p = GMOSImage(adoutputs) p.detectSources() diff --git a/geminidr/gmos/tests/image/test_add_oiwfs_to_dq.py b/geminidr/gmos/tests/image/test_add_oiwfs_to_dq.py index de4248ebba..d83dd83088 100644 --- a/geminidr/gmos/tests/image/test_add_oiwfs_to_dq.py +++ b/geminidr/gmos/tests/image/test_add_oiwfs_to_dq.py @@ -26,7 +26,7 @@ def test_oiwfs_not_used_in_observation(caplog, filename): """ caplog.set_level(logging.DEBUG) file_path = download_from_archive(filename) - ad = astrodata.open(file_path) + ad = astrodata.from_file(file_path) p = GMOSImage([ad]) p.addOIWFSToDQ() @@ -50,7 +50,7 @@ def test_warn_if_dq_does_not_exist(caplog, filename): """ caplog.set_level(logging.DEBUG) file_path = download_from_archive(filename) - ad = astrodata.open(file_path) + ad = astrodata.from_file(file_path) p = GMOSImage([ad]) p.addOIWFSToDQ() @@ -74,7 +74,7 @@ def test_add_oiwfs_runs_normally(caplog, ext_num, filename, x0, y0): """ caplog.set_level(logging.DEBUG) file_path = download_from_archive(filename) - ad = astrodata.open(file_path) + ad = astrodata.from_file(file_path) p = GMOSImage([ad]) p.addDQ() @@ -114,7 +114,7 @@ def test_add_oiwfs_warns_when_wfs_if_not_in_field(caplog, filename): """ caplog.set_level(logging.DEBUG) file_path = download_from_archive(filename) - ad = astrodata.open(file_path) + ad = astrodata.from_file(file_path) p = GMOSImage([ad]) p.addDQ() diff --git a/geminidr/gmos/tests/longslit/test_add_illum_mask.py b/geminidr/gmos/tests/longslit/test_add_illum_mask.py index 1c654dd17a..3fafc84ff0 100644 --- a/geminidr/gmos/tests/longslit/test_add_illum_mask.py +++ b/geminidr/gmos/tests/longslit/test_add_illum_mask.py @@ -22,7 +22,7 @@ @pytest.mark.parametrize("filename,start_row", datasets_and_locations) def test_add_illum_mask_position(filename, start_row): file_on_disk = download_from_archive(filename) - ad = astrodata.open(file_on_disk) + ad = astrodata.from_file(file_on_disk) p = GMOSLongslit([ad]) p.prepare() @@ -41,7 +41,7 @@ def test_add_illum_mask_position(filename, start_row): @pytest.mark.gmosls def test_add_illum_mask_position_amp5(path_to_inputs, path_to_common_inputs): """Test of bad-amp5 GMOS-S data""" - adinputs = [astrodata.open(os.path.join( + adinputs = [astrodata.from_file(os.path.join( path_to_inputs, f"S20220927S{i:04d}_prepared.fits")) for i in (190, 191)] bpmfile = os.path.join(path_to_common_inputs, diff --git a/geminidr/gmos/tests/longslit/test_flat_correct.py b/geminidr/gmos/tests/longslit/test_flat_correct.py index 783adc8364..6e44be2daa 100644 --- a/geminidr/gmos/tests/longslit/test_flat_correct.py +++ b/geminidr/gmos/tests/longslit/test_flat_correct.py @@ -28,7 +28,7 @@ def ad(path_to_inputs, request): """Return AD object in input directory""" path = os.path.join(path_to_inputs, request.param) if os.path.exists(path): - return astrodata.open(path) + return astrodata.from_file(path) raise FileNotFoundError(path) diff --git a/geminidr/gmos/tests/longslit/test_make_slit_illum.py b/geminidr/gmos/tests/longslit/test_make_slit_illum.py index 58b6ad2492..c291d8891f 100755 --- a/geminidr/gmos/tests/longslit/test_make_slit_illum.py +++ b/geminidr/gmos/tests/longslit/test_make_slit_illum.py @@ -287,7 +287,7 @@ def test_split_mosaic_into_extensions_metadata(filename): """ Tests that the metadata is correctly propagated to the split object. """ - ad = astrodata.open(download_from_archive(filename)) + ad = astrodata.from_file(download_from_archive(filename)) p = primitives_gmos_longslit.GMOSLongslit([ad]) p.prepare() @@ -328,7 +328,7 @@ def ad(request, path_to_inputs): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -388,7 +388,7 @@ def create_inputs_recipe(): twilight_path = [download_from_archive(f) for f in cals['twilight']] bias_path = [download_from_archive(f) for f in cals['bias']] - twilight_ad = astrodata.open(twilight_path[0]) + twilight_ad = astrodata.from_file(twilight_path[0]) data_label = twilight_ad.data_label() print('Reducing BIAS for {:s}'.format(data_label)) @@ -406,7 +406,7 @@ def create_inputs_recipe(): warnings.simplefilter("ignore") p = primitives_gmos_longslit.GMOSLongslit( - [astrodata.open(f) for f in twilight_path]) + [astrodata.from_file(f) for f in twilight_path]) p.prepare() p.addDQ(static_bpm=None) diff --git a/geminidr/gmos/tests/longslit/test_slit_illum_correct.py b/geminidr/gmos/tests/longslit/test_slit_illum_correct.py index b17dd048c3..d8bca20d3f 100755 --- a/geminidr/gmos/tests/longslit/test_slit_illum_correct.py +++ b/geminidr/gmos/tests/longslit/test_slit_illum_correct.py @@ -171,7 +171,7 @@ def _load_file(filename): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - _ad = astrodata.open(path) + _ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -310,7 +310,7 @@ def create_twilight_inputs(): bias_path = [download_from_archive(f) for f in cals['bias']] twilight_path = [download_from_archive(f) for f in cals['twilight']] - twilight_ad = astrodata.open(twilight_path[0]) + twilight_ad = astrodata.from_file(twilight_path[0]) data_label = twilight_ad.data_label() print('Reducing BIAS for {:s}'.format(data_label)) @@ -326,7 +326,7 @@ def create_twilight_inputs(): with warnings.catch_warnings(): warnings.simplefilter("ignore") p = GMOSLongslit( - [astrodata.open(f) for f in twilight_path]) + [astrodata.from_file(f) for f in twilight_path]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) @@ -384,7 +384,7 @@ def create_quartz_inputs(): print('Download raw files') quartz_path = [download_from_archive(f) for f in cals['quartz']] - quartz_ad = astrodata.open(quartz_path[0]) + quartz_ad = astrodata.from_file(quartz_path[0]) data_label = quartz_ad.data_label() print('Reducing quartz lamp:') @@ -392,7 +392,7 @@ def create_quartz_inputs(): with warnings.catch_warnings(): warnings.simplefilter("ignore") p = GMOSLongslit( - [astrodata.open(f) for f in quartz_path]) + [astrodata.from_file(f) for f in quartz_path]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) diff --git a/geminidr/gmos/tests/longslit/test_wavelength_propagation_stability.py b/geminidr/gmos/tests/longslit/test_wavelength_propagation_stability.py index c7dda288c6..387647b997 100644 --- a/geminidr/gmos/tests/longslit/test_wavelength_propagation_stability.py +++ b/geminidr/gmos/tests/longslit/test_wavelength_propagation_stability.py @@ -95,7 +95,7 @@ def compare_peaks(ad, y, ref_peaks, fwidth=4, dw=1): assert abs(mean) < 0.1 * dw with change_working_dir(): - ad_sci = astrodata.open(os.path.join(path_to_inputs, science)) + ad_sci = astrodata.from_file(os.path.join(path_to_inputs, science)) p = GMOSLongslit([ad_sci]) p.prepare() p.addDQ() @@ -105,10 +105,10 @@ def compare_peaks(ad, y, ref_peaks, fwidth=4, dw=1): p.addVAR(poisson_noise=True) if save_and_reload: p.writeOutputs() - ad_sci = astrodata.open(ad_sci.filename) + ad_sci = astrodata.from_file(ad_sci.filename) p = GMOSLongslit([ad_sci]) - ad_arc = astrodata.open(os.path.join(path_to_inputs, arc)) + ad_arc = astrodata.from_file(os.path.join(path_to_inputs, arc)) wtable = ad_arc[0].WAVECAL arc_wave_peaks = wtable["wavelengths"] fwidth = wtable["coefficients"][list(wtable["name"]).index("fwidth")] diff --git a/geminidr/gmos/tests/nodandshuffle/test_combine_nod_and_shuffle_beams.py b/geminidr/gmos/tests/nodandshuffle/test_combine_nod_and_shuffle_beams.py index c0c6d3a46d..fdc20a880c 100644 --- a/geminidr/gmos/tests/nodandshuffle/test_combine_nod_and_shuffle_beams.py +++ b/geminidr/gmos/tests/nodandshuffle/test_combine_nod_and_shuffle_beams.py @@ -32,5 +32,5 @@ def ad(path_to_inputs, request): """Return AD object in input directory""" path = os.path.join(path_to_inputs, request.param) if os.path.exists(path): - return astrodata.open(path) + return astrodata.from_file(path) raise FileNotFoundError(path) diff --git a/geminidr/gmos/tests/nodandshuffle/test_dark_nod_and_shuffle.py b/geminidr/gmos/tests/nodandshuffle/test_dark_nod_and_shuffle.py index e3226b83bc..0a19cc6024 100644 --- a/geminidr/gmos/tests/nodandshuffle/test_dark_nod_and_shuffle.py +++ b/geminidr/gmos/tests/nodandshuffle/test_dark_nod_and_shuffle.py @@ -39,5 +39,5 @@ def ad(path_to_inputs, request): """Return AD object in input directory""" path = os.path.join(path_to_inputs, request.param) if os.path.exists(path): - return astrodata.open(path) + return astrodata.from_file(path) raise FileNotFoundError(path) diff --git a/geminidr/gmos/tests/plots_gmos_spect_longslit_arcs.py b/geminidr/gmos/tests/plots_gmos_spect_longslit_arcs.py index 9b46c7a921..18cd5bec72 100644 --- a/geminidr/gmos/tests/plots_gmos_spect_longslit_arcs.py +++ b/geminidr/gmos/tests/plots_gmos_spect_longslit_arcs.py @@ -137,8 +137,8 @@ def distortion_diagnosis_plots(self): output_file = os.path.join(self.output_folder, self.name + ".fits") reference_file = os.path.join(self.ref_folder, self.name + ".fits") - ad = astrodata.open(output_file) - ad_ref = astrodata.open(reference_file) + ad = astrodata.from_file(output_file) + ad_ref = astrodata.from_file(reference_file) self.show_distortion_map(ad) self.show_distortion_model_difference(ad, ad_ref) diff --git a/geminidr/gmos/tests/spect/test_adjust_wcs_to_reference.py b/geminidr/gmos/tests/spect/test_adjust_wcs_to_reference.py index bb917cb172..b2ab69a28d 100644 --- a/geminidr/gmos/tests/spect/test_adjust_wcs_to_reference.py +++ b/geminidr/gmos/tests/spect/test_adjust_wcs_to_reference.py @@ -33,7 +33,7 @@ def test_adjust_wcs_with_correlation(files, path_to_inputs, caplog): in test_resample_2d.py """ caplog.set_level(20) - adinputs = [astrodata.open(os.path.join(path_to_inputs, f)) for f in files] + adinputs = [astrodata.from_file(os.path.join(path_to_inputs, f)) for f in files] pixel_scale = adinputs[0].pixel_scale() centers = [ad[0].APERTURE['c0'][0] for ad in adinputs] diff --git a/geminidr/gmos/tests/spect/test_attach_wavelength_solution.py b/geminidr/gmos/tests/spect/test_attach_wavelength_solution.py index 2cc272dfff..5a043432a3 100644 --- a/geminidr/gmos/tests/spect/test_attach_wavelength_solution.py +++ b/geminidr/gmos/tests/spect/test_attach_wavelength_solution.py @@ -159,7 +159,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -189,7 +189,7 @@ def arc_ad(path_to_inputs, request): if os.path.exists(path): print(f"Reading input arc: {path}") - arc_ad = astrodata.open(path) + arc_ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -229,8 +229,8 @@ def create_inputs_recipe(): sci_path = download_from_archive(filename) arc_path = download_from_archive(cals['arc']) - sci_ad = astrodata.open(sci_path) - arc_ad = astrodata.open(arc_path) + sci_ad = astrodata.from_file(sci_path) + arc_ad = astrodata.from_file(arc_path) data_label = sci_ad.data_label() logutils.config(file_name='log_arc_{}.txt'.format(data_label)) diff --git a/geminidr/gmos/tests/spect/test_calculate_sensitivity.py b/geminidr/gmos/tests/spect/test_calculate_sensitivity.py index d95f958891..b840c3a375 100644 --- a/geminidr/gmos/tests/spect/test_calculate_sensitivity.py +++ b/geminidr/gmos/tests/spect/test_calculate_sensitivity.py @@ -162,7 +162,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -215,7 +215,7 @@ def create_inputs_recipe(): flat_path = [download_from_archive(f) for f in cals['flat']] arc_path = [download_from_archive(f) for f in cals['arcs']] - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() print('Reducing BIAS for {:s}'.format(data_label)) diff --git a/geminidr/gmos/tests/spect/test_cosmics.py b/geminidr/gmos/tests/spect/test_cosmics.py index 7b9f069b6b..d851f45967 100644 --- a/geminidr/gmos/tests/spect/test_cosmics.py +++ b/geminidr/gmos/tests/spect/test_cosmics.py @@ -21,7 +21,7 @@ @pytest.mark.preprocessed_data @pytest.mark.parametrize('bkgmodel', ['both', 'object', 'skyline', 'none']) def test_cosmics_on_mosaiced_data(path_to_inputs, caplog, bkgmodel): - ad = astrodata.open(os.path.join(path_to_inputs, TESFILE1)) + ad = astrodata.from_file(os.path.join(path_to_inputs, TESFILE1)) ext = ad[0] # add some additional fake cosmics @@ -62,7 +62,7 @@ def test_cosmics_on_mosaiced_data(path_to_inputs, caplog, bkgmodel): @pytest.mark.preprocessed_data @pytest.mark.parametrize('bkgmodel', ['both', 'object', 'skyline', 'none']) def test_cosmics(path_to_inputs, caplog, bkgmodel): - ad = astrodata.open(os.path.join(path_to_inputs, TESFILE2)) + ad = astrodata.from_file(os.path.join(path_to_inputs, TESFILE2)) for ext in ad: # add some additional fake cosmics @@ -124,7 +124,7 @@ def create_inputs_recipe(): print('Current working directory:\n {!s}'.format(path.cwd())) for fname in fnames: - sci_ad = astrodata.open(download_from_archive(fname)) + sci_ad = astrodata.from_file(download_from_archive(fname)) data_label = sci_ad.data_label() print('===== Reducing pre-processed data =====') diff --git a/geminidr/gmos/tests/spect/test_determine_distortion.py b/geminidr/gmos/tests/spect/test_determine_distortion.py index bfe150111d..d7c8bd0651 100644 --- a/geminidr/gmos/tests/spect/test_determine_distortion.py +++ b/geminidr/gmos/tests/spect/test_determine_distortion.py @@ -264,7 +264,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -475,7 +475,7 @@ def create_inputs_recipe(): print('Downloading files...') basename = filename.split("_")[0] + ".fits" sci_path = download_from_archive(basename) - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() print('Reducing pre-processed data:') diff --git a/geminidr/gmos/tests/spect/test_determine_wavelength_solution.py b/geminidr/gmos/tests/spect/test_determine_wavelength_solution.py index 4f9cf77467..757d44a55a 100644 --- a/geminidr/gmos/tests/spect/test_determine_wavelength_solution.py +++ b/geminidr/gmos/tests/spect/test_determine_wavelength_solution.py @@ -185,7 +185,7 @@ def test_regression_determine_wavelength_solution( if record.levelname == "WARNING": assert "No acceptable wavelength solution found" not in record.message - ref_ad = astrodata.open(os.path.join(path_to_refs, wcalibrated_ad.filename)) + ref_ad = astrodata.from_file(os.path.join(path_to_refs, wcalibrated_ad.filename)) model = am.get_named_submodel(wcalibrated_ad[0].wcs.forward_transform, "WAVE") ref_model = am.get_named_submodel(ref_ad[0].wcs.forward_transform, "WAVE") @@ -282,7 +282,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -460,7 +460,7 @@ def create_inputs_recipe(): print('Downloading files...') basename = filename.split("_")[0] + ".fits" sci_path = download_from_archive(basename) - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() print('Reducing pre-processed data:') diff --git a/geminidr/gmos/tests/spect/test_distortion_correct.py b/geminidr/gmos/tests/spect/test_distortion_correct.py index 9994f91d0b..8f87e41f4d 100644 --- a/geminidr/gmos/tests/spect/test_distortion_correct.py +++ b/geminidr/gmos/tests/spect/test_distortion_correct.py @@ -198,7 +198,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -240,8 +240,8 @@ def create_inputs_recipe(): flat_paths = [download_from_archive(f) for f in cals['flat']] arc_paths = [download_from_archive(f) for f in cals['arcs']] - sci_ad = astrodata.open(sci_path) - arc_ad = astrodata.open(arc_paths[0]) + sci_ad = astrodata.from_file(sci_path) + arc_ad = astrodata.from_file(arc_paths[0]) data_label = sci_ad.data_label() # is_ham = sci_ad.detector_name(pretty=True).startswith('Hamamatsu') diff --git a/geminidr/gmos/tests/spect/test_distortion_correct_with_wavelength_solution.py b/geminidr/gmos/tests/spect/test_distortion_correct_with_wavelength_solution.py index 5ac3b87544..ad643c0a15 100644 --- a/geminidr/gmos/tests/spect/test_distortion_correct_with_wavelength_solution.py +++ b/geminidr/gmos/tests/spect/test_distortion_correct_with_wavelength_solution.py @@ -209,7 +209,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -251,8 +251,8 @@ def create_inputs_recipe(): flat_paths = [download_from_archive(f) for f in cals['flat']] arc_paths = [download_from_archive(f) for f in cals['arcs']] - sci_ad = astrodata.open(sci_path) - arc_ad = astrodata.open(arc_paths[0]) + sci_ad = astrodata.from_file(sci_path) + arc_ad = astrodata.from_file(arc_paths[0]) data_label = sci_ad.data_label() logutils.config(file_name='log_bias_{}.txt'.format(data_label)) diff --git a/geminidr/gmos/tests/spect/test_extract_1d_spectra.py b/geminidr/gmos/tests/spect/test_extract_1d_spectra.py index f3953e1351..0bde70329a 100644 --- a/geminidr/gmos/tests/spect/test_extract_1d_spectra.py +++ b/geminidr/gmos/tests/spect/test_extract_1d_spectra.py @@ -112,7 +112,7 @@ def ad(request, path_to_inputs): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -206,7 +206,7 @@ def create_inputs_recipe(): print('Downloading files...') basename = filename.split("_")[0] + ".fits" sci_path = download_from_archive(basename) - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() arcs_path = [download_from_archive(f) for f in arcs_name] diff --git a/geminidr/gmos/tests/spect/test_find_source_apertures.py b/geminidr/gmos/tests/spect/test_find_source_apertures.py index ef8df4dc7d..2543c0ba24 100644 --- a/geminidr/gmos/tests/spect/test_find_source_apertures.py +++ b/geminidr/gmos/tests/spect/test_find_source_apertures.py @@ -177,7 +177,7 @@ def ad_and_center(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -214,7 +214,7 @@ def ad_center_tolerance_snr(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -259,7 +259,7 @@ def create_inputs_recipe(): sci_path = download_from_archive(sci_fname) arc_path = download_from_archive(arc_fname) - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() logutils.config(file_name='log_arc_{}.txt'.format(data_label)) diff --git a/geminidr/gmos/tests/spect/test_flux_calibration.py b/geminidr/gmos/tests/spect/test_flux_calibration.py index 60383e5905..5721e5b5b1 100644 --- a/geminidr/gmos/tests/spect/test_flux_calibration.py +++ b/geminidr/gmos/tests/spect/test_flux_calibration.py @@ -175,7 +175,7 @@ def ad(request, path_to_inputs): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -226,7 +226,7 @@ def create_inputs_recipe(): flat_path = [download_from_archive(f) for f in cals['flat']] arc_path = [download_from_archive(f) for f in cals['arcs']] - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() print('Reducing BIAS for {:s}'.format(data_label)) diff --git a/geminidr/gmos/tests/spect/test_qe_correct.py b/geminidr/gmos/tests/spect/test_qe_correct.py index d998887c5d..af2bde78cd 100644 --- a/geminidr/gmos/tests/spect/test_qe_correct.py +++ b/geminidr/gmos/tests/spect/test_qe_correct.py @@ -266,7 +266,7 @@ def ad(path_to_inputs, request): if os.path.exists(path): print(f"Reading input file: {path}") - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -296,7 +296,7 @@ def arc_ad(path_to_inputs, request): if os.path.exists(path): print(f"Reading input arc: {path}") - arc_ad = astrodata.open(path) + arc_ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -782,7 +782,7 @@ def create_inputs_recipe(use_branch_name=False): flat_paths = [download_from_archive(f) for f in cals['flat']] arc_paths = [download_from_archive(f) for f in cals['arcs']] - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() logutils.config(file_name='log_bias_{}.txt'.format(data_label)) diff --git a/geminidr/gmos/tests/spect/test_resample.py b/geminidr/gmos/tests/spect/test_resample.py index 98a1fbc9cd..6242574972 100644 --- a/geminidr/gmos/tests/spect/test_resample.py +++ b/geminidr/gmos/tests/spect/test_resample.py @@ -139,7 +139,7 @@ def input_ad_list(path_to_inputs): input_path = os.path.join(path_to_inputs, input_fname) if os.path.exists(input_path): - ad = astrodata.open(input_path) + ad = astrodata.from_file(input_path) else: raise FileNotFoundError(input_path) @@ -185,7 +185,7 @@ def create_inputs_recipe(): sci_path = download_from_archive(filename) arc_path = [download_from_archive(f) for f in cals['arcs']] - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() print('Reducing ARC for {:s}'.format(data_label)) diff --git a/geminidr/gmos/tests/spect/test_resample_2d.py b/geminidr/gmos/tests/spect/test_resample_2d.py index 001ef93f51..e1383c9e60 100644 --- a/geminidr/gmos/tests/spect/test_resample_2d.py +++ b/geminidr/gmos/tests/spect/test_resample_2d.py @@ -34,7 +34,7 @@ def test_simple_correlation_test(path_to_inputs, offset): """A simple correlation test that uses a single image, shifted, to avoid difficulties in centroiding. Placed here because it uses datasets and functions in this module""" - adinputs = [astrodata.open(os.path.join(path_to_inputs, test_datasets[0])) + adinputs = [astrodata.from_file(os.path.join(path_to_inputs, test_datasets[0])) for i in (0, 1, 2)] add_fake_offset(adinputs, offset=offset) p = GMOSLongslit(adinputs) @@ -162,13 +162,13 @@ def add_fake_offset(adinputs, offset=10): @pytest.fixture(scope='function') def adinputs(path_to_inputs): - return [astrodata.open(os.path.join(path_to_inputs, f)) + return [astrodata.from_file(os.path.join(path_to_inputs, f)) for f in test_datasets] @pytest.fixture(scope='function') def adinputs2(path_to_inputs): - return [astrodata.open(os.path.join(path_to_inputs, f)) + return [astrodata.from_file(os.path.join(path_to_inputs, f)) for f in test_datasets2] @@ -216,16 +216,16 @@ def create_inputs_recipe(): sci_path = download_from_archive(fname) arc_path = download_from_archive(arc_fname) - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() print('Reducing ARC for {:s}'.format(data_label)) logutils.config(file_name='log_arc_{}.txt'.format(data_label)) if os.path.exists(arc_fname.replace('.fits', '_distortionDetermined.fits')): - arc = astrodata.open(arc_fname.replace('.fits', '_distortionDetermined.fits')) + arc = astrodata.from_file(arc_fname.replace('.fits', '_distortionDetermined.fits')) else: - p = GMOSLongslit([astrodata.open(arc_path)]) + p = GMOSLongslit([astrodata.from_file(arc_path)]) p.prepare() p.addDQ(static_bpm=None) p.addVAR(read_noise=True) diff --git a/geminidr/gmos/tests/spect/test_sky_correct_from_slit.py b/geminidr/gmos/tests/spect/test_sky_correct_from_slit.py index 126eead1dd..14cea928f8 100644 --- a/geminidr/gmos/tests/spect/test_sky_correct_from_slit.py +++ b/geminidr/gmos/tests/spect/test_sky_correct_from_slit.py @@ -66,7 +66,7 @@ def test_regression_sky_correct_from_slit(filename, params, refname, path_to_refs): path = os.path.join(path_to_inputs, filename) - ad = astrodata.open(path) + ad = astrodata.from_file(path) with change_working_dir(): logutils.config(file_name=f'log_regression_{ad.data_label()}.txt') @@ -74,7 +74,7 @@ def test_regression_sky_correct_from_slit(filename, params, refname, p.skyCorrectFromSlit(**params) sky_subtracted_ad = p.writeOutputs(outfilename=refname).pop() - ref_ad = astrodata.open(os.path.join(path_to_refs, refname)) + ref_ad = astrodata.from_file(os.path.join(path_to_refs, refname)) # Require that <1% of unmasked pixels differ by >1 sigma for ext, ref_ext in zip(sky_subtracted_ad, ref_ad): @@ -121,7 +121,7 @@ def create_inputs_recipe(): sci_path = download_from_archive(filename) arc_path = download_from_archive(pars['arc']) - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() print('Reducing ARC for {:s}'.format(data_label)) diff --git a/geminidr/gmos/tests/spect/test_trace_apertures.py b/geminidr/gmos/tests/spect/test_trace_apertures.py index abe7d0b4c3..0e38151787 100755 --- a/geminidr/gmos/tests/spect/test_trace_apertures.py +++ b/geminidr/gmos/tests/spect/test_trace_apertures.py @@ -122,7 +122,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -168,7 +168,7 @@ def create_inputs_recipe(): print('Downloading files...') sci_path = download_from_archive(filename) - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() print('Reducing pre-processed data:') diff --git a/geminidr/gmos/tests/spect/test_write_1d_spectra.py b/geminidr/gmos/tests/spect/test_write_1d_spectra.py index 4a2b298553..20770b5e21 100644 --- a/geminidr/gmos/tests/spect/test_write_1d_spectra.py +++ b/geminidr/gmos/tests/spect/test_write_1d_spectra.py @@ -96,7 +96,7 @@ def ad(request, path_to_inputs): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) diff --git a/geminidr/gmos/tests/test_gmos_spect_longslit_arcs.py b/geminidr/gmos/tests/test_gmos_spect_longslit_arcs.py index 07996f211a..682277353b 100644 --- a/geminidr/gmos/tests/test_gmos_spect_longslit_arcs.py +++ b/geminidr/gmos/tests/test_gmos_spect_longslit_arcs.py @@ -202,7 +202,7 @@ def __init__(self, filename, input_dir, output_dir, ref_dir): @staticmethod def reduce(filename): - _p = GMOSLongslit([astrodata.open(filename)]) + _p = GMOSLongslit([astrodata.from_file(filename)]) _p.viewer = geminidr.dormantViewer(_p, None) _p.prepare() @@ -264,7 +264,7 @@ def test_reduced_arcs_contains_stable_wavelength_solution(config): pytest.fail("Reference file not found: {}".format(reference)) ad_out = config.ad - ad_ref = astrodata.open(reference) + ad_ref = astrodata.from_file(reference) for ext_out, ext_ref in zip(ad_out, ad_ref): model = am.get_named_submodel(ext_out.wcs.forward_transform, 'WAVE') @@ -294,7 +294,7 @@ def test_reduced_arcs_are_similar(config): pytest.fail("Reference file not found: {}".format(reference)) ad_out = config.ad - ad_ref = astrodata.open(reference) + ad_ref = astrodata.from_file(reference) # Test reduced arcs are similar for ext_out, ext_ref in zip(ad_out, ad_ref): @@ -326,7 +326,7 @@ def test_distortion_correct(config): p = GMOSLongslit([]) - ad_out = astrodata.open(output) + ad_out = astrodata.from_file(output) ad_out_corrected_with_out = p.distortionCorrect([ad_out], arc=output)[0] ad_out_corrected_with_ref = p.distortionCorrect([ad_out], arc=reference)[0] @@ -372,7 +372,7 @@ def filename(self, name): # B600:0.500 HAM, ROI="Central Spectrum" --- cs = Config("N20171016S0010.fits") - cs.ad = astrodata.open(os.path.join(cs.output_dir, cs.filename)) + cs.ad = astrodata.from_file(os.path.join(cs.output_dir, cs.filename)) # B600:0.500 HAM, ROI="Full Frame" --- ff = Config("N20171016S0127.fits") @@ -423,7 +423,7 @@ def test_distortion_correction_is_applied_the_same_way(config): # if not os.path.exists(reference): # pytest.fail('Reference file not found: {}'.format(reference)) # - # ad_ref = astrodata.open(reference) + # ad_ref = astrodata.from_file(reference) os.rename(filename, os.path.join(config.output_dir, filename)) # Evaluate them --- diff --git a/geminidr/gnirs/recipes/sq/tests/test_ls_spect.py b/geminidr/gnirs/recipes/sq/tests/test_ls_spect.py index 255eb557b9..3fc35f6490 100644 --- a/geminidr/gnirs/recipes/sq/tests/test_ls_spect.py +++ b/geminidr/gnirs/recipes/sq/tests/test_ls_spect.py @@ -121,9 +121,9 @@ def test_reduce_ls_spect(path_to_inputs, path_to_refs, change_working_dir, output = reduce(sci_paths, f"sci_{test_case}", cals, user_pars=datasets[test_case]["user_pars"], return_output=True) - ad_out_2d = astrodata.open(output[0].replace("1D", "2D")) - ad_out_1d = astrodata.open(output[0]) - ad_ref_1d = astrodata.open(os.path.join(path_to_refs, output[0])) + ad_out_2d = astrodata.from_file(output[0].replace("1D", "2D")) + ad_out_1d = astrodata.from_file(output[0]) + ad_ref_1d = astrodata.from_file(os.path.join(path_to_refs, output[0])) # Check fewer than 4 apertures extracted assert len(ad_out_2d[0].APERTURE) < 4 @@ -193,7 +193,7 @@ def reduce(file_list, label, calib_files, recipe_name=None, save_to=None, # -- Fixtures ----------------------------------------------------------------- @pytest.fixture(scope='function') def gnirs_files(files): - return [astrodata.open(download_from_archive(f) for f in files)] + return [astrodata.from_file(download_from_archive(f) for f in files)] @pytest.fixture(scope='module') def keep_data(request): diff --git a/geminidr/gnirs/recipes/sq/tests/test_xd_spect.py b/geminidr/gnirs/recipes/sq/tests/test_xd_spect.py index 5de1a76786..b5d5ad4c70 100644 --- a/geminidr/gnirs/recipes/sq/tests/test_xd_spect.py +++ b/geminidr/gnirs/recipes/sq/tests/test_xd_spect.py @@ -97,11 +97,11 @@ def test_reduce_xd_spect(path_to_inputs, path_to_refs, change_working_dir, output = reduce(sci_paths, f"sci_{test_case}", cals, user_pars=upars, return_output=True) - ad_out_2d = astrodata.open(output[0].replace("1D", "2D")) - ad_out_1d = astrodata.open(output[0]) + ad_out_2d = astrodata.from_file(output[0].replace("1D", "2D")) + ad_out_1d = astrodata.from_file(output[0]) single = "single" if single_wave_scale else "notsingle" ref_filename = output[0].replace("_", f"_{single}_") - ad_ref_1d = astrodata.open(os.path.join(path_to_refs, ref_filename)) + ad_ref_1d = astrodata.from_file(os.path.join(path_to_refs, ref_filename)) # Check fewer than 3 apertures extracted assert len(ad_out_2d[0].APERTURE) < 3 @@ -169,7 +169,7 @@ def reduce(file_list, label, calib_files, recipe_name=None, save_to=None, # -- Fixtures ----------------------------------------------------------------- @pytest.fixture(scope='function') def gnirs_files(files): - return [astrodata.open(download_from_archive(f) for f in files)] + return [astrodata.from_file(download_from_archive(f) for f in files)] @pytest.fixture(scope='module') def keep_data(request): diff --git a/geminidr/gnirs/tests/crossdispersed/test_cut_slits.py b/geminidr/gnirs/tests/crossdispersed/test_cut_slits.py index b7b13515bb..7f7faa4b47 100644 --- a/geminidr/gnirs/tests/crossdispersed/test_cut_slits.py +++ b/geminidr/gnirs/tests/crossdispersed/test_cut_slits.py @@ -29,7 +29,7 @@ def test_cut_slits(adinputs, path_to_inputs): Check that, upon the slits being cut out, input coordinates are recovered successfully when transformed to world coordinates and back. """ - p = GNIRSCrossDispersed([astrodata.open(os.path.join(path_to_inputs, adinputs))]) + p = GNIRSCrossDispersed([astrodata.from_file(os.path.join(path_to_inputs, adinputs))]) adout = p.cutSlits()[0] abs_diff = 20 if 'Long' in adout.camera() else 6 # Roughly 1" for both diff --git a/geminidr/gnirs/tests/crossdispersed/test_determine_distortion.py b/geminidr/gnirs/tests/crossdispersed/test_determine_distortion.py index 6c69fa3f44..b2a4feceb3 100644 --- a/geminidr/gnirs/tests/crossdispersed/test_determine_distortion.py +++ b/geminidr/gnirs/tests/crossdispersed/test_determine_distortion.py @@ -97,7 +97,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) diff --git a/geminidr/gnirs/tests/crossdispersed/test_distortion_correct.py b/geminidr/gnirs/tests/crossdispersed/test_distortion_correct.py index 1506a72c7d..e9a4b2e829 100644 --- a/geminidr/gnirs/tests/crossdispersed/test_distortion_correct.py +++ b/geminidr/gnirs/tests/crossdispersed/test_distortion_correct.py @@ -30,7 +30,7 @@ @pytest.mark.parametrize("filename", test_files) def test_distortion_correct_coords_roundtrip(filename, path_to_inputs): - ad_in = astrodata.open(os.path.join(path_to_inputs, filename)) + ad_in = astrodata.from_file(os.path.join(path_to_inputs, filename)) abs_diff = 10 if 'Long' in ad_in.camera() else 6 # Roughly 1" for both diff --git a/geminidr/gnirs/tests/crossdispersed/test_flat_correct.py b/geminidr/gnirs/tests/crossdispersed/test_flat_correct.py index d686c53f41..3a78d6f817 100644 --- a/geminidr/gnirs/tests/crossdispersed/test_flat_correct.py +++ b/geminidr/gnirs/tests/crossdispersed/test_flat_correct.py @@ -40,7 +40,7 @@ def ad(path_to_inputs, request): """Return AD object in input directory""" path = os.path.join(path_to_inputs, request.param) if os.path.exists(path): - return astrodata.open(path) + return astrodata.from_file(path) raise FileNotFoundError(path) diff --git a/geminidr/gnirs/tests/crossdispersed/test_resample_2d.py b/geminidr/gnirs/tests/crossdispersed/test_resample_2d.py index f28d4a961d..b263c019b6 100644 --- a/geminidr/gnirs/tests/crossdispersed/test_resample_2d.py +++ b/geminidr/gnirs/tests/crossdispersed/test_resample_2d.py @@ -26,7 +26,7 @@ # Local fixtures and helper functions ---------------------------------- @pytest.fixture(scope='function') def adinputs(path_to_inputs): - return [astrodata.open(os.path.join(path_to_inputs, f)) + return [astrodata.from_file(os.path.join(path_to_inputs, f)) for f in test_datasets] def _check_params(records, expected): diff --git a/geminidr/gnirs/tests/crossdispersed/test_straight_slit_edges.py b/geminidr/gnirs/tests/crossdispersed/test_straight_slit_edges.py index c1297a6117..cbf8e3075e 100644 --- a/geminidr/gnirs/tests/crossdispersed/test_straight_slit_edges.py +++ b/geminidr/gnirs/tests/crossdispersed/test_straight_slit_edges.py @@ -18,7 +18,7 @@ @pytest.mark.gnirsxd @pytest.mark.parametrize('filename', ["N20130821S0308_stack.fits"]) def test_edges_and_slit_centers(filename, path_to_inputs): - ad = astrodata.open(os.path.join(path_to_inputs, filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, filename)) # Clear the mask so only pixels beyond the edge are masked for ext in ad: ext.mask = None @@ -31,7 +31,7 @@ def test_edges_and_slit_centers(filename, path_to_inputs): p.cutSlits() ad_masked = p.maskBeyondSlit().pop() - ad = astrodata.open(os.path.join(path_to_inputs, filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, filename)) p = GNIRSCrossDispersed([ad]) ad = p.flatCorrect(flat=ad_masked).pop() for ext in ad: diff --git a/geminidr/gnirs/tests/crossdispersed/test_trace_pinhole_apertures.py b/geminidr/gnirs/tests/crossdispersed/test_trace_pinhole_apertures.py index ac1a292e4a..6428ffa63e 100644 --- a/geminidr/gnirs/tests/crossdispersed/test_trace_pinhole_apertures.py +++ b/geminidr/gnirs/tests/crossdispersed/test_trace_pinhole_apertures.py @@ -104,7 +104,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) diff --git a/geminidr/gnirs/tests/crossdispersed/test_transfer_attribute.py b/geminidr/gnirs/tests/crossdispersed/test_transfer_attribute.py index 9d2fe6a54f..3e48549b67 100644 --- a/geminidr/gnirs/tests/crossdispersed/test_transfer_attribute.py +++ b/geminidr/gnirs/tests/crossdispersed/test_transfer_attribute.py @@ -19,7 +19,7 @@ @pytest.mark.parametrize("dataset", datasets, indirect=False) def test_flat_correct(dataset, change_working_dir): with change_working_dir(): - adinputs = [astrodata.open(download_from_archive(filename)) for + adinputs = [astrodata.from_file(download_from_archive(filename)) for filename in dataset] p = GNIRSCrossDispersed(adinputs) diff --git a/geminidr/gnirs/tests/image/test_add_illum_mask.py b/geminidr/gnirs/tests/image/test_add_illum_mask.py index 6ddd61cac5..e00206f1a1 100644 --- a/geminidr/gnirs/tests/image/test_add_illum_mask.py +++ b/geminidr/gnirs/tests/image/test_add_illum_mask.py @@ -26,9 +26,9 @@ @pytest.mark.parametrize("filename,result", DATASETS) def test_add_illum_mask(filename, result, change_working_dir, path_to_inputs): if filename.startswith("N2022"): - ad = astrodata.open(os.path.join(path_to_inputs, filename)) + ad = astrodata.from_file(os.path.join(path_to_inputs, filename)) else: - ad = astrodata.open(download_from_archive(filename)) + ad = astrodata.from_file(download_from_archive(filename)) with change_working_dir(): p = GNIRSImage([ad]) p.prepare() # bad_wcs="ignore") diff --git a/geminidr/gnirs/tests/longslit/test_adjust_wcs_to_reference.py b/geminidr/gnirs/tests/longslit/test_adjust_wcs_to_reference.py index a11647eb6a..7bbb5b83be 100644 --- a/geminidr/gnirs/tests/longslit/test_adjust_wcs_to_reference.py +++ b/geminidr/gnirs/tests/longslit/test_adjust_wcs_to_reference.py @@ -37,7 +37,7 @@ def test_adjust_wcs_with_correlation(files, path_to_inputs, caplog): params = {'max_apertures': 1, 'percentile': 80, 'min_sky_region': 50, 'min_snr': 5.0, 'use_snr': True, 'threshold': 0.1, 'section': ""} - adinputs = [astrodata.open(os.path.join(path_to_inputs, f)) for f in files] + adinputs = [astrodata.from_file(os.path.join(path_to_inputs, f)) for f in files] pixel_scale = adinputs[0].pixel_scale() centers = [] # GMOS version can use pre-found apertures; GNIRS doesn't have findApertures() diff --git a/geminidr/gnirs/tests/longslit/test_determine_distortion.py b/geminidr/gnirs/tests/longslit/test_determine_distortion.py index 2a0d664521..bdd31045b8 100644 --- a/geminidr/gnirs/tests/longslit/test_determine_distortion.py +++ b/geminidr/gnirs/tests/longslit/test_determine_distortion.py @@ -189,7 +189,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -408,7 +408,7 @@ def create_inputs_recipe(): arc_path = download_from_archive(filename) flat_path = [download_from_archive(f) for f in cals['flat']] - arc_ad = astrodata.open(arc_path) + arc_ad = astrodata.from_file(arc_path) data_label = arc_ad.data_label() logutils.config(file_name='log_flat_{}.txt'.format(data_label)) @@ -445,7 +445,7 @@ def create_refs_recipe(): print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, params in input_pars: - ad = astrodata.open(os.path.join('inputs', filename)) + ad = astrodata.from_file(os.path.join('inputs', filename)) p = GNIRSLongslit([ad]) p.determineDistortion(**{**fixed_parameters_for_determine_distortion, **params}) diff --git a/geminidr/gnirs/tests/longslit/test_determine_wavelength_solution.py b/geminidr/gnirs/tests/longslit/test_determine_wavelength_solution.py index 249dded7f4..40c2746fe8 100644 --- a/geminidr/gnirs/tests/longslit/test_determine_wavelength_solution.py +++ b/geminidr/gnirs/tests/longslit/test_determine_wavelength_solution.py @@ -254,7 +254,7 @@ def test_regression_determine_wavelength_solution( if record.levelname == "WARNING": assert "No acceptable wavelength solution found" not in record.message - ref_ad = astrodata.open(os.path.join(path_to_refs, wcalibrated_ad.filename)) + ref_ad = astrodata.from_file(os.path.join(path_to_refs, wcalibrated_ad.filename)) model = am.get_named_submodel(wcalibrated_ad[0].wcs.forward_transform, "WAVE") ref_model = am.get_named_submodel(ref_ad[0].wcs.forward_transform, "WAVE") @@ -313,7 +313,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -556,7 +556,7 @@ def create_inputs_recipe(): print('Downloading files...') basename = filename.split("_")[0] + ".fits" sci_path = download_from_archive(basename) - sci_ad = astrodata.open(sci_path) + sci_ad = astrodata.from_file(sci_path) data_label = sci_ad.data_label() print('Reducing pre-processed data:') @@ -582,7 +582,7 @@ def create_inputs_recipe(): arc_path = download_from_archive(filename) flat_path = [download_from_archive(f) for f in cals['flat']] - arc_ad = astrodata.open(arc_path) + arc_ad = astrodata.from_file(arc_path) data_label = arc_ad.data_label() logutils.config(file_name='log_flat_{}.txt'.format(data_label)) @@ -617,7 +617,7 @@ def create_inputs_recipe(): flat_path = [download_from_archive(f) for f in cals['flat']] arc_arc_path = [download_from_archive(f) for f in cals['arc']] - arc_ad = astrodata.open(arc_path) + arc_ad = astrodata.from_file(arc_path) data_label = arc_ad.data_label() logutils.config(file_name='log_flat_{}.txt'.format(data_label)) @@ -674,7 +674,7 @@ def create_refs_recipe(): print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, params in input_pars: - ad = astrodata.open(os.path.join('inputs', filename)) + ad = astrodata.from_file(os.path.join('inputs', filename)) p = GNIRSLongslit([ad]) p.determineWavelengthSolution(**{**determine_wavelength_solution_parameters, **params}) diff --git a/geminidr/gnirs/tests/longslit/test_distortion_correct.py b/geminidr/gnirs/tests/longslit/test_distortion_correct.py index d243134975..8a78e38a14 100644 --- a/geminidr/gnirs/tests/longslit/test_distortion_correct.py +++ b/geminidr/gnirs/tests/longslit/test_distortion_correct.py @@ -31,10 +31,10 @@ def test_distortion_correct(filename, path_to_inputs, path_to_refs, change_working_dir): with change_working_dir(path_to_inputs): - ad_in = astrodata.open(filename) + ad_in = astrodata.from_file(filename) with change_working_dir(path_to_refs): - ad_ref = astrodata.open(filename.replace('_readoutCleaned.fits', + ad_ref = astrodata.from_file(filename.replace('_readoutCleaned.fits', '_distortionCorrected.fits')) p = GNIRSLongslit([ad_in]) diff --git a/geminidr/gnirs/tests/longslit/test_flat_correct.py b/geminidr/gnirs/tests/longslit/test_flat_correct.py index 5c170d06bd..433d1b7642 100644 --- a/geminidr/gnirs/tests/longslit/test_flat_correct.py +++ b/geminidr/gnirs/tests/longslit/test_flat_correct.py @@ -38,7 +38,7 @@ def ad(path_to_inputs, request): """Return AD object in input directory""" path = os.path.join(path_to_inputs, request.param) if os.path.exists(path): - return astrodata.open(path) + return astrodata.from_file(path) raise FileNotFoundError(path) diff --git a/geminidr/gnirs/tests/longslit/test_gnirs_longslit.py b/geminidr/gnirs/tests/longslit/test_gnirs_longslit.py index 0e56cc8dec..d1c356e8da 100644 --- a/geminidr/gnirs/tests/longslit/test_gnirs_longslit.py +++ b/geminidr/gnirs/tests/longslit/test_gnirs_longslit.py @@ -15,7 +15,7 @@ @pytest.mark.dragons_remote_data def test_addMDF(): - p = GNIRSLongslit([astrodata.open( + p = GNIRSLongslit([astrodata.from_file( download_from_archive('N20100915S0138.fits'))]) ad = p.prepare()[0] # Includes addMDF() as a step. diff --git a/geminidr/gnirs/tests/longslit/test_prepare.py b/geminidr/gnirs/tests/longslit/test_prepare.py index 8f1813fc91..cd0ef072ab 100644 --- a/geminidr/gnirs/tests/longslit/test_prepare.py +++ b/geminidr/gnirs/tests/longslit/test_prepare.py @@ -18,7 +18,7 @@ def test_longslit_wcs(change_working_dir, filename): """ with change_working_dir(): file_path = download_from_archive(filename) - ad = astrodata.open(file_path) + ad = astrodata.from_file(file_path) p = GNIRSLongslit([ad]) coords1 = ad[0].wcs(X, Y) p.prepare() @@ -26,7 +26,7 @@ def test_longslit_wcs(change_working_dir, filename): assert len(coords2) == 3 np.testing.assert_allclose(coords1, coords2[1:], atol=1e-6) ad.write("test.fits", overwrite=True) - ad2 = astrodata.open("test.fits") + ad2 = astrodata.from_file("test.fits") assert "FITS-WCS" not in ad2.phu # not APPROXIMATE coords3 = ad[0].wcs(X, Y) assert len(coords3) == 3 diff --git a/geminidr/gnirs/tests/longslit/test_resample_2d.py b/geminidr/gnirs/tests/longslit/test_resample_2d.py index dde8dd6c3a..880f5b4db1 100644 --- a/geminidr/gnirs/tests/longslit/test_resample_2d.py +++ b/geminidr/gnirs/tests/longslit/test_resample_2d.py @@ -38,7 +38,7 @@ def test_resample_to_common_frame_with_defaults(input_ad_list, path_to_refs, ad_out = p.stackFrames()[0] _check_params(caplog.records, 'w1=1525.174 w2=1806.038 dw=0.138 npix=2029') assert 'ALIGN' in ad_out[0].phu - ref = astrodata.open(os.path.join(path_to_refs, + ref = astrodata.from_file(os.path.join(path_to_refs, 'N20240329S0022_stack_defaults.fits')) np.testing.assert_allclose(ad_out[0].data, ref[0].data) @@ -54,7 +54,7 @@ def test_resample_to_common_frame_trim_spectral(input_ad_list, path_to_refs, ad_out = p.stackFrames()[0] _check_params(caplog.records, 'w1=1664.096 w2=1666.589 dw=0.138 npix=19') assert 'ALIGN' in ad_out[0].phu - ref = astrodata.open(os.path.join(path_to_refs, + ref = astrodata.from_file(os.path.join(path_to_refs, 'N20240329S0022_stack_trim_spectral_True.fits')) np.testing.assert_allclose(ad_out[0].data, ref[0].data) @@ -71,7 +71,7 @@ def test_resample_to_common_frame_trim_spatial(input_ad_list, path_to_refs, # This should be the same as test_resample_to_common_frame_with_defaults() _check_params(caplog.records, 'w1=1525.174 w2=1806.038 dw=0.138 npix=2029') assert 'ALIGN' in ad_out[0].phu - ref = astrodata.open(os.path.join(path_to_refs, + ref = astrodata.from_file(os.path.join(path_to_refs, 'N20240329S0022_stack_trim_spatial_False.fits')) np.testing.assert_allclose(ad_out[0].data, ref[0].data) @@ -113,7 +113,7 @@ def input_ad_list(path_to_inputs): input_path = os.path.join(path_to_inputs, input_fname) if os.path.exists(input_path): - ad = astrodata.open(input_path) + ad = astrodata.from_file(input_path) else: raise FileNotFoundError(input_path) diff --git a/geminidr/gnirs/tests/longslit/test_sky_correct_from_slit.py b/geminidr/gnirs/tests/longslit/test_sky_correct_from_slit.py index 503c687558..0d1501ba4e 100644 --- a/geminidr/gnirs/tests/longslit/test_sky_correct_from_slit.py +++ b/geminidr/gnirs/tests/longslit/test_sky_correct_from_slit.py @@ -36,7 +36,7 @@ def test_sky_correct_from_slit(file, order, function, change_working_dir, path_to_inputs): - ad = astrodata.open(os.path.join(path_to_inputs, file)) + ad = astrodata.from_file(os.path.join(path_to_inputs, file)) p = GNIRSLongslit([deepcopy(ad)]) ad_out = p.skyCorrectFromSlit(order=order, function=function, **parameters)[0] diff --git a/geminidr/gnirs/tests/longslit/test_sky_stacking.py b/geminidr/gnirs/tests/longslit/test_sky_stacking.py index 0dbddba37e..4fb82b194c 100644 --- a/geminidr/gnirs/tests/longslit/test_sky_stacking.py +++ b/geminidr/gnirs/tests/longslit/test_sky_stacking.py @@ -17,7 +17,7 @@ # ---- Fixtures --------------------------------------------------------------- @pytest.fixture def gnirs_abba(): - return [astrodata.open(download_from_archive(f)) for f in + return [astrodata.from_file(download_from_archive(f)) for f in ('N20141119S0331.fits', 'N20141119S0332.fits', 'N20141119S0333.fits', 'N20141119S0334.fits')] @@ -104,7 +104,7 @@ def test_associate_sky_quasi_abcde(): 'N20220220S0108.fits', 'N20220220S0109.fits', 'N20220220S0110.fits'] - data = [astrodata.open(download_from_archive(f)) for f in files] + data = [astrodata.from_file(download_from_archive(f)) for f in files] p = GNIRSLongslit(data) p.prepare() diff --git a/geminidr/gnirs/tests/test_determine_slit_edges.py b/geminidr/gnirs/tests/test_determine_slit_edges.py index 1ec726dbcc..e8a179edf5 100644 --- a/geminidr/gnirs/tests/test_determine_slit_edges.py +++ b/geminidr/gnirs/tests/test_determine_slit_edges.py @@ -209,7 +209,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) diff --git a/geminidr/gsaoi/tests/test_gsaoi_image.py b/geminidr/gsaoi/tests/test_gsaoi_image.py index 6edaa3a59a..ec30b7720a 100644 --- a/geminidr/gsaoi/tests/test_gsaoi_image.py +++ b/geminidr/gsaoi/tests/test_gsaoi_image.py @@ -28,8 +28,8 @@ def test_gsaoi_adjust_wcs_no_refcat(change_working_dir, path_to_refs, adinputs): p.resampleToCommonFrame(interpolant="linear") p.writeOutputs() for ad in p.streams['main']: - ad = astrodata.open(ad.filename) - ref_ad = astrodata.open(os.path.join(path_to_refs, ad.filename)) + ad = astrodata.from_file(ad.filename) + ref_ad = astrodata.from_file(os.path.join(path_to_refs, ad.filename)) # CJS: The outputs I get on my MacBook apparently do not agree with # those on the Jenkins server, so need to increase the tolerances. # This should still be fine. @@ -72,6 +72,6 @@ def test_gsaoi_resample_to_refcat(path_to_inputs, adinputs): def adinputs(path_to_inputs): adinputs = [] for i in range(148, 151): - adinputs.append(astrodata.open( + adinputs.append(astrodata.from_file( os.path.join(path_to_inputs, f'S20200305S{i:04d}_sourcesDetected.fits'))) return adinputs diff --git a/geminidr/niri/tests/longslit/test_determine_distortion.py b/geminidr/niri/tests/longslit/test_determine_distortion.py index 50bb43fcd7..96a3b4c7df 100644 --- a/geminidr/niri/tests/longslit/test_determine_distortion.py +++ b/geminidr/niri/tests/longslit/test_determine_distortion.py @@ -166,7 +166,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -380,7 +380,7 @@ def create_inputs_recipe(): arc_path = download_from_archive(filename) flat_path = [download_from_archive(f) for f in cals['flat']] - arc_ad = astrodata.open(arc_path) + arc_ad = astrodata.from_file(arc_path) data_label = arc_ad.data_label() logutils.config(file_name='log_flat_{}.txt'.format(data_label)) @@ -417,7 +417,7 @@ def create_refs_recipe(): print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, params in input_pars: - ad = astrodata.open(os.path.join('inputs', filename)) + ad = astrodata.from_file(os.path.join('inputs', filename)) p = NIRILongslit([ad]) p.determineDistortion(**{**fixed_parameters_for_determine_distortion, **params}) diff --git a/geminidr/niri/tests/longslit/test_determine_wavelength_solution.py b/geminidr/niri/tests/longslit/test_determine_wavelength_solution.py index f6cf69e3c6..0d814936e3 100644 --- a/geminidr/niri/tests/longslit/test_determine_wavelength_solution.py +++ b/geminidr/niri/tests/longslit/test_determine_wavelength_solution.py @@ -156,7 +156,7 @@ def test_regression_determine_wavelength_solution( if record.levelname == "WARNING": assert "No acceptable wavelength solution found" not in record.message - ref_ad = astrodata.open(os.path.join(path_to_refs, wcalibrated_ad.filename)) + ref_ad = astrodata.from_file(os.path.join(path_to_refs, wcalibrated_ad.filename)) model = am.get_named_submodel(wcalibrated_ad[0].wcs.forward_transform, "WAVE") ref_model = am.get_named_submodel(ref_ad[0].wcs.forward_transform, "WAVE") @@ -218,7 +218,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) @@ -452,7 +452,7 @@ def create_inputs_recipe(): arc_path = download_from_archive(filename) flat_path = [download_from_archive(f) for f in cals['flat']] - arc_ad = astrodata.open(arc_path) + arc_ad = astrodata.from_file(arc_path) data_label = arc_ad.data_label() logutils.config(file_name='log_flat_{}.txt'.format(data_label)) @@ -488,7 +488,7 @@ def create_inputs_recipe(): flat_path = [download_from_archive(f) for f in cals['flat']] arc_arc_path = [download_from_archive(f) for f in cals['arc']] - arc_ad = astrodata.open(arc_path) + arc_ad = astrodata.from_file(arc_path) data_label = arc_ad.data_label() logutils.config(file_name='log_flat_{}.txt'.format(data_label)) @@ -546,7 +546,7 @@ def create_refs_recipe(): print('Current working directory:\n {:s}'.format(os.getcwd())) for filename, params in input_pars: - ad = astrodata.open(os.path.join('inputs', filename)) + ad = astrodata.from_file(os.path.join('inputs', filename)) p = NIRILongslit([ad]) p.determineWavelengthSolution(**{**determine_wavelength_solution_parameters, **params}) diff --git a/geminidr/niri/tests/longslit/test_flat_correct.py b/geminidr/niri/tests/longslit/test_flat_correct.py index 1a38ccca19..fe9172af04 100644 --- a/geminidr/niri/tests/longslit/test_flat_correct.py +++ b/geminidr/niri/tests/longslit/test_flat_correct.py @@ -36,7 +36,7 @@ def ad(path_to_inputs, request): """Return AD object in input directory""" path = os.path.join(path_to_inputs, request.param) if os.path.exists(path): - return astrodata.open(path) + return astrodata.from_file(path) raise FileNotFoundError(path) diff --git a/geminidr/niri/tests/longslit/test_niri_longslit.py b/geminidr/niri/tests/longslit/test_niri_longslit.py index 8fd70bf3d7..b38b9ef3ab 100644 --- a/geminidr/niri/tests/longslit/test_niri_longslit.py +++ b/geminidr/niri/tests/longslit/test_niri_longslit.py @@ -15,7 +15,7 @@ @pytest.mark.dragons_remote_data def test_addMDF(): - p = NIRILongslit([astrodata.open( + p = NIRILongslit([astrodata.from_file( download_from_archive('N20100620S0116.fits'))]) ad = p.prepare()[0] # Includes addMDF() as a step. diff --git a/geminidr/niri/tests/longslit/test_sky_stacking.py b/geminidr/niri/tests/longslit/test_sky_stacking.py index b10fb00cf2..e8cc299dde 100644 --- a/geminidr/niri/tests/longslit/test_sky_stacking.py +++ b/geminidr/niri/tests/longslit/test_sky_stacking.py @@ -13,13 +13,13 @@ # ---- Fixtures --------------------------------------------------------------- @pytest.fixture def niri_abba(): - return [astrodata.open(download_from_archive(f)) for f in + return [astrodata.from_file(download_from_archive(f)) for f in ('N20100602S0437.fits', 'N20100602S0438.fits', 'N20100602S0439.fits', 'N20100602S0440.fits')] @pytest.fixture def niri_abcde(): - return [astrodata.open(download_from_archive(f)) for f in + return [astrodata.from_file(download_from_archive(f)) for f in ('N20070204S0098.fits', 'N20070204S0099.fits', 'N20070204S0100.fits', @@ -110,7 +110,7 @@ def test_associate_sky_abcde(niri_abcde): 'N20070204S0103.fits': [2, 3, 4, 6], 'N20070204S0104.fits': [3, 4, 5]} - # data = [astrodata.open(download_from_archive(f)) for f in niri_abcde] + # data = [astrodata.from_file(download_from_archive(f)) for f in niri_abcde] p = NIRILongslit(niri_abcde) # Some frames have bad WCS information, so use 'fix' to take care of it. @@ -127,7 +127,7 @@ def test_associate_sky_abcde(niri_abcde): @pytest.mark.dragons_remote_data @pytest.mark.nirils def test_associate_sky_abcde_exclude_some(niri_abcde): - # data = [astrodata.open(download_from_archive(f)) for f in niri_abcde] + # data = [astrodata.from_file(download_from_archive(f)) for f in niri_abcde] p = NIRILongslit(niri_abcde) p.prepare(bad_wcs='fix') diff --git a/geminidr/niri/tests/test_determine_slit_edges.py b/geminidr/niri/tests/test_determine_slit_edges.py index 6ebd737b36..111e763ed6 100644 --- a/geminidr/niri/tests/test_determine_slit_edges.py +++ b/geminidr/niri/tests/test_determine_slit_edges.py @@ -74,7 +74,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) diff --git a/geminidr/niri/tests/test_nonlinearity_correct.py b/geminidr/niri/tests/test_nonlinearity_correct.py index 46ef8dd89c..d6b263d0e2 100644 --- a/geminidr/niri/tests/test_nonlinearity_correct.py +++ b/geminidr/niri/tests/test_nonlinearity_correct.py @@ -34,7 +34,7 @@ def ad(path_to_inputs, request): path = os.path.join(path_to_inputs, filename) if os.path.exists(path): - ad = astrodata.open(path) + ad = astrodata.from_file(path) else: raise FileNotFoundError(path) return ad @@ -56,7 +56,7 @@ def create_inputs(): raw_filename = filename.replace("_varAdded", "") print('Downloading files...') sci_path = download_from_archive(raw_filename) - ad = astrodata.open(sci_path) + ad = astrodata.from_file(sci_path) print(f'Reducing {raw_filename}') p = NIRIImage([ad]) diff --git a/geminidr/tests/test_geminidr.py b/geminidr/tests/test_geminidr.py index 29a09a3d0a..4ab31c19b1 100644 --- a/geminidr/tests/test_geminidr.py +++ b/geminidr/tests/test_geminidr.py @@ -29,7 +29,7 @@ def test_unrecognized_uparm(parm, expected): testfile = download_from_archive("N20160524S0119.fits") with pytest.raises(UnrecognizedParameterException) as upe: - GMOSLongslit(astrodata.open(testfile), mode='sq', ucals={}, uparms=parm, upload=None, config_file=None) + GMOSLongslit(astrodata.from_file(testfile), mode='sq', ucals={}, uparms=parm, upload=None, config_file=None) assert expected in str(upe.value) diff --git a/gempy/adlibrary/dataselect.py b/gempy/adlibrary/dataselect.py index c507969869..7368f31ff6 100644 --- a/gempy/adlibrary/dataselect.py +++ b/gempy/adlibrary/dataselect.py @@ -139,7 +139,7 @@ def select_data(inputs, tags=[], xtags=[], expression='True', adpkg=None): selected_data = [] for input in inputs: - ad = astrodata.open(input) + ad = astrodata.from_file(input) adtags = ad.tags if set(tags).issubset(adtags) and \ not len(set(xtags).intersection(adtags)) and \ diff --git a/gempy/gemini/eti/gemcombinefile.py b/gempy/gemini/eti/gemcombinefile.py index 686168e141..bb79eeac3f 100644 --- a/gempy/gemini/eti/gemcombinefile.py +++ b/gempy/gemini/eti/gemcombinefile.py @@ -114,7 +114,7 @@ def prepare(self): def recover(self): log.debug("OufileETIFile recover()") - ad = astrodata.open(self.tmp_name) + ad = astrodata.from_file(self.tmp_name) ad.filename = self.ad_name ad = gemini_tools.obsmode_del(ad) log.fullinfo(self.tmp_name + " was loaded into memory") diff --git a/gempy/gemini/eti/gmosaicfile.py b/gempy/gemini/eti/gmosaicfile.py index d37cc32eed..32fb1feaa9 100644 --- a/gempy/gemini/eti/gmosaicfile.py +++ b/gempy/gemini/eti/gmosaicfile.py @@ -127,7 +127,7 @@ def recover(self): log.debug("OutAtList recover()") adlist = [] for i, tmpname in enumerate(self.diskoutlist): - ad = astrodata.open(tmpname) + ad = astrodata.from_file(tmpname) ad.filename = self.ad_name[i] ad = gemini_tools.obsmode_del(ad) adlist.append(ad) diff --git a/gempy/gemini/eti/tests/test_gemcombine.py b/gempy/gemini/eti/tests/test_gemcombine.py index 3c0ff14487..58ec9c90d9 100644 --- a/gempy/gemini/eti/tests/test_gemcombine.py +++ b/gempy/gemini/eti/tests/test_gemcombine.py @@ -85,7 +85,7 @@ def test_niri_default(self): # where is the fits diff tool? inputs = [] for filename in TestGemcombine.niri_files: - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) inputs.append(ad) parameters = TESTDEFAULTPARAMS gemcombine_task = \ diff --git a/gempy/gemini/eti/tests/test_gmosaic.py b/gempy/gemini/eti/tests/test_gmosaic.py index 8fcc5150fa..78e96cafa5 100644 --- a/gempy/gemini/eti/tests/test_gmosaic.py +++ b/gempy/gemini/eti/tests/test_gmosaic.py @@ -61,7 +61,7 @@ def test_gmos_default(self): from gempy.gemini.eti import gmosaiceti # where is the fits diff tool? - ad = astrodata.open(TestGmosaic.gmos_file) + ad = astrodata.from_file(TestGmosaic.gmos_file) inputs = [] parameters = TESTDEFAULTPARAMS gmosaic_task = \ diff --git a/gempy/gemini/gemini_tools.py b/gempy/gemini/gemini_tools.py index 833dcf7afe..db1d2c976b 100644 --- a/gempy/gemini/gemini_tools.py +++ b/gempy/gemini/gemini_tools.py @@ -1227,7 +1227,7 @@ def cut_to_match_auxiliary_data(adinput=None, aux=None, aux_type=None, new_ad.append(ext.nddata[detsec.asslice()]) new_ad[-1].SLITEDGE = auxext.SLITEDGE new_ad[-1].hdr[ad._keyword_for('detector_section')] =\ - detsec.asIRAFsection(binning=(xbin, ybin)) + detsec.as_iraf_section(binning=(xbin, ybin)) new_ad[-1].hdr['SPECORDR'] = auxext.hdr['SPECORDR'] # By default, when a section is cut out of an array the WCS is @@ -1728,7 +1728,7 @@ def make_lists(*args, **kwargs): for x in set(_list): if x not in ad_map_dict: try: - ad_map_dict.update({x: astrodata.open(x) + ad_map_dict.update({x: astrodata.from_file(x) if isinstance(x, str) else x}) except OSError: ad_map_dict.update({x: None}) @@ -2383,7 +2383,7 @@ def trim_to_data_section(adinput=None, keyword_comments=None): for i, (oldsec, newsec) in enumerate(zip(sections, new_sections), start=1): #oldsec, newsec = Section(*oldsec), Section(*newsec) - datasecStr = oldsec.asIRAFsection() + datasecStr = oldsec.as_iraf_section() log.fullinfo(f'For {ad.filename} extension {ext.id}, ' f'keeping the data from the section {datasecStr}') newslice = newsec.asslice() @@ -2406,7 +2406,7 @@ def trim_to_data_section(adinput=None, keyword_comments=None): continue # Update logger with the section being kept - datasecStr = datasec.asIRAFsection() + datasecStr = datasec.as_iraf_section() log.fullinfo(f'For {ad.filename} extension {ext.id}, keeping ' f'the data from the section {datasecStr}') diff --git a/gempy/gemini/tests/test_gemini_tools.py b/gempy/gemini/tests/test_gemini_tools.py index 50e7697ad4..fdb63ed832 100644 --- a/gempy/gemini/tests/test_gemini_tools.py +++ b/gempy/gemini/tests/test_gemini_tools.py @@ -44,14 +44,14 @@ def test_convert_to_cal_header(caltype, obj, change_working_dir): """Check that header keywords have been updated and """ # A random NIRI image - ad = astrodata.open(astrodata.testing.download_from_archive('N20200127S0023.fits')) + ad = astrodata.from_file(astrodata.testing.download_from_archive('N20200127S0023.fits')) ad_out = gt.convert_to_cal_header(ad, caltype=caltype, keyword_comments=keyword_comments) # FITS WCS keywords only get changed at write-time, so we need to # write the file to disk and read it back in to confirm. with change_working_dir(): ad_out.write("temp.fits", overwrite=True) - ad = astrodata.open("temp.fits") + ad = astrodata.from_file("temp.fits") assert ad.observation_type() == caltype.upper() # Let's not worry about upper/lowercase @@ -71,7 +71,7 @@ def test_fit_continuum_slit_image(fname, fwhm, change_working_dir): log_file = 'log_{}.log'.format(fname.replace('.fits', '')) logutils.config(file_name=log_file) - ad = astrodata.open(astrodata.testing.download_from_archive(fname)) + ad = astrodata.from_file(astrodata.testing.download_from_archive(fname)) p = GMOSImage([ad]) p.prepare(attach_mdf=True) p.addDQ() @@ -114,7 +114,7 @@ def test_measure_bg_from_image_fake_sections(section, gaussfit, fake_image): @pytest.mark.parametrize("gaussfit", [True, False]) def test_measure_bg_from_image_real(gaussfit): # A random GMOS-N image - ad = astrodata.open(astrodata.testing.download_from_archive("N20191210S0338.fits")) + ad = astrodata.from_file(astrodata.testing.download_from_archive("N20191210S0338.fits")) mean, stddev, nsamples = gt.measure_bg_from_image( ad[8].nddata[ad[8].data_section().asslice()], gaussfit=gaussfit) assert abs(mean - 807) < 1 diff --git a/gempy/scripts/fwhm_histogram.py b/gempy/scripts/fwhm_histogram.py index 45cff72ae5..38e3cae783 100755 --- a/gempy/scripts/fwhm_histogram.py +++ b/gempy/scripts/fwhm_histogram.py @@ -24,7 +24,7 @@ def main(): filename = sys.argv[1] - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) objcat = ad[0].OBJCAT fwhm_arcsec = objcat.field("FWHM_WORLD") * 3600. diff --git a/gempy/scripts/profile_all_obj.py b/gempy/scripts/profile_all_obj.py index 560d75fffc..a110031153 100755 --- a/gempy/scripts/profile_all_obj.py +++ b/gempy/scripts/profile_all_obj.py @@ -154,7 +154,7 @@ def profile_loop(data, xcenter, ycenter, bkg, total_flux, max_flux, size): def main(): filename = sys.argv[1] - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) objcat = ad[0].OBJCAT data = ad[0].data catx = objcat.field("X_IMAGE") diff --git a/gempy/scripts/psf_plot.py b/gempy/scripts/psf_plot.py index 251f75c62a..c64c1a1645 100755 --- a/gempy/scripts/psf_plot.py +++ b/gempy/scripts/psf_plot.py @@ -13,7 +13,7 @@ def main(): filename = sys.argv[1] - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) objcat = ad[0].OBJCAT catx = objcat.field("X_IMAGE") caty = objcat.field("Y_IMAGE") diff --git a/gempy/scripts/showpars.py b/gempy/scripts/showpars.py index 41a1869c96..eae5fb2c4b 100755 --- a/gempy/scripts/showpars.py +++ b/gempy/scripts/showpars.py @@ -40,7 +40,7 @@ def get_pars(filename, adpkg=None, drpkg=None): if adpkg is not None: import_module(adpkg) - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) dtags = set(list(ad.tags)[:]) instpkg = ad.instrument(generic=True).lower() diff --git a/gempy/scripts/typewalk.py b/gempy/scripts/typewalk.py index 74b57d21c1..a707a6040d 100755 --- a/gempy/scripts/typewalk.py +++ b/gempy/scripts/typewalk.py @@ -230,7 +230,7 @@ def typewalk(self, directory=os.getcwd(), only=None, filemask=None, fname = os.path.join(root, tfile) try: - fl = astrodata.open(fname) + fl = astrodata.from_file(fname) dtypes = list(fl.tags) except AttributeError: print(" Bad headers in file: {}".format(tfile)) diff --git a/gempy/scripts/zp_histogram.py b/gempy/scripts/zp_histogram.py index 89db2e3db0..489ce5327f 100755 --- a/gempy/scripts/zp_histogram.py +++ b/gempy/scripts/zp_histogram.py @@ -23,7 +23,7 @@ def main(): filename = sys.argv[1] - ad = astrodata.open(filename) + ad = astrodata.from_file(filename) objcat = ad[0].OBJCAT # check if there's a REFCAT. If not, then the OBJCAT will be diff --git a/gempy/utils/showrecipes.py b/gempy/utils/showrecipes.py index fdbf67b8d4..d2b08f5099 100644 --- a/gempy/utils/showrecipes.py +++ b/gempy/utils/showrecipes.py @@ -38,7 +38,7 @@ def showrecipes(_file, adpkg=None, drpkg=None): # Find the file and open it with astrodata try: - ad = astrodata.open(_file) + ad = astrodata.from_file(_file) tags = ad.tags except AstroDataError: result += ("There was an issue using the selected file, please check " @@ -151,7 +151,7 @@ def showprims(_file, mode='sq', recipe='_default', adpkg=None, drpkg=None): # Find the file and open it with astrodata try: - ad = astrodata.open(_file) + ad = astrodata.from_file(_file) tags = ad.tags except AstroDataError: print("There was an issue using the selected file, please check " diff --git a/gempy/utils/tests/test_decorators.py b/gempy/utils/tests/test_decorators.py index ffac7f81f0..cd974a4aa2 100644 --- a/gempy/utils/tests/test_decorators.py +++ b/gempy/utils/tests/test_decorators.py @@ -20,4 +20,4 @@ def fn(data, exposure_time=5): @pytest.fixture(scope='function') def ad(): ad_path = download_from_archive("N20150123S0337.fits") - return astrodata.open(ad_path) + return astrodata.from_file(ad_path) diff --git a/recipe_system/cal_service/tests/test_caldb.py b/recipe_system/cal_service/tests/test_caldb.py index 8e2ea1ece8..827b068257 100644 --- a/recipe_system/cal_service/tests/test_caldb.py +++ b/recipe_system/cal_service/tests/test_caldb.py @@ -82,7 +82,7 @@ def test_retrieval(path_to_inputs, change_working_dir): cal_file = os.path.join(path_to_inputs, cal) caldb.add_cal(cal_file) - ad_sci = astrodata.open(os.path.join(path_to_inputs, sci)) + ad_sci = astrodata.from_file(os.path.join(path_to_inputs, sci)) for caltype, calfile in cals.items(): cal_return = caldb.get_calibrations([ad_sci], caltype=caltype) assert len(cal_return) == 1 diff --git a/recipe_system/cal_service/tests/test_calrequestlib.py b/recipe_system/cal_service/tests/test_calrequestlib.py index 22e28d9683..790c7bfb88 100644 --- a/recipe_system/cal_service/tests/test_calrequestlib.py +++ b/recipe_system/cal_service/tests/test_calrequestlib.py @@ -17,7 +17,7 @@ @pytest.mark.dragons_remote_data def test_get_cal_requests_dictdescriptor(): path = astrodata.testing.download_from_archive("S20221209S0007.fits") - ad = astrodata.open(path) + ad = astrodata.from_file(path) requests = get_cal_requests([ad], 'bias') # This descriptor works off the decomposed per-arm values for x/y binning # so it's a good test that this worked with a dictionary-based descriptor diff --git a/recipe_system/mappers/primitiveMapper.py b/recipe_system/mappers/primitiveMapper.py index b7f163b625..9b2f100b87 100644 --- a/recipe_system/mappers/primitiveMapper.py +++ b/recipe_system/mappers/primitiveMapper.py @@ -22,7 +22,7 @@ class PrimitiveMapper(Mapper): Retrieve the appropriate primitive class for a dataset, using all defined defaults: - >>> ad = astrodata.open() + >>> ad = astrodata.from_file() >>> dtags = set(list(ad.tags)[:]) >>> instpkg = ad.instrument(generic=True).lower() >>> pm = PrimitiveMapper(dtags, instpkg) diff --git a/recipe_system/mappers/recipeMapper.py b/recipe_system/mappers/recipeMapper.py index fe602eacb1..8627c7f3d8 100644 --- a/recipe_system/mappers/recipeMapper.py +++ b/recipe_system/mappers/recipeMapper.py @@ -22,7 +22,7 @@ class RecipeMapper(Mapper): """ Retrieve the appropriate recipe for a dataset, using all defined defaults: - >>> ad = astrodata.open() + >>> ad = astrodata.from_file() >>> dtags = set(list(ad.tags)[:]) >>> instpkg = ad.instrument(generic=True).lower() >>> rm = RecipeMapper(dtags, instpkg) diff --git a/recipe_system/reduction/coreReduce.py b/recipe_system/reduction/coreReduce.py index 5b6f8526e2..5e4383244c 100644 --- a/recipe_system/reduction/coreReduce.py +++ b/recipe_system/reduction/coreReduce.py @@ -243,7 +243,7 @@ def _convert_inputs(self, inputs): allinputs = [] for inp in inputs: try: - ad = astrodata.open(inp) + ad = astrodata.from_file(inp) except AstroDataError as err: log.warning("Can't Load Dataset: %s" % inp) log.warning(err) @@ -362,7 +362,7 @@ def _convert_inputs(inputs): allinputs = [] for inp in inputs: try: - ad = astrodata.open(inp) + ad = astrodata.from_file(inp) except AstroDataError as err: log.warning("Can't Load Dataset: %s" % inp) log.warning(err) diff --git a/recipe_system/scripts/provenance.py b/recipe_system/scripts/provenance.py index 235535238d..c332ec12a3 100644 --- a/recipe_system/scripts/provenance.py +++ b/recipe_system/scripts/provenance.py @@ -32,7 +32,7 @@ def parse_args(): options, args = parse_args() for arg in args: try: - ad = astrodata.open(arg) + ad = astrodata.from_file(arg) print(f"Reading Provenance for {arg}\n") print(provenance_summary(ad, provenance=options.provenance, history=options.history)) except astrodata.AstroDataError: diff --git a/recipe_system/testing.py b/recipe_system/testing.py index 55a64fc673..e778cc6428 100644 --- a/recipe_system/testing.py +++ b/recipe_system/testing.py @@ -40,9 +40,9 @@ def _get_master_arc(ad, pre_process): if pre_process: with change_working_dir(): - master_arc = astrodata.open(arc_filename) + master_arc = astrodata.from_file(arc_filename) else: - master_arc = astrodata.open( + master_arc = astrodata.from_file( os.path.join(path_to_inputs, arc_filename)) return master_arc @@ -144,7 +144,7 @@ def _reduce_flat(data_label, flat_fnames, master_bias): reduce.runr() master_flat = reduce.output_filenames.pop() - master_flat_ad = astrodata.open(master_flat) + master_flat_ad = astrodata.from_file(master_flat) return master_flat_ad @@ -169,7 +169,7 @@ def ref_ad_factory(path_to_refs): def _reference_ad(filename): print(f"Loading reference file: {filename}") path = os.path.join(path_to_refs, filename) - return astrodata.open(path) + return astrodata.from_file(path) return _reference_ad diff --git a/recipe_system/utils/reduce_utils.py b/recipe_system/utils/reduce_utils.py index 9c18426e13..232007b311 100644 --- a/recipe_system/utils/reduce_utils.py +++ b/recipe_system/utils/reduce_utils.py @@ -357,7 +357,7 @@ def normalize_ucals(cals): ctype, cpath = cal.split(":") scal, stype = ctype.split("_") caltags = {scal.upper(), stype.upper()} - cad = astrodata.open(cpath) + cad = astrodata.from_file(cpath) try: assert caltags.issubset(cad.tags) except AssertionError: diff --git a/recipe_system/utils/tests/test_decorator.py b/recipe_system/utils/tests/test_decorator.py index 5a2ba72237..ee01cc8fd3 100644 --- a/recipe_system/utils/tests/test_decorator.py +++ b/recipe_system/utils/tests/test_decorator.py @@ -14,7 +14,7 @@ def test_skip_primitive(change_working_dir, primitive_name, num_outputs): """Reduce some biases and confirm that the correct thing is skipped (or not)""" with change_working_dir(): files = [download_from_archive(f"N20210101S{i:04d}.fits") for i in range(534, 537)] - adinputs = [astrodata.open(f) for f in files] + adinputs = [astrodata.from_file(f) for f in files] p = GMOS(adinputs, uparms={f'{primitive_name}:skip_primitive': True}) makeProcessedBias(p) assert len(p.streams['main']) == num_outputs diff --git a/setup.cfg b/setup.cfg index f0df935b25..7009df2fc7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,7 +42,7 @@ convention=numpy [tool.isort] default_section = THIRDPARTY -known_first_party = astrodata,geminidr,gemini_instruments,gempy,recipe_system +known_first_party = geminidr,gemini_instruments,gempy,recipe_system multi_line_output = 0 balanced_wrapping = true include_trailing_comma = false diff --git a/setup.py b/setup.py index 864faf860a..79517bc44f 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ Setup script for gemini_python In this package: - astrodata gemini_instruments geminidr gempy @@ -80,6 +79,7 @@ 'Topic :: Scientific/Engineering :: Astronomy', ], install_requires=[ + 'astrodata', 'asdf>=2.7,!=2.10.0', 'astropy>=4.3,!=5.3.0,!=6.1.5,!=6.1.6', 'astroquery>=0.4', diff --git a/tox.ini b/tox.ini index 574a3cce51..ebbf2f10a0 100644 --- a/tox.ini +++ b/tox.ini @@ -44,6 +44,7 @@ conda_deps = pytest>=5.2 python-dateutil>=2.5.3 requests>=2.22 + setuptools scikit-image>=0.21 scipy>=1.3 sextractor>=2.8.6 @@ -62,6 +63,7 @@ extras = test docs: docs deps = + astrodata>=2.10.2 git+https://github.com/GeminiDRSoftware/FitsStorage.git@v3.4.0b2 git+https://github.com/GeminiDRSoftware/pytest_dragons.git@v1.0.5#egg=pytest_dragons changedir = @@ -75,17 +77,17 @@ commands = pip install --no-use-pep517 git+https://github.com/GeminiDRSoftware/AstroFaker#egg=AstroFaker conda list noop: python -c "pass" # just install deps & ensure python runs - unit: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "not integration_test and not gmos and not gmosls and not gmosimage and not f2 and not f2ls and not f2image and not gsaoi and not niri and not nirils and not niriimage and not gnirs and not gnirsls and not gnirsimage and not gnirsxd and not wavecal and not regression and not slow and not ghost and not ghostbundle and not ghostslit and not ghostspect" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - gmos: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(gmos or gmosimage) and not slow and not wavecal" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - gmosls: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "gmosls and not slow and not wavecal" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - wavecal: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "wavecal" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - f2: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(f2 or f2ls or f2image) and not slow and not wavecal" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - gsaoi: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "gsaoi and not slow" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - niri: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(niri or nirils or niriimage) and not slow and not wavecal" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - gnirs: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(gnirs or gnirsls or gnirsimage or gnirsxd) and not slow and not wavecal" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - ghost: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(ghost or ghostbundle or ghostslit or ghostspect) and not slow" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - reg: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "regression and not slow and not wavecal" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} - slow: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "slow and not wavecal" {posargs:astrodata geminidr gemini_instruments gempy recipe_system} + unit: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "not integration_test and not gmos and not gmosls and not gmosimage and not f2 and not f2ls and not f2image and not gsaoi and not niri and not nirils and not niriimage and not gnirs and not gnirsls and not gnirsimage and not gnirsxd and not wavecal and not regression and not slow and not ghost and not ghostbundle and not ghostslit and not ghostspect" {posargs:geminidr gemini_instruments gempy recipe_system} + gmos: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(gmos or gmosimage) and not slow and not wavecal" {posargs:geminidr gemini_instruments gempy recipe_system} + gmosls: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "gmosls and not slow and not wavecal" {posargs:geminidr gemini_instruments gempy recipe_system} + wavecal: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "wavecal" {posargs:geminidr gemini_instruments gempy recipe_system} + f2: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(f2 or f2ls or f2image) and not slow and not wavecal" {posargs:geminidr gemini_instruments gempy recipe_system} + gsaoi: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "gsaoi and not slow" {posargs:geminidr gemini_instruments gempy recipe_system} + niri: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(niri or nirils or niriimage) and not slow and not wavecal" {posargs:geminidr gemini_instruments gempy recipe_system} + gnirs: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(gnirs or gnirsls or gnirsimage or gnirsxd) and not slow and not wavecal" {posargs:geminidr gemini_instruments gempy recipe_system} + ghost: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "(ghost or ghostbundle or ghostslit or ghostspect) and not slow" {posargs:geminidr gemini_instruments gempy recipe_system} + reg: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "regression and not slow and not wavecal" {posargs:geminidr gemini_instruments gempy recipe_system} + slow: python -m coverage run --rcfile={toxinidir}/.coveragerc -m pytest -v --dragons-remote-data --durations=20 -m "slow and not wavecal" {posargs:geminidr gemini_instruments gempy recipe_system} docs: sphinx-build {posargs} . _build/html [testenv:covreport] @@ -114,9 +116,9 @@ whitelist_externals = commands = mkdir -p reports bash -c \'pylint --exit-zero --rcfile=gempy/support_files/pylintrc \ - astrodata gemini_instruments gempy geminidr recipe_system \ + gemini_instruments gempy geminidr recipe_system \ > reports/pylint.log\' bash -c \'pydocstyle --add-ignore D400,D401,D205,D105,D105 \ --match="(?!test_|conf).*\.py" \ - astrodata gemini_instruments gempy geminidr recipe_system \ + gemini_instruments gempy geminidr recipe_system \ > reports/pydocstyle.log || exit 0\'