Merge remote-tracking branch 'upstream/main' into develop

Author: juanmcloaiza

CHANGES.rst

@@ -34,6 +34,7 @@ gaia
 ^^^^
 
 - New datalink DR4 retrieval type RESIDUAL_IMAGE. [#3489]
+- The method ``load_data`` parses ecsv files [#3500].
 
 esa.hubble
 ^^^^^^^^^^

astroquery/gaia/core.py

@@ -20,6 +20,7 @@
 from astropy.coordinates import Angle
 from astropy.io import fits
 from astropy.io import votable
+from astropy.io.fits import TableHDU, BinTableHDU
 from astropy.table import Table
 from astropy.units import Quantity
 from astropy.utils.decorators import deprecated_renamed_argument
@@ -194,9 +195,9 @@ def load_data(self, ids, *, data_release=None, data_structure='INDIVIDUAL', retr
             'EPOCH_PHOTOMETRY_CROWDED_FIELD', 'EPOCH_IMAGE', 'EPOCH_PHOTOMETRY_CCD', 'EPOCH_SPECTRUM_XP_SSO',
             'EPOCH_SPECTRUM_XP_CROWDING', 'MEAN_SPECTRUM_XP', 'EPOCH_SPECTRUM_XP', 'CROWDED_FIELD_IMAGE',
             'EPOCH_ASTROMETRY_BRIGHT', 'MEAN_SPECTRUM_XP_GRAVLENS', 'EPOCH_FLAGS_NSS', 'EPOCH_PARAMETERS_RVS_SINGLE',
-            'EPOCH_PARAMETERS_RVS_DOUBLE', 'EPOCH_FLAGS_VARI']. Note that for 'CROWDED_FIELD_IMAGE', only the format
-            'fits' can be used, and its image, in the principal header, will not be available in the returned
-            dictionary. Set 'output_file' to retrieve all data: image + tables. Note that for 'RESIDUAL_IMAGE',
+            'EPOCH_PARAMETERS_RVS_DOUBLE', 'EPOCH_FLAGS_VARI', 'RESIDUAL_IMAGE']. Note that for 'CROWDED_FIELD_IMAGE',
+            only the format 'fits' can be used, and its image, in the principal header, will not be available in the
+            returned dictionary. Set 'output_file' to retrieve all data: image + tables. Note that for 'RESIDUAL_IMAGE',
             only the format 'fits' can be used. Since the fits files only contain images, the returned table will be
             empty. Therefore, set 'output_file' to save the files to get access to their content.
         linking_parameter : str, optional, default SOURCE_ID, valid values: SOURCE_ID, TRANSIT_ID, IMAGE_ID
@@ -308,7 +309,9 @@ def load_data(self, ids, *, data_release=None, data_structure='INDIVIDUAL', retr
                 shutil.rmtree(path)
             else:
                 for file in files.keys():
-                    os.remove(os.path.join(os.getcwd(), path, file))
+                    final_file = os.path.join(os.getcwd(), path, file)
+                    if os.path.isfile(final_file):
+                        os.remove(final_file)
 
         if verbose:
             if output_file_specified:
@@ -340,13 +343,13 @@ def __get_data_files(output_file, path):
 
             if key.endswith('.fits'):
                 tables = []
-                with fits.open(value) as hduList:
-                    num_hdus = len(hduList)
-                    for i in range(1, num_hdus):
-                        table = Table.read(hduList[i], format='fits')
-                        Gaia.correct_table_units(table)
-                        tables.append(table)
-                    files[key] = tables
+                with fits.open(value, memmap=False) as hduList:
+                    for hdu in hduList:
+                        if isinstance(hdu, (TableHDU, BinTableHDU)):
+                            table = Table.read(hdu, format='fits')
+                            Gaia.correct_table_units(table)
+                            tables.append(table)
+                files[key] = tables
 
             elif key.endswith('.xml'):
                 tables = []
@@ -360,6 +363,12 @@ def __get_data_files(output_file, path):
                 tables.append(table)
                 files[key] = tables
 
+            elif key.endswith('.ecsv'):
+                tables = []
+                table = Table.read(value, format='ascii.ecsv', fast_reader=False)
+                tables.append(table)
+                files[key] = tables
+
             elif key.endswith('.json'):
                 tables = []
                 with open(value) as f:

astroquery/gaia/tests/test_gaiatap.py

@@ -942,11 +942,14 @@ def test_datalink_querier_load_data_ecsv(mock_datalink_querier_ecsv):
 
     assert len(result_dict) == 3
 
-    files = list(result_dict.keys())
-    files.sort()
-    assert files[0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.ecsv'
-    assert files[1] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.ecsv'
-    assert files[2] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.ecsv'
+    files = list(sorted(result_dict.items()))
+    assert files[0][0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.ecsv'
+    assert files[1][0] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.ecsv'
+    assert files[2][0] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.ecsv'
+
+    assert isinstance(files[0][1][0], Table)
+    assert isinstance(files[1][1][0], Table)
+    assert isinstance(files[2][1][0], Table)
 
     os.remove(os.path.join(os.getcwd(), datalink_output))
 
@@ -980,18 +983,21 @@ def test_datalink_querier_load_data_csv(mock_datalink_querier_csv):
 
     assert len(result_dict) == 3
 
-    files = list(result_dict.keys())
-    files.sort()
-    assert files[0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.csv'
-    assert files[1] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.csv'
-    assert files[2] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.csv'
+    files = list(sorted(result_dict.items()))
+    assert files[0][0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.csv'
+    assert files[1][0] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.csv'
+    assert files[2][0] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.csv'
+
+    assert isinstance(files[0][1][0], Table)
+    assert isinstance(files[1][1][0], Table)
+    assert isinstance(files[2][1][0], Table)
 
     os.remove(os.path.join(os.getcwd(), datalink_output))
 
     assert not os.path.exists(datalink_output)
 
 
-@pytest.mark.skip(reason="Thes fits files generate an error relatate to the unit 'log(cm.s**-2)")
+@pytest.mark.filterwarnings("ignore:")
 def test_datalink_querier_load_data_fits(mock_datalink_querier_fits):
     result_dict = mock_datalink_querier_fits.load_data(ids=[5937083312263887616], data_release='Gaia DR3',
                                                        data_structure='INDIVIDUAL',
@@ -1019,11 +1025,14 @@ def test_datalink_querier_load_data_fits(mock_datalink_querier_fits):
 
     assert len(result_dict) == 3
 
-    files = list(result_dict.keys())
-    files.sort()
-    assert files[0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.fits'
-    assert files[1] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.fits'
-    assert files[2] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.fits'
+    files = list(sorted(result_dict.items()))
+    assert files[0][0] == 'MCMC_MSC-Gaia DR3 5937083312263887616.fits'
+    assert files[1][0] == 'XP_CONTINUOUS-Gaia DR3 5937083312263887616.fits'
+    assert files[2][0] == 'XP_SAMPLED-Gaia DR3 5937083312263887616.fits'
+
+    assert isinstance(files[0][1][0], Table)
+    assert isinstance(files[1][1][0], Table)
+    assert isinstance(files[2][1][0], Table)
 
     os.remove(os.path.join(os.getcwd(), datalink_output))
 
@@ -1072,10 +1081,12 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
     path.unlink()
 
 
-@pytest.mark.skip(reason="Thes fits files generate an error relatate to the unit 'log(cm.s**-2)")
-def test_load_data_fits(monkeypatch, tmp_path, tmp_path_factory):
+@pytest.mark.filterwarnings("ignore:")
+def test_load_data_fits(monkeypatch, tmp_path, tmp_path_factory, patch_datetime_now):
+    assert datetime.datetime.now(datetime.timezone.utc) == FAKE_TIME
+
     now = datetime.datetime.now(datetime.timezone.utc)
-    output_file = 'datalink_output_' + now.strftime("%Y%m%dT%H%M%S") + '.zip'
+    output_file = 'datalink_output_' + now.strftime("%Y%m%dT%H%M%S.%f") + '.zip'
 
     path = Path(os.getcwd(), output_file)
 
@@ -1092,7 +1103,7 @@ def load_data_monkeypatched(self, params_dict, output_file, verbose):
             "RETRIEVAL_TYPE": "epoch_photometry",
             "DATA_STRUCTURE": "INDIVIDUAL",
             "USE_ZIP_ALWAYS": "true"}
-        assert output_file == Path(os.getcwd(), 'datalink_output.zip')
+        assert str(path) == output_file
         assert verbose is True
 
     monkeypatch.setattr(TapPlus, "load_data", load_data_monkeypatched)

astroquery/simbad/tests/test_utils.py

@@ -1,4 +1,3 @@
-
 import pytest
 
 from astroquery.simbad.utils import (CriteriaTranslator, _parse_coordinate_and_convert_to_icrs,
@@ -82,9 +81,6 @@ def test_tokenizer():
 
 
 @pytest.mark.parametrize("test, result", [
-    ("region(GAL,180 0,2d) & otype = 'G' & (nbref >= 10|bibyear >= 2000)",
-     ("CONTAINS(POINT('ICRS', ra, dec), CIRCLE('ICRS', 86.40498828654475, 28.93617776179148, 2.0)) = 1"
-      " AND otypes.otype = 'G' AND (nbref >= 10 OR bibyear >= 2000)")),
     ("author ∼ 'egret*'", "regexp(author, '^egret.*$') = 1"),
     ("cat in ('hd','hip','ppm')", "cat IN ('hd','hip','ppm')"),
     ("author !~ 'test'", "regexp(author, '^test$') = 0"),
@@ -103,6 +99,17 @@ def test_transpiler(test, result):
     assert translated == result
 
 
+def test_transpiler_with_floats():
+    # This test is not part of the previous one as the floats in the string have
+    # some version dependent variations. Expected translation:
+    # ("CONTAINS(POINT('ICRS', ra, dec), CIRCLE('ICRS', 86.40498828654475, 28.93617776179148, 2.0)) = 1"
+    #  " AND otypes.otype = 'G' AND (nbref >= 10 OR bibyear >= 2000)")
+    translation = CriteriaTranslator.parse("region(GAL,180 0,2d) & otype = 'G' & (nbref >= 10|bibyear >= 2000)")
+
+    assert "CONTAINS(POINT('ICRS', ra, dec), CIRCLE('ICRS', 86.404988286" == translation[:60]
+    assert ", 2.0)) = 1 AND otypes.otype = 'G' AND (nbref >= 10 OR bibyear >= 2000)" == translation[-71:]
+
+
 def test_transpiler_errors(capsys):
     with pytest.raises(ValueError, match="Syntax error for sim-script criteria"):
         CriteriaTranslator.parse("otype % 'G'")

docs/eso/eso.rst

@@ -17,7 +17,7 @@ ESO Queries (`astroquery.eso`)
 
     In TAP, ``column_filters`` accepts ADQL expressions. For example:
 
-    .. doctest-skip::
+    .. code-block:: python
 
         column_filters = {
             'some_int_column': "< 5",
@@ -234,7 +234,6 @@ as certain columns with datatype ``char`` actually define timestamps or regions
 
     Number of records present in the table ist.midi:
     421764
-     [astroquery.eso.core]
 
 **Note:** for a deeper description of each column, the following query can be issued
 on the ESO **Programmatic Access** website (https://archive.eso.org/programmatic/#TAP):
@@ -249,6 +248,7 @@ target ``NGC 4151`` between ``2008-01-01`` and ``2009-05-12`` are searched, and
 return two columns: the date of observation and the name of the object.
 
 .. doctest-remote-data::
+
     >>> table = eso.query_instrument(
     ...             'midi',
     ...             column_filters={
@@ -477,7 +477,7 @@ Two concrete and relevant examples of fields present in WDB but not present in T
 are ``stime`` and ``etime``. The following snippet shows how to adapt the filters to
 the TAP / ADQL syntax:
 
-.. doctest-skip::
+.. code-block:: python
 
     # The following filters won't work:
     column_filters = {