diff --git a/.github/workflows/manual-create-job-from-cronjob.yml b/.github/workflows/manual-create-job-from-cronjob.yml index b883f8d4..35c46a8d 100644 --- a/.github/workflows/manual-create-job-from-cronjob.yml +++ b/.github/workflows/manual-create-job-from-cronjob.yml @@ -20,6 +20,8 @@ name: 'Manual: Create Job from CronJob' - noaa-gefs-forecast-35-day-validate - noaa-gfs-forecast-update - noaa-gfs-forecast-validate + - noaa-hrrr-analysis-update + - noaa-hrrr-analysis-validate - noaa-hrrr-forecast-48-hour-update - noaa-hrrr-forecast-48-hour-validate - noaa-ndvi-cdr-analysis-update diff --git a/src/reformatters/__main__.py b/src/reformatters/__main__.py index 97645eba..85ef4f64 100644 --- a/src/reformatters/__main__.py +++ b/src/reformatters/__main__.py @@ -1,4 +1,5 @@ import contextlib +import faulthandler import multiprocessing import os from collections.abc import Sequence @@ -8,7 +9,6 @@ with contextlib.suppress(RuntimeError): # skip if already set multiprocessing.set_start_method("spawn", force=True) - import sentry_sdk import typer from sentry_sdk.integrations.typer import TyperIntegration @@ -33,10 +33,15 @@ GefsForecast35DayDataset, ) from reformatters.noaa.gfs.forecast import NoaaGfsForecastDataset +from reformatters.noaa.hrrr.analysis.dynamical_dataset import ( + NoaaHrrrAnalysisDataset, +) from reformatters.noaa.hrrr.forecast_48_hour.dynamical_dataset import ( NoaaHrrrForecast48HourDataset, ) +faulthandler.enable() + class NoaaHrrrIcechunkAwsOpenDataDatasetStorageConfig(StorageConfig): """Configuration for the storage of a AWS Open Data dataset.""" @@ -98,6 +103,10 @@ class UpstreamGriddedZarrsDatasetStorageConfig(StorageConfig): primary_storage_config=SourceCoopZarrDatasetStorageConfig(), replica_storage_configs=[NoaaHrrrIcechunkAwsOpenDataDatasetStorageConfig()], ), + NoaaHrrrAnalysisDataset( + primary_storage_config=SourceCoopZarrDatasetStorageConfig(), + replica_storage_configs=[NoaaHrrrIcechunkAwsOpenDataDatasetStorageConfig()], + ), # ECMWF EcmwfIfsEnsForecast15Day025DegreeDataset( primary_storage_config=SourceCoopZarrDatasetStorageConfig(), diff --git a/src/reformatters/noaa/hrrr/analysis/__init__.py b/src/reformatters/noaa/hrrr/analysis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/reformatters/noaa/hrrr/analysis/dynamical_dataset.py b/src/reformatters/noaa/hrrr/analysis/dynamical_dataset.py new file mode 100644 index 00000000..db95bd3f --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/dynamical_dataset.py @@ -0,0 +1,67 @@ +from collections.abc import Iterable, Sequence +from datetime import timedelta + +from reformatters.common import validation +from reformatters.common.dynamical_dataset import DynamicalDataset +from reformatters.common.kubernetes import ( + CronJob, + ReformatCronJob, + ValidationCronJob, +) +from reformatters.noaa.hrrr.hrrr_config_models import NoaaHrrrDataVar +from reformatters.noaa.hrrr.region_job import NoaaHrrrSourceFileCoord + +from .region_job import NoaaHrrrAnalysisRegionJob +from .template_config import NoaaHrrrAnalysisTemplateConfig + + +class NoaaHrrrAnalysisDataset( + DynamicalDataset[NoaaHrrrDataVar, NoaaHrrrSourceFileCoord] +): + """DynamicalDataset implementation for NOAA HRRR analysis.""" + + template_config: NoaaHrrrAnalysisTemplateConfig = NoaaHrrrAnalysisTemplateConfig() + region_job_class: type[NoaaHrrrAnalysisRegionJob] = NoaaHrrrAnalysisRegionJob + + def operational_kubernetes_resources(self, image_tag: str) -> Iterable[CronJob]: + """Define Kubernetes cron jobs for operational updates and validation.""" + operational_update_cron_job = ReformatCronJob( + name=f"{self.dataset_id}-update", + # Every 3 hours at 57 minutes past the hour. + # Data for forecast hour 0 is available 54 mins after init time on most issuances + # We could of course increase this to hourly + schedule="57 */3 * * *", + pod_active_deadline=timedelta(minutes=20), + image=image_tag, + dataset_id=self.dataset_id, + cpu="14", + memory="45G", + shared_memory="16.5G", + ephemeral_storage="42G", + secret_names=self.store_factory.k8s_secret_names(), + suspend=True, + ) + + validation_cron_job = ValidationCronJob( + name=f"{self.dataset_id}-validate", + # Run this 23 mins after the operational update, which is at 57 */3 * * * + # That is, at 57 + 23 = 80th min of the hour, which is 20 mins into the next hour. + # To get every third hour, but offset by +1 hr 20 min versus the main cron, + # we do "20 1-23/3 * * *" + schedule="20 1-23/3 * * *", + pod_active_deadline=timedelta(minutes=10), + image=image_tag, + dataset_id=self.dataset_id, + cpu="0.7", + memory="3.5G", + secret_names=self.store_factory.k8s_secret_names(), + suspend=True, + ) + + return [operational_update_cron_job, validation_cron_job] + + def validators(self) -> Sequence[validation.DataValidator]: + return ( + validation.check_analysis_current_data, + validation.check_analysis_recent_nans, + ) diff --git a/src/reformatters/noaa/hrrr/analysis/region_job.py b/src/reformatters/noaa/hrrr/analysis/region_job.py new file mode 100644 index 00000000..99522eb7 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/region_job.py @@ -0,0 +1,61 @@ +from collections.abc import Mapping, Sequence + +import pandas as pd +import xarray as xr + +from reformatters.common.iterating import item +from reformatters.common.logging import get_logger +from reformatters.common.region_job import ( + CoordinateValueOrRange, +) +from reformatters.common.types import ( + Dim, +) +from reformatters.noaa.hrrr.hrrr_config_models import ( + NoaaHrrrDataVar, +) +from reformatters.noaa.hrrr.region_job import NoaaHrrrRegionJob, NoaaHrrrSourceFileCoord +from reformatters.noaa.noaa_utils import has_hour_0_values + +log = get_logger(__name__) + + +class NoaaHrrrAnalysisSourceFileCoord(NoaaHrrrSourceFileCoord): + def out_loc(self) -> Mapping[Dim, CoordinateValueOrRange]: + return {"time": self.init_time + self.lead_time} + + +class NoaaHrrrAnalysisRegionJob(NoaaHrrrRegionJob): + """Region job for HRRR analysis data processing.""" + + def get_processing_region(self) -> slice: + """Buffer start by one step to allow deaccumulation without gaps in resulting output.""" + return slice(max(0, self.region.start - 1), self.region.stop) + + def generate_source_file_coords( + self, + processing_region_ds: xr.Dataset, + data_var_group: Sequence[NoaaHrrrDataVar], + ) -> Sequence[NoaaHrrrAnalysisSourceFileCoord]: + times = pd.to_datetime(processing_region_ds["time"].values) + group_has_hour_0 = item({has_hour_0_values(var) for var in data_var_group}) + + if group_has_hour_0: + init_times = times + lead_time = pd.Timedelta("0h") + else: + init_times = times - pd.Timedelta(hours=1) + lead_time = pd.Timedelta("1h") + + file_type = item({var.internal_attrs.hrrr_file_type for var in data_var_group}) + + return [ + NoaaHrrrAnalysisSourceFileCoord( + init_time=init_time, + lead_time=lead_time, + domain="conus", + file_type=file_type, + data_vars=data_var_group, + ) + for init_time in init_times + ] diff --git a/src/reformatters/noaa/hrrr/analysis/template_config.py b/src/reformatters/noaa/hrrr/analysis/template_config.py new file mode 100644 index 00000000..76729b5a --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/template_config.py @@ -0,0 +1,130 @@ +from collections.abc import Sequence +from typing import Any + +import numpy as np +import pandas as pd +import xarray as xr +from pydantic import computed_field + +from reformatters.common.config_models import ( + Coordinate, + CoordinateAttrs, + DatasetAttributes, + Encoding, + StatisticsApproximate, +) +from reformatters.common.template_config import SPATIAL_REF_COORDS +from reformatters.common.time_utils import whole_hours +from reformatters.common.types import AppendDim, Dim, Timedelta, Timestamp +from reformatters.common.zarr import ( + BLOSC_4BYTE_ZSTD_LEVEL3_SHUFFLE, + BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE, +) +from reformatters.noaa.hrrr.hrrr_config_models import ( + NoaaHrrrDataVar, +) +from reformatters.noaa.hrrr.template_config import NoaaHrrrCommonTemplateConfig + + +class NoaaHrrrAnalysisTemplateConfig(NoaaHrrrCommonTemplateConfig): + dims: tuple[Dim, ...] = ("time", "y", "x") + append_dim: AppendDim = "time" + # Could go as early as 2018-07-13T12:00, but we start at 2018-09-16T00:00 to avoid a high rate of missing data in the first 2 months. + append_dim_start: Timestamp = pd.Timestamp("2018-09-16T00:00") + append_dim_frequency: Timedelta = pd.Timedelta("1h") + + @computed_field # type: ignore[prop-decorator] + @property + def dataset_attributes(self) -> DatasetAttributes: + return DatasetAttributes( + dataset_id="noaa-hrrr-analysis", + dataset_version="0.1.0", + name="NOAA HRRR analysis", + description="Analysis data from the High-Resolution Rapid Refresh (HRRR) model operated by NOAA NWS NCEP.", + attribution="NOAA NWS NCEP HRRR data processed by dynamical.org from NOAA Open Data Dissemination archives.", + spatial_domain="Continental United States", + spatial_resolution="3 km", + time_domain=f"{self.append_dim_start} UTC to Present", + time_resolution=f"{whole_hours(self.append_dim_frequency)} hour", + ) + + def dimension_coordinates(self) -> dict[str, Any]: + y_coords, x_coords = self._y_x_coordinates() + return { + "time": self.append_dim_coordinates( + self.append_dim_start + self.append_dim_frequency + ), + "y": y_coords, + "x": x_coords, + } + + def derive_coordinates( + self, ds: xr.Dataset + ) -> dict[str, xr.DataArray | tuple[tuple[str, ...], np.ndarray[Any, Any]]]: + latitudes, longitudes = self._latitude_longitude_coordinates( + ds["x"].values, ds["y"].values + ) + + return { + "latitude": (("y", "x"), latitudes), + "longitude": (("y", "x"), longitudes), + "spatial_ref": SPATIAL_REF_COORDS, + } + + @computed_field # type: ignore[prop-decorator] + @property + def coords(self) -> Sequence[Coordinate]: + append_dim_coordinate_chunk_size = self.append_dim_coordinate_chunk_size() + + hrrr_common_coords = super().coords + + return [ + *hrrr_common_coords, + Coordinate( + name="time", + encoding=Encoding( + dtype="int64", + fill_value=0, + compressors=[BLOSC_8BYTE_ZSTD_LEVEL3_SHUFFLE], + calendar="proleptic_gregorian", + units="seconds since 1970-01-01 00:00:00", + chunks=append_dim_coordinate_chunk_size, + shards=None, + ), + attrs=CoordinateAttrs( + units="seconds since 1970-01-01 00:00:00", + statistics_approximate=StatisticsApproximate( + min=self.append_dim_start.isoformat(), max="Present" + ), + ), + ), + ] + + @computed_field # type: ignore[prop-decorator] + @property + def data_vars(self) -> Sequence[NoaaHrrrDataVar]: + # ~18MB uncompressed, ~3.5MB compressed + var_chunks: dict[Dim, int] = { + "time": 90 * 24, # 90 days of hourly data + # Chunks are 135 km by 135 km spatially + "x": 45, # 40 chunks over 1799 pixels + "y": 45, # 24 chunks over 1059 pixels + } + + # ~2GB uncompressed, ~400MB compressed + var_shards: dict[Dim, int] = { + "time": var_chunks["time"], + # Shards are 1350 km by 1620 km spatially + "x": var_chunks["x"] * 10, # 4 shards over 1799 pixels + "y": var_chunks["y"] * 12, # 2 shards over 1059 pixels + } + + encoding_float32_default = Encoding( + dtype="float32", + fill_value=np.nan, + chunks=tuple(var_chunks[d] for d in self.dims), + shards=tuple(var_shards[d] for d in self.dims), + compressors=[BLOSC_4BYTE_ZSTD_LEVEL3_SHUFFLE], + ) + + return self.get_data_vars(encoding_float32_default) diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_freezing_rain_surface/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_freezing_rain_surface/zarr.json new file mode 100644 index 00000000..eb976cde --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_freezing_rain_surface/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Categorical freezing rain", + "short_name": "cfrzr", + "units": "0=no; 1=yes", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_ice_pellets_surface/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_ice_pellets_surface/zarr.json new file mode 100644 index 00000000..b10f0d15 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_ice_pellets_surface/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Categorical ice pellets", + "short_name": "cicep", + "units": "0=no; 1=yes", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_rain_surface/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_rain_surface/zarr.json new file mode 100644 index 00000000..3111fab1 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_rain_surface/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Categorical rain", + "short_name": "crain", + "units": "0=no; 1=yes", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_snow_surface/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_snow_surface/zarr.json new file mode 100644 index 00000000..b214cbeb --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/categorical_snow_surface/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Categorical snow", + "short_name": "csnow", + "units": "0=no; 1=yes", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/composite_reflectivity/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/composite_reflectivity/zarr.json new file mode 100644 index 00000000..c704f90d --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/composite_reflectivity/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Composite reflectivity", + "short_name": "refc", + "units": "dBZ", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/downward_long_wave_radiation_flux_surface/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/downward_long_wave_radiation_flux_surface/zarr.json new file mode 100644 index 00000000..4320ea9f --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/downward_long_wave_radiation_flux_surface/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Surface downward long-wave radiation flux", + "short_name": "sdlwrf", + "units": "W/(m^2)", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/downward_short_wave_radiation_flux_surface/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/downward_short_wave_radiation_flux_surface/zarr.json new file mode 100644 index 00000000..f7fec58f --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/downward_short_wave_radiation_flux_surface/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Surface downward short-wave radiation flux", + "short_name": "sdswrf", + "units": "W/(m^2)", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/geopotential_height_cloud_ceiling/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/geopotential_height_cloud_ceiling/zarr.json new file mode 100644 index 00000000..3eb2ced7 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/geopotential_height_cloud_ceiling/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Geopotential height", + "short_name": "gh", + "units": "gpm", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/latitude/c/0/0 b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/latitude/c/0/0 new file mode 100644 index 00000000..b621ae39 Binary files /dev/null and b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/latitude/c/0/0 differ diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/latitude/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/latitude/zarr.json new file mode 100644 index 00000000..50549e2a --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/latitude/zarr.json @@ -0,0 +1,56 @@ +{ + "shape": [ + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1059, + 1799 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "degrees_north", + "statistics_approximate": { + "min": 21.138123, + "max": 52.615653 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/longitude/c/0/0 b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/longitude/c/0/0 new file mode 100644 index 00000000..d5c8af9a Binary files /dev/null and b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/longitude/c/0/0 differ diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/longitude/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/longitude/zarr.json new file mode 100644 index 00000000..b1bba202 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/longitude/zarr.json @@ -0,0 +1,56 @@ +{ + "shape": [ + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1059, + 1799 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "degrees_east", + "statistics_approximate": { + "min": -134.09548, + "max": -60.917192 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/percent_frozen_precipitation_surface/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/percent_frozen_precipitation_surface/zarr.json new file mode 100644 index 00000000..f43262c1 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/percent_frozen_precipitation_surface/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Percent frozen precipitation", + "short_name": "cpofp", + "units": "%", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/precipitable_water_atmosphere/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/precipitable_water_atmosphere/zarr.json new file mode 100644 index 00000000..d89f80aa --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/precipitable_water_atmosphere/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Precipitable water", + "short_name": "pwat", + "units": "kg/(m^2)", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/precipitation_surface/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/precipitation_surface/zarr.json new file mode 100644 index 00000000..4a158fa6 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/precipitation_surface/zarr.json @@ -0,0 +1,84 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Total Precipitation", + "short_name": "tp", + "units": "mm/s", + "comment": "Average precipitation rate since the previous forecast step.", + "step_type": "avg", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/pressure_reduced_to_mean_sea_level/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/pressure_reduced_to_mean_sea_level/zarr.json new file mode 100644 index 00000000..36c4454e --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/pressure_reduced_to_mean_sea_level/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Pressure reduced to MSL", + "short_name": "prmsl", + "units": "Pa", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/pressure_surface/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/pressure_surface/zarr.json new file mode 100644 index 00000000..24f5cf71 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/pressure_surface/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Surface pressure", + "short_name": "sp", + "units": "Pa", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/relative_humidity_2m/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/relative_humidity_2m/zarr.json new file mode 100644 index 00000000..3366a020 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/relative_humidity_2m/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "2 metre relative humidity", + "short_name": "r2", + "units": "%", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/spatial_ref/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/spatial_ref/zarr.json new file mode 100644 index 00000000..5781a177 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/spatial_ref/zarr.json @@ -0,0 +1,58 @@ +{ + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "crs_wkt": "PROJCS[\"unnamed\",GEOGCS[\"Coordinate System imported from GRIB file\",DATUM[\"unnamed\",SPHEROID[\"Sphere\",6371229,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]]],PROJECTION[\"Lambert_Conformal_Conic_2SP\"],PARAMETER[\"latitude_of_origin\",38.5],PARAMETER[\"central_meridian\",-97.5],PARAMETER[\"standard_parallel_1\",38.5],PARAMETER[\"standard_parallel_2\",38.5],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"Metre\",1],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH]]", + "semi_major_axis": 6371229.0, + "semi_minor_axis": 6371229.0, + "inverse_flattening": 0.0, + "reference_ellipsoid_name": "Sphere", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "Coordinate System imported from GRIB file", + "horizontal_datum_name": "unnamed", + "grid_mapping_name": "lambert_conformal_conic", + "spatial_ref": "PROJCS[\"unnamed\",GEOGCS[\"Coordinate System imported from GRIB file\",DATUM[\"unnamed\",SPHEROID[\"Sphere\",6371229,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]]],PROJECTION[\"Lambert_Conformal_Conic_2SP\"],PARAMETER[\"latitude_of_origin\",38.5],PARAMETER[\"central_meridian\",-97.5],PARAMETER[\"standard_parallel_1\",38.5],PARAMETER[\"standard_parallel_2\",38.5],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"Metre\",1],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH]]", + "GeoTransform": "-2699020.142521929 3000.0 0.0 1588193.847443335 0.0 -3000.0", + "false_easting": 0.0, + "false_northing": 0.0, + "latitude_of_projection_origin": 38.5, + "longitude_of_central_meridian": -97.5, + "projected_crs_name": "unnamed", + "standard_parallel": [ + 38.5, + 38.5 + ] + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/temperature_2m/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/temperature_2m/zarr.json new file mode 100644 index 00000000..978ddfae --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/temperature_2m/zarr.json @@ -0,0 +1,84 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "2 metre temperature", + "short_name": "t2m", + "standard_name": "air_temperature", + "units": "C", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/time/c/0 b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/time/c/0 new file mode 100644 index 00000000..44913bcc Binary files /dev/null and b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/time/c/0 differ diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/time/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/time/zarr.json new file mode 100644 index 00000000..e66ed0c4 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/time/zarr.json @@ -0,0 +1,53 @@ +{ + "shape": [ + 1 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 192720 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "2018-09-16T00:00:00", + "max": "Present" + }, + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/total_cloud_cover_atmosphere/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/total_cloud_cover_atmosphere/zarr.json new file mode 100644 index 00000000..2622c579 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/total_cloud_cover_atmosphere/zarr.json @@ -0,0 +1,83 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Total Cloud Cover", + "short_name": "tcc", + "units": "%", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_u_10m/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_u_10m/zarr.json new file mode 100644 index 00000000..19d931c8 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_u_10m/zarr.json @@ -0,0 +1,84 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "10 metre U wind component", + "short_name": "u10", + "standard_name": "eastward_wind", + "units": "m/s", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_u_80m/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_u_80m/zarr.json new file mode 100644 index 00000000..4bb3d971 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_u_80m/zarr.json @@ -0,0 +1,84 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "U-component of wind (80 m above ground)", + "short_name": "u80", + "standard_name": "eastward_wind", + "units": "m/s", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_v_10m/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_v_10m/zarr.json new file mode 100644 index 00000000..2ff73ead --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_v_10m/zarr.json @@ -0,0 +1,84 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "10 metre V wind component", + "short_name": "v10", + "standard_name": "northward_wind", + "units": "m/s", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_v_80m/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_v_80m/zarr.json new file mode 100644 index 00000000..e8824192 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/wind_v_80m/zarr.json @@ -0,0 +1,84 @@ +{ + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "V-component of wind (80 m above ground)", + "short_name": "v80", + "standard_name": "northward_wind", + "units": "m/s", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/x/c/0 b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/x/c/0 new file mode 100644 index 00000000..c06ba1cf Binary files /dev/null and b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/x/c/0 differ diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/x/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/x/zarr.json new file mode 100644 index 00000000..e0f8426c --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/x/zarr.json @@ -0,0 +1,53 @@ +{ + "shape": [ + 1799 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1799 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "m", + "statistics_approximate": { + "min": -2700000.0, + "max": 2700000.0 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/y/c/0 b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/y/c/0 new file mode 100644 index 00000000..616c4dbc Binary files /dev/null and b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/y/c/0 differ diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/y/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/y/zarr.json new file mode 100644 index 00000000..f70cf8d1 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/y/zarr.json @@ -0,0 +1,53 @@ +{ + "shape": [ + 1059 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1059 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "m", + "statistics_approximate": { + "min": -1600000.0, + "max": 1600000.0 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/zarr.json b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/zarr.json new file mode 100644 index 00000000..fa4f3a82 --- /dev/null +++ b/src/reformatters/noaa/hrrr/analysis/templates/latest.zarr/zarr.json @@ -0,0 +1,2016 @@ +{ + "attributes": { + "dataset_id": "noaa-hrrr-analysis", + "dataset_version": "0.1.0", + "name": "NOAA HRRR analysis", + "description": "Analysis data from the High-Resolution Rapid Refresh (HRRR) model operated by NOAA NWS NCEP.", + "attribution": "NOAA NWS NCEP HRRR data processed by dynamical.org from NOAA Open Data Dissemination archives.", + "spatial_domain": "Continental United States", + "spatial_resolution": "3 km", + "time_domain": "2018-09-16 00:00:00 UTC to Present", + "time_resolution": "1 hour" + }, + "zarr_format": 3, + "consolidated_metadata": { + "kind": "inline", + "must_understand": false, + "metadata": { + "categorical_freezing_rain_surface": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Categorical freezing rain", + "short_name": "cfrzr", + "units": "0=no; 1=yes", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "categorical_ice_pellets_surface": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Categorical ice pellets", + "short_name": "cicep", + "units": "0=no; 1=yes", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "categorical_rain_surface": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Categorical rain", + "short_name": "crain", + "units": "0=no; 1=yes", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "categorical_snow_surface": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Categorical snow", + "short_name": "csnow", + "units": "0=no; 1=yes", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "composite_reflectivity": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Composite reflectivity", + "short_name": "refc", + "units": "dBZ", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "downward_long_wave_radiation_flux_surface": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Surface downward long-wave radiation flux", + "short_name": "sdlwrf", + "units": "W/(m^2)", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "downward_short_wave_radiation_flux_surface": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Surface downward short-wave radiation flux", + "short_name": "sdswrf", + "units": "W/(m^2)", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "geopotential_height_cloud_ceiling": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Geopotential height", + "short_name": "gh", + "units": "gpm", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "latitude": { + "shape": [ + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1059, + 1799 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "degrees_north", + "statistics_approximate": { + "min": 21.138123, + "max": 52.615653 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "longitude": { + "shape": [ + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1059, + 1799 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "degrees_east", + "statistics_approximate": { + "min": -134.09548, + "max": -60.917192 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "percent_frozen_precipitation_surface": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Percent frozen precipitation", + "short_name": "cpofp", + "units": "%", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "precipitable_water_atmosphere": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Precipitable water", + "short_name": "pwat", + "units": "kg/(m^2)", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "precipitation_surface": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Total Precipitation", + "short_name": "tp", + "units": "mm/s", + "comment": "Average precipitation rate since the previous forecast step.", + "step_type": "avg", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "pressure_reduced_to_mean_sea_level": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Pressure reduced to MSL", + "short_name": "prmsl", + "units": "Pa", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "pressure_surface": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Surface pressure", + "short_name": "sp", + "units": "Pa", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "relative_humidity_2m": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "2 metre relative humidity", + "short_name": "r2", + "units": "%", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "spatial_ref": { + "shape": [], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "zstd", + "configuration": { + "level": 0, + "checksum": false + } + } + ], + "attributes": { + "crs_wkt": "PROJCS[\"unnamed\",GEOGCS[\"Coordinate System imported from GRIB file\",DATUM[\"unnamed\",SPHEROID[\"Sphere\",6371229,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]]],PROJECTION[\"Lambert_Conformal_Conic_2SP\"],PARAMETER[\"latitude_of_origin\",38.5],PARAMETER[\"central_meridian\",-97.5],PARAMETER[\"standard_parallel_1\",38.5],PARAMETER[\"standard_parallel_2\",38.5],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"Metre\",1],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH]]", + "semi_major_axis": 6371229.0, + "semi_minor_axis": 6371229.0, + "inverse_flattening": 0.0, + "reference_ellipsoid_name": "Sphere", + "longitude_of_prime_meridian": 0.0, + "prime_meridian_name": "Greenwich", + "geographic_crs_name": "Coordinate System imported from GRIB file", + "horizontal_datum_name": "unnamed", + "grid_mapping_name": "lambert_conformal_conic", + "spatial_ref": "PROJCS[\"unnamed\",GEOGCS[\"Coordinate System imported from GRIB file\",DATUM[\"unnamed\",SPHEROID[\"Sphere\",6371229,0]],PRIMEM[\"Greenwich\",0],UNIT[\"degree\",0.0174532925199433,AUTHORITY[\"EPSG\",\"9122\"]]],PROJECTION[\"Lambert_Conformal_Conic_2SP\"],PARAMETER[\"latitude_of_origin\",38.5],PARAMETER[\"central_meridian\",-97.5],PARAMETER[\"standard_parallel_1\",38.5],PARAMETER[\"standard_parallel_2\",38.5],PARAMETER[\"false_easting\",0],PARAMETER[\"false_northing\",0],UNIT[\"Metre\",1],AXIS[\"Easting\",EAST],AXIS[\"Northing\",NORTH]]", + "GeoTransform": "-2699020.142521929 3000.0 0.0 1588193.847443335 0.0 -3000.0", + "false_easting": 0.0, + "false_northing": 0.0, + "latitude_of_projection_origin": 38.5, + "longitude_of_central_meridian": -97.5, + "projected_crs_name": "unnamed", + "standard_parallel": [ + 38.5, + 38.5 + ] + }, + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "temperature_2m": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "2 metre temperature", + "short_name": "t2m", + "standard_name": "air_temperature", + "units": "C", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "time": { + "shape": [ + 1 + ], + "data_type": "int64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 192720 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": 0, + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "statistics_approximate": { + "min": "2018-09-16T00:00:00", + "max": "Present" + }, + "units": "seconds since 1970-01-01", + "calendar": "proleptic_gregorian" + }, + "dimension_names": [ + "time" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "total_cloud_cover_atmosphere": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "Total Cloud Cover", + "short_name": "tcc", + "units": "%", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "wind_u_10m": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "10 metre U wind component", + "short_name": "u10", + "standard_name": "eastward_wind", + "units": "m/s", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "wind_u_80m": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "U-component of wind (80 m above ground)", + "short_name": "u80", + "standard_name": "eastward_wind", + "units": "m/s", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "wind_v_10m": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "10 metre V wind component", + "short_name": "v10", + "standard_name": "northward_wind", + "units": "m/s", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "wind_v_80m": { + "shape": [ + 1, + 1059, + 1799 + ], + "data_type": "float32", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 2160, + 540, + 450 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "sharding_indexed", + "configuration": { + "chunk_shape": [ + 2160, + 45, + 45 + ], + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 4, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "index_codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "crc32c" + } + ], + "index_location": "end" + } + } + ], + "attributes": { + "long_name": "V-component of wind (80 m above ground)", + "short_name": "v80", + "standard_name": "northward_wind", + "units": "m/s", + "step_type": "instant", + "coordinates": "latitude longitude spatial_ref", + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "time", + "y", + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "x": { + "shape": [ + 1799 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1799 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "m", + "statistics_approximate": { + "min": -2700000.0, + "max": 2700000.0 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "x" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + }, + "y": { + "shape": [ + 1059 + ], + "data_type": "float64", + "chunk_grid": { + "name": "regular", + "configuration": { + "chunk_shape": [ + 1059 + ] + } + }, + "chunk_key_encoding": { + "name": "default", + "configuration": { + "separator": "/" + } + }, + "fill_value": "NaN", + "codecs": [ + { + "name": "bytes", + "configuration": { + "endian": "little" + } + }, + { + "name": "blosc", + "configuration": { + "typesize": 8, + "cname": "zstd", + "clevel": 3, + "shuffle": "shuffle", + "blocksize": 0 + } + } + ], + "attributes": { + "units": "m", + "statistics_approximate": { + "min": -1600000.0, + "max": 1600000.0 + }, + "_FillValue": "AAAAAAAA+H8=" + }, + "dimension_names": [ + "y" + ], + "zarr_format": 3, + "node_type": "array", + "storage_transformers": [] + } + } + }, + "node_type": "group" +} \ No newline at end of file diff --git a/src/reformatters/noaa/hrrr/forecast_48_hour/template_config.py b/src/reformatters/noaa/hrrr/forecast_48_hour/template_config.py index 15f06faa..30ddec2b 100644 --- a/src/reformatters/noaa/hrrr/forecast_48_hour/template_config.py +++ b/src/reformatters/noaa/hrrr/forecast_48_hour/template_config.py @@ -47,10 +47,10 @@ def dataset_attributes(self) -> DatasetAttributes: dataset_id="noaa-hrrr-forecast-48-hour", dataset_version="0.1.0", name="NOAA HRRR forecast, 48 hour", - description="Weather forecasts from the High Resolution Rapid Refresh (HRRR) model operated by NOAA NWS NCEP.", + description="Weather forecasts from the High-Resolution Rapid Refresh (HRRR) model operated by NOAA NWS NCEP.", attribution="NOAA NWS NCEP HRRR data processed by dynamical.org from NOAA Open Data Dissemination archives.", spatial_domain="Continental United States", - spatial_resolution="3km", + spatial_resolution="3 km", time_domain=f"Forecasts initialized {self.append_dim_start} UTC to Present", time_resolution="Forecasts initialized every 6 hours", forecast_domain="Forecast lead time 0-48 hours ahead", diff --git a/src/reformatters/noaa/hrrr/forecast_48_hour/templates/latest.zarr/zarr.json b/src/reformatters/noaa/hrrr/forecast_48_hour/templates/latest.zarr/zarr.json index ad2e3d0b..11670eb6 100644 --- a/src/reformatters/noaa/hrrr/forecast_48_hour/templates/latest.zarr/zarr.json +++ b/src/reformatters/noaa/hrrr/forecast_48_hour/templates/latest.zarr/zarr.json @@ -3,10 +3,10 @@ "dataset_id": "noaa-hrrr-forecast-48-hour", "dataset_version": "0.1.0", "name": "NOAA HRRR forecast, 48 hour", - "description": "Weather forecasts from the High Resolution Rapid Refresh (HRRR) model operated by NOAA NWS NCEP.", + "description": "Weather forecasts from the High-Resolution Rapid Refresh (HRRR) model operated by NOAA NWS NCEP.", "attribution": "NOAA NWS NCEP HRRR data processed by dynamical.org from NOAA Open Data Dissemination archives.", "spatial_domain": "Continental United States", - "spatial_resolution": "3km", + "spatial_resolution": "3 km", "time_domain": "Forecasts initialized 2018-07-13 12:00:00 UTC to Present", "time_resolution": "Forecasts initialized every 6 hours", "forecast_domain": "Forecast lead time 0-48 hours ahead", diff --git a/src/reformatters/noaa/hrrr/region_job.py b/src/reformatters/noaa/hrrr/region_job.py index fd828f29..567efd1b 100644 --- a/src/reformatters/noaa/hrrr/region_job.py +++ b/src/reformatters/noaa/hrrr/region_job.py @@ -171,11 +171,14 @@ def apply_data_transformations( """Apply in-place data transformations to the output data array for a given data variable.""" if data_var.internal_attrs.deaccumulate_to_rate: assert data_var.internal_attrs.window_reset_frequency is not None - log.info(f"Converting {data_var.name} from accumulations to rates") + deaccum_dim = "lead_time" if "lead_time" in data_array.dims else "time" + log.info( + f"Converting {data_var.name} from accumulations to rates along {deaccum_dim}" + ) try: deaccumulate_to_rates_inplace( data_array, - dim="lead_time", + dim=deaccum_dim, reset_frequency=data_var.internal_attrs.window_reset_frequency, ) except ValueError: diff --git a/tests/conftest.py b/tests/conftest.py index fb423d2c..eaf5ea81 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import contextlib +import faulthandler import multiprocessing import os import sys @@ -7,6 +8,8 @@ import numpy as np import pytest +faulthandler.enable() + # Spawn new processes since fork isn't safe with threads with contextlib.suppress(RuntimeError): # skip if already set multiprocessing.set_start_method("spawn", force=True) @@ -19,7 +22,8 @@ # Config.env is set to test. os.environ["DYNAMICAL_ENV"] = "test" -from reformatters.common import storage + +from reformatters.common import storage # noqa: E402 def pytest_xdist_auto_num_workers(config: pytest.Config) -> int | None: diff --git a/tests/noaa/hrrr/analysis/__init__.py b/tests/noaa/hrrr/analysis/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/noaa/hrrr/analysis/dynamical_dataset_test.py b/tests/noaa/hrrr/analysis/dynamical_dataset_test.py new file mode 100644 index 00000000..d9c0332f --- /dev/null +++ b/tests/noaa/hrrr/analysis/dynamical_dataset_test.py @@ -0,0 +1,180 @@ +import numpy as np +import pandas as pd +import pytest +import xarray as xr +from numpy.testing import assert_allclose, assert_array_equal + +from reformatters.common import validation +from reformatters.common.storage import DatasetFormat, StorageConfig +from reformatters.noaa.hrrr.analysis.dynamical_dataset import ( + NoaaHrrrAnalysisDataset, +) +from tests.common.dynamical_dataset_test import NOOP_STORAGE_CONFIG +from tests.xarray_testing import assert_no_nulls + + +@pytest.fixture +def dataset() -> NoaaHrrrAnalysisDataset: + return make_dataset() + + +def make_dataset() -> NoaaHrrrAnalysisDataset: + return NoaaHrrrAnalysisDataset( + primary_storage_config=NOOP_STORAGE_CONFIG, + replica_storage_configs=[ + StorageConfig( + base_path="s3://replica-bucket/path", format=DatasetFormat.ICECHUNK + ) + ], + ) + + +@pytest.mark.slow +def test_backfill_local_and_operational_update(monkeypatch: pytest.MonkeyPatch) -> None: + dataset = make_dataset() + + filter_variable_names = [ + "temperature_2m", + "precipitation_surface", + ] + + dataset.backfill_local( + append_dim_end=pd.Timestamp("2018-09-16T02:00"), + filter_variable_names=filter_variable_names, + ) + + backfill_ds = xr.open_zarr( + dataset.store_factory.primary_store(), chunks=None, decode_timedelta=True + ) + time_start = pd.Timestamp("2018-09-16T00:00") + assert_array_equal( + backfill_ds["time"], + pd.date_range(time_start, "2018-09-16T01:00", freq="1h"), + ) + space_subset_ds = backfill_ds.isel(x=slice(-10, -1), y=slice(0, 10)) + + assert_no_nulls( + space_subset_ds[ + [v for v in filter_variable_names if v != "precipitation_surface"] + ] + ) + assert_no_nulls(space_subset_ds["precipitation_surface"].isel(time=slice(1, None))) + + point_ds = backfill_ds.isel(x=1, y=-2) + + assert_array_equal(point_ds["temperature_2m"].values, [24.5, 24.5]) + assert_allclose(point_ds["precipitation_surface"].values, [np.nan, 0.0]) + + dataset = make_dataset() + append_dim_end = pd.Timestamp("2018-09-16T03:00") + monkeypatch.setattr( + dataset.region_job_class, + "_update_append_dim_end", + lambda: append_dim_end, + ) + orig_get_jobs = dataset.region_job_class.get_jobs + monkeypatch.setattr( + dataset.region_job_class, + "get_jobs", + lambda *args, **kwargs: orig_get_jobs( + *args, **{**kwargs, "filter_variable_names": filter_variable_names} + ), + ) + + dataset.update("test-update") + + updated_ds = xr.open_zarr( + dataset.store_factory.primary_store(), chunks=None, decode_timedelta=True + ) + + expected_times = pd.date_range( + time_start, + append_dim_end, + freq=dataset.template_config.append_dim_frequency, + inclusive="left", + ) + assert_array_equal( + expected_times, + pd.DatetimeIndex(["2018-09-16T00:00", "2018-09-16T01:00", "2018-09-16T02:00"]), + ) + assert_array_equal(updated_ds["time"], expected_times) + + space_subset_ds = updated_ds.isel(x=slice(-10, 0), y=slice(0, 10)) + assert_no_nulls(space_subset_ds) + + point_ds = updated_ds.sel(x=400_000, y=760_000, method="nearest") + assert_array_equal(point_ds["temperature_2m"].values, [28.0, 25.75, 24.25]) + assert_array_equal(point_ds["precipitation_surface"].values, [np.nan, 0.0, 0.0]) + + +def test_operational_kubernetes_resources( + dataset: NoaaHrrrAnalysisDataset, +) -> None: + cron_jobs = list(dataset.operational_kubernetes_resources("test-image-tag")) + + assert len(cron_jobs) == 2 + update_cron_job, validation_cron_job = cron_jobs + + assert update_cron_job.name == f"{dataset.dataset_id}-update" + assert len(update_cron_job.secret_names) > 0 + + assert validation_cron_job.name == f"{dataset.dataset_id}-validate" + assert len(validation_cron_job.secret_names) > 0 + + +def test_validators(dataset: NoaaHrrrAnalysisDataset) -> None: + validators = tuple(dataset.validators()) + assert len(validators) == 2 + assert all(isinstance(v, validation.DataValidator) for v in validators) + + +@pytest.mark.slow +def test_precipitation_not_null_at_shard_boundary() -> None: + """ + Test that precipitation_surface is not NaN at the start of the 2nd shard. + + When deaccumulating, the first timestep of each shard processing could incorrectly + get NaN values because deaccumulation needs a previous value to compute the difference. + However, at shard boundaries (not the dataset start), we should have valid data. + """ + dataset = make_dataset() + config = dataset.template_config + + # Compute shard_2_start from dataset metadata + precip_var = next(v for v in config.data_vars if v.name == "precipitation_surface") + time_dim_index = config.dims.index("time") + assert isinstance(precip_var.encoding.shards, tuple) + time_shard_size = precip_var.encoding.shards[time_dim_index] + + shard_2_start = ( + config.append_dim_start + time_shard_size * config.append_dim_frequency + ) + + # Verify our computed value matches expected (90 days * 24 hours = 2160 hours after start) + assert time_shard_size == 2160 + assert shard_2_start == pd.Timestamp("2018-12-15T00:00") + + dataset.backfill_local( + # Get first 3 timesteps of 2nd shard (00:00, 01:00, 02:00) + append_dim_end=pd.Timestamp("2018-12-15T03:00"), + filter_start=shard_2_start, + filter_variable_names=["precipitation_surface"], + ) + + ds = xr.open_zarr( + dataset.store_factory.primary_store(), chunks=None, decode_timedelta=True + ) + + # Get only the times at the start of shard 2 + shard_2_ds = ds.sel(time=slice(shard_2_start, None)) + + expected_times = pd.DatetimeIndex( + ["2018-12-15T00:00", "2018-12-15T01:00", "2018-12-15T02:00"] + ) + assert_array_equal(shard_2_ds["time"].values, expected_times) + + # All 3 timesteps at start of shard 2 should have valid (non-NaN) precipitation. + # The first timestep of the entire dataset (2018-09-16T00:00) is expected to be NaN, + # but shard boundaries should NOT have NaN values. + precip = shard_2_ds["precipitation_surface"].isel(x=100, y=100) + assert_no_nulls(precip) diff --git a/tests/noaa/hrrr/analysis/region_job_test.py b/tests/noaa/hrrr/analysis/region_job_test.py new file mode 100644 index 00000000..57536aba --- /dev/null +++ b/tests/noaa/hrrr/analysis/region_job_test.py @@ -0,0 +1,268 @@ +from pathlib import Path +from unittest.mock import Mock + +import pandas as pd +import pytest + +from reformatters.common import template_utils +from reformatters.common.storage import DatasetFormat, StorageConfig, StoreFactory +from reformatters.noaa.hrrr.analysis.region_job import ( + NoaaHrrrAnalysisRegionJob, + NoaaHrrrAnalysisSourceFileCoord, +) +from reformatters.noaa.hrrr.analysis.template_config import ( + NoaaHrrrAnalysisTemplateConfig, +) + + +@pytest.fixture +def template_config() -> NoaaHrrrAnalysisTemplateConfig: + return NoaaHrrrAnalysisTemplateConfig() + + +def test_source_file_coord_out_loc( + template_config: NoaaHrrrAnalysisTemplateConfig, +) -> None: + """Test output location mapping.""" + coord = NoaaHrrrAnalysisSourceFileCoord( + init_time=pd.Timestamp("2024-02-29T00:00"), + lead_time=pd.Timedelta(hours=0), + domain="conus", + file_type="sfc", + data_vars=template_config.data_vars, + ) + + out_loc = coord.out_loc() + assert out_loc == { + "time": pd.Timestamp("2024-02-29T00:00"), + } + + +def test_source_file_coord_out_loc_with_lead_time( + template_config: NoaaHrrrAnalysisTemplateConfig, +) -> None: + """Test output location mapping with lead time.""" + coord = NoaaHrrrAnalysisSourceFileCoord( + init_time=pd.Timestamp("2024-02-29T00:00"), + lead_time=pd.Timedelta(hours=1), + domain="conus", + file_type="sfc", + data_vars=template_config.data_vars, + ) + + out_loc = coord.out_loc() + assert out_loc == { + "time": pd.Timestamp("2024-02-29T01:00"), + } + + +def test_region_job_generate_source_file_coords( + template_config: NoaaHrrrAnalysisTemplateConfig, +) -> None: + """Test source file coordinate generation.""" + template_ds = template_config.get_template(pd.Timestamp("2018-09-16T03:00")) + + test_ds = template_ds.isel(time=slice(0, 3)) + + region_job = NoaaHrrrAnalysisRegionJob.model_construct( + tmp_store=Mock(), + template_ds=test_ds, + data_vars=template_config.data_vars[:1], + append_dim=template_config.append_dim, + region=slice(0, 3), + reformat_job_name="test", + ) + + processing_region_ds, output_region_ds = region_job._get_region_datasets() + # Processing region is buffered by 1 for deaccumulation, but not at the very start of the dataset + assert len(processing_region_ds.time) == len(output_region_ds.time) + + source_coords = region_job.generate_source_file_coords( + processing_region_ds, template_config.data_vars[:1] + ) + + assert len(source_coords) == 3 + + for coord in source_coords: + assert isinstance(coord, NoaaHrrrAnalysisSourceFileCoord) + assert coord.domain == "conus" + assert ( + coord.file_type + == template_config.data_vars[0].internal_attrs.hrrr_file_type + ) + + +@pytest.mark.parametrize( + ("region", "expected_processing_region"), + [ + (slice(0, 100), slice(0, 100)), # At start: no buffer possible, clips to 0 + (slice(1, 100), slice(0, 100)), # At index 1: buffer clips to 0 + (slice(2, 100), slice(1, 100)), # At index 2+: full buffer of 1 + (slice(10, 100), slice(9, 100)), # Mid-dataset: full buffer of 1 + ], +) +def test_get_processing_region( + template_config: NoaaHrrrAnalysisTemplateConfig, + region: slice, + expected_processing_region: slice, +) -> None: + """Test that get_processing_region buffers by 1, clamped to 0 at dataset start.""" + region_job = NoaaHrrrAnalysisRegionJob.model_construct( + tmp_store=Mock(), + template_ds=Mock(), + data_vars=template_config.data_vars[:1], + append_dim=template_config.append_dim, + region=region, + reformat_job_name="test", + ) + + assert region_job.get_processing_region() == expected_processing_region + + +def test_region_job_processing_region_buffered( + template_config: NoaaHrrrAnalysisTemplateConfig, +) -> None: + """Test that processing region is buffered by 1 step for deaccumulation (not at dataset start).""" + template_ds = template_config.get_template(pd.Timestamp("2018-09-16T05:00")) + + test_ds = template_ds.isel(time=slice(0, 5)) + + # Region starting at index 2 (not the start of the dataset) + region_job = NoaaHrrrAnalysisRegionJob.model_construct( + tmp_store=Mock(), + template_ds=test_ds, + data_vars=template_config.data_vars[:1], + append_dim=template_config.append_dim, + region=slice(2, 5), + reformat_job_name="test", + ) + + processing_region_ds, output_region_ds = region_job._get_region_datasets() + # Processing region is buffered by 1 for deaccumulation + assert len(processing_region_ds.time) == len(output_region_ds.time) + 1 + assert processing_region_ds.time[0] == output_region_ds.time[0] - pd.Timedelta("1h") + + +def test_region_job_generate_source_file_coords_hour_0( + template_config: NoaaHrrrAnalysisTemplateConfig, +) -> None: + """Test that hour 0 variables use lead_time=0.""" + template_ds = template_config.get_template(pd.Timestamp("2018-09-16T04:00")) + + test_ds = template_ds.isel(time=slice(0, 4)) + + instant_vars = [ + v for v in template_config.data_vars if v.attrs.step_type == "instant" + ][:1] + assert len(instant_vars) == 1 + + # Use region starting at index 2 to test buffering behavior + region_job = NoaaHrrrAnalysisRegionJob.model_construct( + tmp_store=Mock(), + template_ds=test_ds, + data_vars=instant_vars, + append_dim=template_config.append_dim, + region=slice(2, 4), + reformat_job_name="test", + ) + + processing_region_ds, _output_region_ds = region_job._get_region_datasets() + + source_coords = region_job.generate_source_file_coords( + processing_region_ds, instant_vars + ) + + # 3 source coords: 1 buffer + 2 output + assert len(source_coords) == 3 + + expected_init_times = pd.date_range( + "2018-09-16T01:00", "2018-09-16T03:00", freq="1h" + ) + for coord, expected_init_time in zip( + source_coords, expected_init_times, strict=True + ): + assert isinstance(coord, NoaaHrrrAnalysisSourceFileCoord) + assert coord.init_time == expected_init_time + assert coord.lead_time == pd.Timedelta("0h") + + +def test_region_job_generate_source_file_coords_hour_1( + template_config: NoaaHrrrAnalysisTemplateConfig, +) -> None: + """Test that non-hour 0 variables use lead_time=1.""" + template_ds = template_config.get_template(pd.Timestamp("2018-09-16T04:00")) + + test_ds = template_ds.isel(time=slice(0, 4)) + + avg_vars = [v for v in template_config.data_vars if v.attrs.step_type == "avg"][:1] + assert len(avg_vars) == 1 + + # Use region starting at index 2 to test buffering behavior + region_job = NoaaHrrrAnalysisRegionJob.model_construct( + tmp_store=Mock(), + template_ds=test_ds, + data_vars=avg_vars, + append_dim=template_config.append_dim, + region=slice(2, 4), + reformat_job_name="test", + ) + + processing_region_ds, _output_region_ds = region_job._get_region_datasets() + + source_coords = region_job.generate_source_file_coords( + processing_region_ds, avg_vars + ) + + # 3 source coords: 1 buffer + 2 output + assert len(source_coords) == 3 + + expected_init_times = pd.date_range( + "2018-09-16T00:00", "2018-09-16T02:00", freq="1h" + ) + + for coord, expected_init_time in zip( + source_coords, expected_init_times, strict=True + ): + assert isinstance(coord, NoaaHrrrAnalysisSourceFileCoord) + assert coord.init_time == expected_init_time + assert coord.lead_time == pd.Timedelta("1h") + expected_time = coord.init_time + coord.lead_time + assert coord.out_loc()["time"] == expected_time + + +def test_operational_update_jobs( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + template_config = NoaaHrrrAnalysisTemplateConfig() + store_factory = StoreFactory( + primary_storage_config=StorageConfig( + base_path="fake-prod-path", + format=DatasetFormat.ZARR3, + ), + dataset_id="test-dataset-A", + template_config_version="test-version", + ) + + monkeypatch.setattr( + pd.Timestamp, + "now", + classmethod(lambda *args, **kwargs: pd.Timestamp("2018-09-16T06:34")), + ) + existing_ds = template_config.get_template(pd.Timestamp("2018-09-16T05:01")) + template_utils.write_metadata(existing_ds, store_factory) + + jobs, template_ds = NoaaHrrrAnalysisRegionJob.operational_update_jobs( + primary_store=store_factory.primary_store(), + tmp_store=tmp_path / "tmp_ds.zarr", + get_template_fn=template_config.get_template, + append_dim=template_config.append_dim, + all_data_vars=template_config.data_vars, + reformat_job_name="test_job", + ) + + assert template_ds.time.max() == pd.Timestamp("2018-09-16T06:00") + + assert len(jobs) == 1 + for job in jobs: + assert isinstance(job, NoaaHrrrAnalysisRegionJob) + assert job.data_vars == template_config.data_vars diff --git a/tests/noaa/hrrr/analysis/template_config_test.py b/tests/noaa/hrrr/analysis/template_config_test.py new file mode 100644 index 00000000..cc869d0a --- /dev/null +++ b/tests/noaa/hrrr/analysis/template_config_test.py @@ -0,0 +1,181 @@ +import numpy as np +import pandas as pd +import xarray as xr + +from reformatters.common.download import http_download_to_disk +from reformatters.noaa.hrrr.analysis.template_config import ( + NoaaHrrrAnalysisTemplateConfig, +) +from reformatters.noaa.hrrr.region_job import NoaaHrrrSourceFileCoord +from reformatters.noaa.noaa_grib_index import grib_message_byte_ranges_from_index + + +def test_spatial_coordinates() -> None: + """Ensure the template has the expected coordinate system.""" + template_config = NoaaHrrrAnalysisTemplateConfig() + ds = template_config.get_template( + template_config.append_dim_start + pd.Timedelta(days=10) + ) + + assert "latitude" in ds.coords + assert "longitude" in ds.coords + assert "x" in ds.coords + assert "y" in ds.coords + + assert ds.latitude.dims == ("y", "x") + assert ds.longitude.dims == ("y", "x") + assert ds.x.dims == ("x",) + assert ds.y.dims == ("y",) + + assert len(ds.x) == 1799 + assert (ds.x.diff(dim="x") == 3000.0).all() + assert np.isclose(ds.x.min() - (3000 / 2), -2699020.143) + assert np.isclose(ds.x.max() + (3000 / 2), 2697979.857) + assert len(ds.y) == 1059 + assert (ds.y.diff(dim="y") == -3000.0).all() + assert np.isclose(ds.y.min() - (3000 / 2), -1588806.153) + assert np.isclose(ds.y.max() + (3000 / 2), 1588193.847) + + assert ds.latitude.min() == 21.138123 + assert ds.latitude.mean() == 37.152527 + assert ds.latitude.max() == 52.615654 + assert np.isclose(ds.latitude.diff(dim="y").min(), -0.02698135) + assert np.isclose(ds.latitude.diff(dim="y").max(), -0.0245285) + + assert ds.longitude.min() == -134.09547 + assert ds.longitude.mean() == -97.50583 + assert ds.longitude.max() == -60.917194 + assert np.isclose(ds.longitude.diff(dim="x").min(), 0.02666473) + assert np.isclose(ds.longitude.diff(dim="x").max(), 0.04299164) + + +def test_template_config_attrs() -> None: + """Test basic template configuration attributes.""" + config = NoaaHrrrAnalysisTemplateConfig() + + assert config.dims == ("time", "y", "x") + assert config.append_dim == "time" + + assert config.append_dim_start == pd.Timestamp("2018-09-16T00:00") + assert config.append_dim_frequency == pd.Timedelta("1h") + + data_vars = config.data_vars + assert len(data_vars) > 0 + + refc_vars = [v for v in data_vars if v.internal_attrs.grib_element == "REFC"] + assert len(refc_vars) == 1 + assert refc_vars[0].name == "composite_reflectivity" + assert refc_vars[0].internal_attrs.hrrr_file_type == "sfc" + + +def test_dimension_coordinates() -> None: + """Test dimension coordinates are properly configured.""" + config = NoaaHrrrAnalysisTemplateConfig() + dim_coords = config.dimension_coordinates() + + assert "time" in dim_coords + assert "x" in dim_coords + assert "y" in dim_coords + + assert ( + dim_coords["time"] + == pd.date_range("2018-09-16T00:00", "2018-09-16T00:00", freq="1h") + ).all() + + assert len(dim_coords["x"]) == 1799 + assert len(dim_coords["y"]) == 1059 + + +def test_template_variables_have_required_attrs() -> None: + """Test that all data variables have required attributes.""" + config = NoaaHrrrAnalysisTemplateConfig() + + for var in config.data_vars: + assert var.name + assert var.encoding + + assert var.internal_attrs.grib_element + assert var.internal_attrs.grib_index_level + assert var.internal_attrs.hrrr_file_type in ["sfc", "prs", "nat", "subh"] + + assert var.attrs.short_name + assert var.attrs.long_name + assert var.attrs.units + assert var.attrs.step_type in ["instant", "avg", "accum", "max", "min"] + + +def test_coordinate_configs() -> None: + """Test coordinate configurations.""" + config = NoaaHrrrAnalysisTemplateConfig() + coords = config.coords + + coord_names = [coord.name for coord in coords] + + required_coords = { + "time", + "x", + "y", + "latitude", + "longitude", + "spatial_ref", + } + + assert set(coord_names) == required_coords, ( + f"Coordinate mismatch. Expected: {required_coords}, Got: {set(coord_names)}" + ) + + +def test_derive_coordinates_integration() -> None: + """Integration test for derive_coordinates method.""" + config = NoaaHrrrAnalysisTemplateConfig() + template_ds = config.get_template(pd.Timestamp("2018-09-16T12:00")) + + assert ( + template_ds.coords["time"] + == pd.date_range("2018-09-16T00:00", "2018-09-16T11:00", freq="1h") + ).all() + + +def test_spatial_info_matches_file() -> None: + """Test that hard coded spatial information matches the real values derived from a source file.""" + config = NoaaHrrrAnalysisTemplateConfig() + shape, bounds, resolution, crs = config._spatial_info() + + coord = NoaaHrrrSourceFileCoord( + init_time=pd.Timestamp("2023-10-01T00:00"), + lead_time=pd.Timedelta("0h"), + domain="conus", + file_type="sfc", + data_vars=[config.data_vars[0]], + ) + idx_local_path = http_download_to_disk(coord.get_idx_url(), config.dataset_id) + byte_range_starts, byte_range_ends = grib_message_byte_ranges_from_index( + idx_local_path, coord.data_vars, coord.init_time, coord.lead_time + ) + local_path = http_download_to_disk( + coord.get_url(), + config.dataset_id, + byte_ranges=(byte_range_starts, byte_range_ends), + local_path_suffix="spatial-info-test", + ) + + ds = xr.open_dataset(local_path, engine="rasterio") + + assert shape == ds.rio.shape + assert np.allclose(bounds, ds.rio.bounds()) + assert resolution == ds.rio.resolution() + assert crs == ds.rio.crs.to_proj4() + + template_ds = config.get_template(pd.Timestamp("2025-01-01")) + assert ds.spatial_ref.attrs["standard_parallel"] == (38.5, 38.5) + ds.spatial_ref.attrs["standard_parallel"] = list( + ds.spatial_ref.attrs["standard_parallel"] + ) + if ( + ds.spatial_ref.attrs["GeoTransform"] + == "-2699020.142521929 3000.0 0.0 1588193.8474433345 0.0 -3000.0" + ): + ds.spatial_ref.attrs["GeoTransform"] = ( + "-2699020.142521929 3000.0 0.0 1588193.847443335 0.0 -3000.0" + ) + assert ds.spatial_ref.attrs == template_ds.spatial_ref.attrs diff --git a/tests/noaa/hrrr/region_job_test.py b/tests/noaa/hrrr/region_job_test.py index e55ef5df..c621b89c 100644 --- a/tests/noaa/hrrr/region_job_test.py +++ b/tests/noaa/hrrr/region_job_test.py @@ -4,7 +4,9 @@ import numpy as np import pandas as pd import pytest +import xarray as xr +from reformatters.common.pydantic import replace from reformatters.noaa.hrrr.forecast_48_hour.template_config import ( NoaaHrrrForecast48HourTemplateConfig, ) @@ -229,3 +231,190 @@ def test_region_job_read_data( assert result.dtype == np.float32 rasterio_reader.read.assert_called_once_with(1, out_dtype=np.float32) + + +def test_region_job_read_data_no_matching_bands( + template_config: NoaaHrrrCommonTemplateConfig, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test that read_data raises an error when no matching bands are found.""" + coord = NoaaHrrrSourceFileCoord( + init_time=pd.Timestamp("2024-02-29T00:00"), + lead_time=pd.Timedelta(hours=0), + domain="conus", + file_type="sfc", + data_vars=template_config.data_vars[:1], + downloaded_path=Path("fake/path/to/downloaded/file.grib2"), + ) + + region_job = NoaaHrrrRegionJob.model_construct( + tmp_store=Mock(), + template_ds=Mock(), + data_vars=template_config.data_vars[:1], + append_dim=template_config.append_dim, + region=slice(0, 1), + reformat_job_name="test", + ) + + rasterio_reader = Mock() + rasterio_reader.__enter__ = Mock(return_value=rasterio_reader) + rasterio_reader.__exit__ = Mock(return_value=False) + rasterio_reader.count = 1 + rasterio_reader.descriptions = ["Wrong description"] + rasterio_reader.tags = Mock(return_value={"GRIB_ELEMENT": "WRONG"}) + monkeypatch.setattr( + "reformatters.noaa.hrrr.region_job.rasterio.open", + Mock(return_value=rasterio_reader), + ) + + with pytest.raises( + AssertionError, match="Expected exactly 1 matching band, found 0" + ): + region_job.read_data(coord, template_config.data_vars[0]) + + +def test_region_job_read_data_multiple_matching_bands( + template_config: NoaaHrrrCommonTemplateConfig, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test that read_data raises an error when multiple matching bands are found.""" + coord = NoaaHrrrSourceFileCoord( + init_time=pd.Timestamp("2024-02-29T00:00"), + lead_time=pd.Timedelta(hours=0), + domain="conus", + file_type="sfc", + data_vars=template_config.data_vars[:1], + downloaded_path=Path("fake/path/to/downloaded/file.grib2"), + ) + + region_job = NoaaHrrrRegionJob.model_construct( + tmp_store=Mock(), + template_ds=Mock(), + data_vars=template_config.data_vars[:1], + append_dim=template_config.append_dim, + region=slice(0, 1), + reformat_job_name="test", + ) + + rasterio_reader = Mock() + rasterio_reader.__enter__ = Mock(return_value=rasterio_reader) + rasterio_reader.__exit__ = Mock(return_value=False) + rasterio_reader.count = 2 + rasterio_reader.descriptions = [ + '0[-] EATM="Entire Atmosphere"', + '0[-] EATM="Entire Atmosphere"', + ] + rasterio_reader.tags = Mock(return_value={"GRIB_ELEMENT": "REFC"}) + monkeypatch.setattr( + "reformatters.noaa.hrrr.region_job.rasterio.open", + Mock(return_value=rasterio_reader), + ) + + with pytest.raises( + AssertionError, match="Expected exactly 1 matching band, found 2" + ): + region_job.read_data(coord, template_config.data_vars[0]) + + +def test_apply_data_transformations_binary_rounding( + template_config: NoaaHrrrCommonTemplateConfig, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test that binary rounding is called when keep_mantissa_bits is set.""" + region_job = NoaaHrrrRegionJob.model_construct( + tmp_store=Mock(), + template_ds=Mock(), + data_vars=template_config.data_vars[:1], + append_dim=template_config.append_dim, + region=slice(0, 1), + reformat_job_name="test", + ) + + # Create a data var with binary rounding enabled + data_var = replace( + template_config.data_vars[0], + internal_attrs=replace( + template_config.data_vars[0].internal_attrs, + keep_mantissa_bits=10, + deaccumulate_to_rate=False, + ), + ) + + test_data = np.array([1.23456789, 2.34567890, 3.45678901], dtype=np.float32) + data_array = xr.DataArray(test_data.copy(), dims=["x"]) + + mock_round = Mock() + monkeypatch.setattr( + "reformatters.noaa.hrrr.region_job.round_float32_inplace", + mock_round, + ) + + region_job.apply_data_transformations(data_array, data_var) + + # Verify rounding was called with correct arguments + mock_round.assert_called_once_with(data_array.values, 10) + + +def test_apply_data_transformations_deaccumulation( + template_config: NoaaHrrrCommonTemplateConfig, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Test that deaccumulation is called when deaccumulate_to_rate is True.""" + region_job = NoaaHrrrRegionJob.model_construct( + tmp_store=Mock(), + template_ds=Mock(), + data_vars=template_config.data_vars[:1], + append_dim=template_config.append_dim, + region=slice(0, 1), + reformat_job_name="test", + ) + + # Create a data var with deaccumulation enabled + data_var = replace( + template_config.data_vars[0], + internal_attrs=replace( + template_config.data_vars[0].internal_attrs, + deaccumulate_to_rate=True, + window_reset_frequency=pd.Timedelta(hours=1), + keep_mantissa_bits="no-rounding", + ), + ) + + times = pd.date_range("2024-01-01", periods=5, freq="1h") + test_data = np.array([0.0, 3.6, 7.2, 10.8, 14.4], dtype=np.float32) + data_array = xr.DataArray( + test_data, dims=["time"], coords={"time": times}, attrs={"units": "mm/s"} + ) + + mock_deaccumulate = Mock() + monkeypatch.setattr( + "reformatters.noaa.hrrr.region_job.deaccumulate_to_rates_inplace", + mock_deaccumulate, + ) + + region_job.apply_data_transformations(data_array, data_var) + + # Verify deaccumulation was called with correct arguments + mock_deaccumulate.assert_called_once() + call_args = mock_deaccumulate.call_args + assert call_args.kwargs["dim"] == "time" + assert call_args.kwargs["reset_frequency"] == pd.Timedelta(hours=1) + + +def test_update_append_dim_end() -> None: + """Test that _update_append_dim_end returns current time.""" + before = pd.Timestamp.now() + result = NoaaHrrrRegionJob._update_append_dim_end() + after = pd.Timestamp.now() + + assert before <= result <= after + + +def test_update_append_dim_start() -> None: + """Test that _update_append_dim_start returns max time from existing data.""" + times = pd.date_range("2024-01-01", periods=10, freq="1h") + time_coord = xr.DataArray(times, dims=["time"]) + + result = NoaaHrrrRegionJob._update_append_dim_start(time_coord) + + assert result == pd.Timestamp("2024-01-01 09:00:00") diff --git a/tests/xarray_testing.py b/tests/xarray_testing.py index ea5432cf..11a776a1 100644 --- a/tests/xarray_testing.py +++ b/tests/xarray_testing.py @@ -1,5 +1,8 @@ import xarray as xr -def assert_no_nulls(ds: xr.Dataset) -> None: - assert (ds.isnull().sum() == 0).all().to_array().all() +def assert_no_nulls(ds: xr.Dataset | xr.DataArray) -> None: + no_nulls = (ds.isnull().sum() == 0).all() + if isinstance(ds, xr.Dataset): + no_nulls = no_nulls.to_array().all() + assert no_nulls