From faa6d57d854fded3963b1444abe0d267993b6cf2 Mon Sep 17 00:00:00 2001 From: Al Handwerger Date: Fri, 30 Jan 2026 16:37:09 -0800 Subject: [PATCH 1/7] Add files via upload adding landslide notebook for CSLC with various notebook options --- CSLC/Landslides/CSLC-S1_for_landslides.ipynb | 2057 +++++++++++++++++ .../environment_opera_cslc_landslides.yml | 29 + 2 files changed, 2086 insertions(+) create mode 100644 CSLC/Landslides/CSLC-S1_for_landslides.ipynb create mode 100644 CSLC/Landslides/environment_opera_cslc_landslides.yml diff --git a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb new file mode 100644 index 0000000..c582c3c --- /dev/null +++ b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb @@ -0,0 +1,2057 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "36315bdf", + "metadata": {}, + "source": [ + "# Generate wrapped interferograms and coherence maps using OPERA CSLC-S1\n", + "\n", + "--- \n", + "\n", + "This notebook:\n", + "- Searches OPERA CSLC-S1 products for your AOI + date range\n", + "- Subsets CSLCs **before download** using opera-utils\n", + "- Builds interferograms/coherence and merges bursts\n", + "- Exports mosaics and visualizations\n", + "\n", + "**Quick start**\n", + "1) Set parameters in the next cell (AOI, date range, pairing)\n", + "2) Run cells top-to-bottom\n", + "3) Outputs land in `savedir/`\n", + "\n", + "**Key toggles**\n", + "- `REPROJECT_FOR_DISPLAY`: reproject plots to WGS84 when True\n", + "- `SAVE_WGS84`: save WGS84 GeoTIFF mosaics when True\n", + "- `DOWNLOAD_WITH_PROGRESS`: show progress bar for downloads\n", + "- `USE_WATER_YEAR`: Oct–Sep calendar layout when True\n", + "- `pair_mode` / `t_span`: control IFG pairing (all vs fixed separation)\n", + "\n", + "**Outputs**\n", + "- Subset CSLC H5: `savedir/subset_cslc/*.h5`\n", + "- Mosaics (IFG/COH, native CRS): `savedir/tifs/merged_ifg_*`, `merged_coh_*`\n", + "- WGS84 mosaics: `savedir/tifs/WGS84/merged_ifg_WGS84_*`, `merged_coh_WGS84_*`, `merged_amp_WGS84_*`\n", + "- Amplitude mosaics (native CRS): `savedir/tifs/merged_amp_*.tif`\n", + "- GIFs: `savedir/gifs/*.gif`\n" + ] + }, + { + "cell_type": "markdown", + "id": "2706d0ca-c490-4a68-bbd7-c3ff1b2adfb9", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Data Used in the Example: \n", + "\n", + "- **10 meter (Northing) x 5 meter (Easting) North America OPERA Coregistered Single Look Complex from Sentinel-1 products**\n", + " - This dataset contains Level-2 OPERA coregistered single-look-complex (CSLC) data from Sentinel-1 (S1). The data in this example are geocoded CSLC-S1 data covering Palos Verdes landslides, California, USA. \n", + " \n", + " - The OPERA project is generating geocoded burst-wise CSLC-S1 products over North America which includes USA and US Territories within 200 km from the US border, Canada, and all mainland countries from the southern US border down to and including Panama. Each pixel within a burst SLC is represented by a complex number and contains both the amplitude and phase information. The CSLC-S1 products are distributed over projected map coordinates using the Universal Transverse Mercator (UTM) projection with spacing in the X- and Y-directions of 5 m and 10 m, respectively. Each OPERA CSLC-S1 product is distributed as a HDF5 file following the CF-1.8 convention with separate groups containing the data raster layers, the low-resolution correction layers, and relevant product metadata.\n", + "\n", + " - For more information about the OPERA project and other products please visit our website at https://www.jpl.nasa.gov/go/opera .\n", + "\n", + "Please refer to the [OPERA Product Specification Document](https://d2pn8kiwq2w21t.cloudfront.net/documents/OPERA_CSLC-S1_ProductSpec_v1.0.0_D-108278_Initial_2023-09-11_URS321269.pdf) for details about the CSLC-S1 product.\n", + "\n", + "*Prepared by Al Handwerger and M. Grace Bato*\n", + "\n", + "---" + ] + }, + { + "cell_type": "markdown", + "id": "9b2e135b", + "metadata": {}, + "source": [ + "## 0. Setup your conda environment\n", + "\n", + "Assuming you have conda installed. Open your terminal and run the following:\n", + "```\n", + "\n", + "# Create the OPERA CSLC environment\n", + "conda env create -f environment_opera_cslc_landslides.yml\n", + "conda activate opera_cslc_landslides\n", + "python -m ipykernel install --user --name opera_cslc_landslides\n", + "\n", + "```\n", + "\n", + "--- " + ] + }, + { + "cell_type": "markdown", + "id": "e0d4309e", + "metadata": {}, + "source": [ + "## 1. Load Python modules" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84a1468d-0aaf-4f06-9875-6b753d94ad42", + "metadata": {}, + "outputs": [], + "source": [ + "## Load necessary modules\n", + "%load_ext watermark\n", + "\n", + "import asf_search as asf\n", + "import geopandas as gpd\n", + "import pandas as pd\n", + "\n", + "import numpy as np\n", + "from netrc import netrc\n", + "from subprocess import Popen\n", + "from platform import system\n", + "from getpass import getpass\n", + "import folium\n", + "import datetime as dt\n", + "from shapely.geometry import box\n", + "from shapely.geometry import Point\n", + "import shapely.wkt as wkt\n", + "import rioxarray\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.patches as patches\n", + "import cartopy.crs as ccrs\n", + "import xarray as xr\n", + "import rasterio\n", + "from rasterio.transform import from_origin\n", + "from rasterio import merge\n", + "from rasterio.crs import CRS\n", + "\n", + "import os, sys\n", + "proj_dir = os.path.join(sys.prefix, \"share\", \"proj\")\n", + "os.environ[\"PROJ_LIB\"] = proj_dir # for older PROJ\n", + "os.environ[\"PROJ_DATA\"] = proj_dir # for newer PROJ\n", + "\n", + "\n", + "%watermark --iversions\n", + "\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9952139c", + "metadata": {}, + "outputs": [], + "source": [ + "# Environment check\n", + "import sys\n", + "import importlib\n", + "\n", + "REQUIRED_PKGS = [\n", + " 'asf_search','cartopy','folium','geopandas','h5py','imageio','matplotlib','numpy','pandas',\n", + " 'pyproj','rasterio','rioxarray','shapely','xarray','opera_utils','tqdm','rich'\n", + "]\n", + "missing = []\n", + "for pkg in REQUIRED_PKGS:\n", + " try:\n", + " importlib.import_module(pkg)\n", + " except Exception:\n", + " missing.append(pkg)\n", + "\n", + "if missing:\n", + " raise ImportError(\"Missing packages: \" + ', '.join(missing) + \". Activate opera_cslc env or install from environment_opera_cslc.yml\")\n", + "\n", + "print(f\"Python: {sys.executable}\")\n", + "print('Environment check OK')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1faa1ef0-3d5e-424e-b602-3392b720af6d", + "metadata": {}, + "outputs": [], + "source": [ + "## Load plotting module\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "%config InlineBackend.figure_format='retina'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "49952732", + "metadata": {}, + "outputs": [], + "source": [ + "## Load pandas and setup config to expand the display of the database\n", + "import pandas as pd\n", + "# pd.set_option('display.max_rows', None)\n", + "pd.set_option('display.max_columns', None)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "137538f3", + "metadata": {}, + "outputs": [], + "source": [ + "def _maybe_reproject(da):\n", + " if REPROJECT_FOR_DISPLAY:\n", + " return da.rio.reproject(\"EPSG:4326\")\n", + " return da\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c30a7294-742e-4ac6-bc9b-82cc493e1e6c", + "metadata": {}, + "outputs": [], + "source": [ + "## Avoid lots of these warnings printing to notebook from asf_search\n", + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "id": "32ef4ad4", + "metadata": {}, + "source": [ + "## 2. Set up your NASA Earthdata Login Credentials" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b6ad5ac-9089-4377-be63-ddf1731263f2", + "metadata": {}, + "outputs": [], + "source": [ + "urs = 'urs.earthdata.nasa.gov'\n", + "prompts = ['Enter NASA Earthdata Login Username: ',\n", + " 'Enter NASA Earthdata Login Password: ']\n", + "\n", + "netrc_name = \"_netrc\" if system() == \"Windows\" else \".netrc\"\n", + "netrc_path = os.path.expanduser(f\"~/{netrc_name}\")\n", + "\n", + "def write_netrc():\n", + " username = getpass(prompt=prompts[0])\n", + " password = getpass(prompt=prompts[1])\n", + " with open(netrc_path, 'a') as f:\n", + " f.write(f\"\\nmachine {urs}\\n\")\n", + " f.write(f\"login {username}\\n\")\n", + " f.write(f\"password {password}\\n\")\n", + " os.chmod(netrc_path, 0o600)\n", + "\n", + "def has_urs_credentials():\n", + " try:\n", + " creds = netrc(netrc_path).authenticators(urs)\n", + " return creds is not None\n", + " except (FileNotFoundError, NetrcParseError):\n", + " return False\n", + "\n", + "if not has_urs_credentials():\n", + " if not os.path.exists(netrc_path):\n", + " open(netrc_path, 'w').close()\n", + " write_netrc()\n", + "\n", + "import os\n", + "os.environ[\"GDAL_HTTP_NETRC\"] = \"YES\"\n", + "os.environ[\"GDAL_HTTP_NETRC_FILE\"] = netrc_path\n" + ] + }, + { + "cell_type": "markdown", + "id": "18311b89", + "metadata": {}, + "source": [ + "## 3. Enter user-defined parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5eeba6e", + "metadata": {}, + "outputs": [], + "source": [ + "# User parameters (edit these)\n", + "# AOI is a WKT polygon in EPSG:4326\n", + "## Enter user-defined parameters\n", + "SITE_NAME = \"Palos_Verdes_Landslides\" # used for output folder naming\n", + "aoi = \"POLYGON((-118.3955 33.7342,-118.3464 33.7342,-118.3464 33.7616,-118.3955 33.7616,-118.3955 33.7342))\"\n", + "orbitPass = \"DESCENDING\"\n", + "pathNumber = 71\n", + "# Optional burst selection before download\n", + "# Use subswath (e.g., 'IW2', 'IW3') or specific OPERA burst ID (e.g., 'T071_151230_IW3')\n", + "BURST_SUBSWATH = 'IW3' # e.g., 'IW2' or ['IW2', 'IW3'] or None \n", + "BURST_ID = None # e.g., 'T071_151230_IW3' or list of burst IDs\n", + "dateStart = dt.datetime.fromisoformat('2017-10-01 00:00:00') #'YYYY-MM-DD HH:MM:SS'\n", + "dateEnd = dt.datetime.fromisoformat('2017-12-01 23:59:59') #'YYYY-MM-DD HH:MM:SS'\n", + "\n", + "# Pairing options\n", + "pair_mode = 't_span' # 'all' or 't_span'\n", + "pair_t_span_days = 12 # int or list of ints (e.g., [12, 24])\n", + "\n", + "# Multilooking (spatial averaging)\n", + "# Set either MULTILOOK (looks_y, looks_x) OR TARGET_PIXEL_M (meters). TARGET overrides MULTILOOK.\n", + "MULTILOOK = (1, 1) # e.g., (3, 6) for 30m from (dy=10m, dx=5m)\n", + "TARGET_PIXEL_M = None # e.g., 30.0 or 90.0\n", + "\n", + "REPROJECT_FOR_DISPLAY = False # set True to reproject plots to EPSG:4326\n", + "SAVE_WGS84 = False # set True to save WGS84 GeoTIFF mosaics\n", + "\n", + "\n", + "\n", + "DOWNLOAD_WITH_PROGRESS = True # set True for per-file progress bar\n", + "\n", + "import os\n", + "\n", + "min_t_span_days = 12 # minimum separation (days)\n", + "max_t_span_days = 12 # maximum separation (days) or None\n", + "max_pairs_per_burst = None # int or None\n", + "max_pairs_total = None # int or None\n", + "\n", + "# Calendar settings\n", + "USE_WATER_YEAR = True # True: Oct–Sep, False: Jan–Dec\n", + "\n", + "DOWNLOAD_PROCESSES = min(8, max(2, (os.cpu_count() or 4) // 2))\n", + "# DOWNLOAD_BATCH_SIZE = 5\n", + "\n", + "\n", + "# Normalize name for filesystem (letters/numbers/_/- only)\n", + "import re\n", + "site_slug = re.sub(r\"[^A-Za-z0-9_-]+\", \"\", SITE_NAME)\n", + "orbit_code = orbitPass[0].upper() # 'A' or 'D'\n", + "savedir = f'./{site_slug}_{orbit_code}{pathNumber:03d}/'\n", + "\n", + "# Water mask options\n", + "# in params cell\n", + "WATER_MASK_PATH = f\"{savedir}/water_mask/water_mask_esa_wc2021.tif\"\n", + "APPLY_WATER_MASK = True\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f072529", + "metadata": {}, + "outputs": [], + "source": [ + "# ESA WorldCover 2021 water mask (GDAL-only)\n", + "from pathlib import Path\n", + "import geopandas as gpd\n", + "import shapely.wkt\n", + "import rasterio\n", + "from osgeo import gdal\n", + "\n", + "ESA_WC_GRID_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/esa_worldcover_grid.fgb\"\n", + "ESA_WC_BASE_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/v200/2021/map\"\n", + "# ESA WorldCover class codes: 80 = Permanent water bodies\n", + "ESA_WC_WATER_CLASSES = {80}\n", + "\n", + "\n", + "def build_worldcover_water_mask(aoi_wkt, out_path, target_res_deg=None):\n", + " # Create a binary land mask from ESA WorldCover (1=land, 0=water).\n", + " out_path = Path(out_path)\n", + " out_path.parent.mkdir(parents=True, exist_ok=True)\n", + " if out_path.exists():\n", + " return out_path\n", + "\n", + "\n", + " aoi_geom = shapely.wkt.loads(aoi_wkt)\n", + " # Load tile grid and select intersecting tiles\n", + " grid = gpd.read_file(ESA_WC_GRID_URL)\n", + " grid = grid.to_crs(\"EPSG:4326\")\n", + " # Find tile id column (varies by grid version)\n", + " tile_col = next((c for c in grid.columns if 'tile' in c.lower()), None)\n", + " if tile_col is None:\n", + " raise RuntimeError(f\"No tile column found in grid columns: {list(grid.columns)}\")\n", + " tiles = grid[grid.intersects(aoi_geom)][tile_col].tolist()\n", + " if not tiles:\n", + " raise RuntimeError(\"No WorldCover tiles intersect AOI\")\n", + "\n", + " print(f\"Selected tiles: {tiles}\")\n", + "\n", + " tile_urls = [\n", + " f\"{ESA_WC_BASE_URL}/ESA_WorldCover_10m_2021_v200_{t}_Map.tif\"\n", + " for t in tiles\n", + " ]\n", + "\n", + " # Quick URL check for first tile\n", + " first_url = tile_urls[0]\n", + " try:\n", + " _ = gdal.Open(first_url)\n", + " except Exception as e:\n", + " raise RuntimeError(f\"GDAL cannot open first tile URL: {first_url}\\n{e}\")\n", + "\n", + " vrt_path = out_path.with_suffix(\".vrt\")\n", + " gdal.BuildVRT(str(vrt_path), tile_urls)\n", + "\n", + " minx, miny, maxx, maxy = aoi_geom.bounds\n", + " warp_kwargs = dict(\n", + " format=\"GTiff\",\n", + " outputBounds=[minx, miny, maxx, maxy],\n", + " multithread=True,\n", + " )\n", + " if target_res_deg is not None:\n", + " warp_kwargs.update(dict(xRes=target_res_deg, yRes=target_res_deg, targetAlignedPixels=True))\n", + "\n", + " tmp_map = out_path.with_name(out_path.stem + \"_map.tif\")\n", + " warp_ds = gdal.Warp(str(tmp_map), str(vrt_path), **warp_kwargs)\n", + " if warp_ds is None:\n", + " raise RuntimeError(\"GDAL Warp returned None. Check network access/URL.\")\n", + " warp_ds = None\n", + "\n", + " with rasterio.open(tmp_map) as src:\n", + " data = src.read(1)\n", + " profile = src.profile\n", + "\n", + " mask = (~np.isin(data, list(ESA_WC_WATER_CLASSES))).astype(\"uint8\")\n", + " profile.update(dtype=\"uint8\", count=1, nodata=0)\n", + "\n", + " with rasterio.open(out_path, \"w\", **profile) as dst:\n", + " dst.write(mask, 1)\n", + "\n", + " return out_path\n", + "\n", + "# Example usage:\n", + "# WATER_MASK_PATH = build_worldcover_water_mask(aoi, f\"{savedir}/water_mask/water_mask_esa_wc2021.tif\")\n", + "# APPLY_WATER_MASK = True\n" + ] + }, + { + "cell_type": "markdown", + "id": "98916bef", + "metadata": {}, + "source": [ + "## 4. Query OPERA CSLCs using `asf_search`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e33d72e-2e75-4099-93d6-5cd058643e35", + "metadata": {}, + "outputs": [], + "source": [ + "## Search for OPERA CSLC data in ASF DAAC\n", + "try:\n", + " search_params = dict(\n", + " intersectsWith= aoi,\n", + " dataset='OPERA-S1',\n", + " processingLevel='CSLC',\n", + " flightDirection = orbitPass,\n", + " start=dateStart,\n", + " end=dateEnd)\n", + "\n", + " ## Return results\n", + " results = asf.search(**search_params)\n", + " print(f\"Length of Results: {len(results)}\")\n", + "\n", + "except TypeError:\n", + " search_params = dict(\n", + " intersectsWith= aoi.wkt,\n", + " dataset='OPERA-S1',\n", + " processingLevel='CSLC',\n", + " flightDirection = orbitPass,\n", + " start=dateStart,\n", + " end=dateEnd)\n", + "\n", + " ## Return results\n", + " results = asf.search(**search_params)\n", + " print(f\"Length of Results: {len(results)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2f5b5df", + "metadata": {}, + "outputs": [], + "source": [ + "## Save the results in a geopandas dataframe\n", + "gf = gpd.GeoDataFrame.from_features(results.geojson(), crs='EPSG:4326')\n", + "\n", + "## Filter data based on specified track number\n", + "gf = gf[gf.pathNumber==pathNumber]\n", + "# gf = gf[gf.pgeVersion==\"2.1.1\"] \n", + "gf" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe3a0963-1d13-4b99-955a-b2c0fa45f1e3", + "metadata": {}, + "outputs": [], + "source": [ + "# Get only relevant metadata\n", + "cslc_df = gf[['operaBurstID', 'fileID', 'startTime', 'stopTime', 'url', 'geometry', 'pgeVersion']]\n", + "cslc_df['startTime'] = pd.to_datetime(cslc_df.startTime).dt.date\n", + "cslc_df['stopTime'] = pd.to_datetime(cslc_df.stopTime).dt.date\n", + "\n", + "# Extract production time from fileID (2nd date token)\n", + "def _prod_time_from_fileid(file_id):\n", + " # Example: OPERA_L2_CSLC-S1_..._20221122T161650Z_20240504T081640Z_...\n", + " parts = str(file_id).split('_')\n", + " return parts[5] if len(parts) > 5 else None\n", + "\n", + "cslc_df['productionTime'] = pd.to_datetime(cslc_df['fileID'].apply(_prod_time_from_fileid), format='%Y%m%dT%H%M%SZ', errors='coerce')\n", + "\n", + "# Keep newest duplicate by productionTime (fallback to pgeVersion, stopTime)\n", + "cslc_df = cslc_df.sort_values(by=['operaBurstID', 'startTime', 'productionTime', 'pgeVersion', 'stopTime'])\n", + "cslc_df = cslc_df.drop_duplicates(subset=['operaBurstID', 'startTime'], keep='last', ignore_index=True)\n", + "\n", + "import re\n", + "\n", + "def _subswath_from_fileid(file_id):\n", + " # Example: ...-IW2_... -> IW2\n", + " m = re.search(r\"-IW[1-3]_\", str(file_id))\n", + " return m.group(0)[1:4] if m else None\n", + "\n", + "cslc_df['burstSubswath'] = cslc_df['fileID'].apply(_subswath_from_fileid)\n", + "\n", + "# Optional filtering by subswath or specific burst IDs\n", + "if BURST_SUBSWATH:\n", + " if isinstance(BURST_SUBSWATH, (list, tuple, set)):\n", + " subswaths = {str(s).upper() for s in BURST_SUBSWATH}\n", + " else:\n", + " subswaths = {str(BURST_SUBSWATH).upper()}\n", + " cslc_df = cslc_df[cslc_df['burstSubswath'].str.upper().isin(subswaths)]\n", + "\n", + "if BURST_ID:\n", + " if isinstance(BURST_ID, (list, tuple, set)):\n", + " burst_ids = {str(b) for b in BURST_ID}\n", + " else:\n", + " burst_ids = {str(BURST_ID)}\n", + " cslc_df = cslc_df[cslc_df['operaBurstID'].isin(burst_ids)]\n", + "cslc_df\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76cb1007", + "metadata": {}, + "outputs": [], + "source": [ + "import shapely.wkt as wkt\n", + "import geopandas as gpd\n", + "\n", + "aoi_geom = wkt.loads(aoi)\n", + "aoi_gdf = gpd.GeoDataFrame(geometry=[aoi_geom], crs=\"EPSG:4326\")\n", + "\n", + "m = cslc_df[['operaBurstID', 'geometry']].explore(\n", + " zoom=9,\n", + " tiles=\"Esri.WorldImagery\",\n", + " style_kwds={\"fill\": True, \"fillColor\": \"blue\", \"fillOpacity\": 0.1, \"weight\": 2},\n", + " name=\"CSLC footprints\",\n", + ")\n", + "\n", + "aoi_gdf.explore(\n", + " m=m,\n", + " color=\"red\",\n", + " style_kwds={\"fill\": True, \"fillColor\": \"blue\", \"fillOpacity\": 0.1, \"weight\": 2},\n", + " name=\"AOI\",\n", + ")\n", + "\n", + "m\n" + ] + }, + { + "cell_type": "markdown", + "id": "9ca8a611", + "metadata": {}, + "source": [ + "## 5. Download the CSLC-S1 locally" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c0cde1fc", + "metadata": {}, + "outputs": [], + "source": [ + "## Download step skipped: CSLC subsets are streamed via opera-utils\n", + "print('Skipping full CSLC downloads; using opera-utils HTTP subsetting.')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aea6a5b7", + "metadata": {}, + "outputs": [], + "source": [ + "# Sort the CSLC-S1 by burstID and date\n", + "cslc_df = cslc_df.sort_values(by=[\"operaBurstID\", \"startTime\"], ignore_index=True)\n", + "cslc_df\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa4801fc", + "metadata": {}, + "outputs": [], + "source": [ + "# Enforce date range on dataframe (useful when re-running with narrower dates)\n", + "date_start_day = dateStart.date()\n", + "date_end_day = dateEnd.date()\n", + "cslc_df = cslc_df[(cslc_df['startTime'] >= date_start_day) & (cslc_df['startTime'] <= date_end_day)]\n", + "cslc_df = cslc_df.reset_index(drop=True)\n", + "cslc_df\n" + ] + }, + { + "cell_type": "markdown", + "id": "48add9c3", + "metadata": {}, + "source": [ + "## 6. Read each CSLC-S1 and stack them together\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01891a96", + "metadata": {}, + "outputs": [], + "source": [ + "cslc_stack = []; cslc_dates = []; bbox_stack = []; xcoor_stack = []; ycoor_stack = []\n", + "\n", + "import os\n", + "import time\n", + "import numpy as np\n", + "import tempfile\n", + "import requests\n", + "import xarray as xr\n", + "import h5py\n", + "from tqdm.auto import tqdm\n", + "from pathlib import Path\n", + "from pyproj import Transformer\n", + "from shapely import wkt\n", + "from shapely.ops import transform as shp_transform\n", + "from opera_utils.credentials import get_earthdata_username_password\n", + "from opera_utils.disp._remote import open_file\n", + "from opera_utils.disp._utils import _get_netcdf_encoding\n", + "\n", + "subset_dir = f\"{savedir}/subset_cslc\"\n", + "os.makedirs(subset_dir, exist_ok=True)\n", + "\n", + "def _extract_subset(input_obj, outpath, rows, cols, chunks=(1,256,256)):\n", + " X0, X1 = (cols.start, cols.stop) if cols is not None else (None, None)\n", + " Y0, Y1 = (rows.start, rows.stop) if rows is not None else (None, None)\n", + " ds = xr.open_dataset(input_obj, engine=\"h5netcdf\", group=\"data\")\n", + " subset = ds.isel(y_coordinates=slice(Y0, Y1), x_coordinates=slice(X0, X1))\n", + " subset.to_netcdf(\n", + " outpath,\n", + " engine=\"h5netcdf\",\n", + " group=\"data\",\n", + " encoding=_get_netcdf_encoding(subset, chunks=chunks),\n", + " )\n", + " for group in (\"metadata\", \"identification\"):\n", + " with h5py.File(input_obj) as hf, h5py.File(outpath, \"a\") as dest_hf:\n", + " hf.copy(group, dest_hf, name=group)\n", + " with h5py.File(outpath, \"a\") as hf:\n", + " ctype = h5py.h5t.py_create(np.complex64)\n", + " ctype.commit(hf[\"/\"].id, np.bytes_(\"complex64\"))\n", + "\n", + "def _subset_h5_to_disk(url, aoi_wkt, out_dir):\n", + " outpath = Path(out_dir) / Path(url).name\n", + " if outpath.exists():\n", + " return outpath\n", + "\n", + " # determine row/col slices by reading coords\n", + " with open_file(url) as in_f:\n", + " ds = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", + " xcoor = ds[\"x_coordinates\"].values\n", + " ycoor = ds[\"y_coordinates\"].values\n", + " epsg = int(ds[\"projection\"].values)\n", + "\n", + " aoi_geom = wkt.loads(aoi_wkt)\n", + " if epsg != 4326:\n", + " transformer = Transformer.from_crs('EPSG:4326', f'EPSG:{epsg}', always_xy=True)\n", + " aoi_geom = shp_transform(transformer.transform, aoi_geom)\n", + " minx, miny, maxx, maxy = aoi_geom.bounds\n", + " x_mask = (xcoor >= minx) & (xcoor <= maxx)\n", + " y_mask = (ycoor >= miny) & (ycoor <= maxy)\n", + " if not x_mask.any() or not y_mask.any():\n", + " raise ValueError('AOI does not intersect this CSLC extent')\n", + " ix = np.where(x_mask)[0]\n", + " iy = np.where(y_mask)[0]\n", + " rows = slice(iy.min(), iy.max()+1)\n", + " cols = slice(ix.min(), ix.max()+1)\n", + "\n", + " if url.startswith('s3://'):\n", + " with open_file(url) as in_f:\n", + " _extract_subset(in_f, outpath, rows, cols)\n", + " else:\n", + " # HTTPS: download to temp then subset\n", + " with tempfile.NamedTemporaryFile(suffix='.h5') as tf:\n", + " if url.startswith('http'):\n", + " session = requests.Session()\n", + " username, password = get_earthdata_username_password()\n", + " session.auth = (username, password)\n", + " resp = session.get(url)\n", + " resp.raise_for_status()\n", + " tf.write(resp.content)\n", + " tf.flush()\n", + " _extract_subset(tf.name, outpath, rows, cols)\n", + " return outpath\n", + "\n", + "def _load_subset(file_id, url, start_date):\n", + " outpath = _subset_h5_to_disk(url, aoi, subset_dir)\n", + " # now read subset locally with h5py (fast)\n", + " with h5py.File(outpath, 'r') as h5:\n", + " cslc = h5['/data/VV'][:]\n", + " xcoor = h5['/data/x_coordinates'][:]\n", + " ycoor = h5['/data/y_coordinates'][:]\n", + " dx = int(h5['/data/x_spacing'][()])\n", + " dy = int(h5['/data/y_spacing'][()])\n", + " epsg = int(h5['/data/projection'][()])\n", + " sensing_start = h5['/metadata/processing_information/input_burst_metadata/sensing_start'][()].astype(str)\n", + " sensing_stop = h5['/metadata/processing_information/input_burst_metadata/sensing_stop'][()].astype(str)\n", + " dims = h5['/metadata/processing_information/input_burst_metadata/shape'][:]\n", + " bounding_polygon = h5['/identification/bounding_polygon'][()].astype(str)\n", + " orbit_direction = h5['/identification/orbit_pass_direction'][()].astype(str)\n", + " center_lon, center_lat = h5['/metadata/processing_information/input_burst_metadata/center']\n", + " wavelength = h5['/metadata/processing_information/input_burst_metadata/wavelength'][()].astype(str)\n", + " subset_bbox = [float(xcoor.min()), float(xcoor.max()), float(ycoor.min()), float(ycoor.max())]\n", + " return cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox\n", + "\n", + "# Subset with progress (parallel)\n", + "from concurrent.futures import ThreadPoolExecutor, as_completed\n", + "\n", + "items = list(zip(cslc_df.fileID, cslc_df.url, cslc_df.startTime))\n", + "# Diagnostic: check pixel spacing before multilooking\n", + "with open_file(items[0][1]) as in_f:\n", + " ds0 = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", + " dx0 = float(ds0[\"x_spacing\"].values)\n", + " dy0 = float(ds0[\"y_spacing\"].values)\n", + "print(f\"Pixel spacing (dx, dy) = ({dx0}, {dy0})\")\n", + "\n", + "results = [None] * len(items)\n", + "_t0 = time.perf_counter()\n", + "with ThreadPoolExecutor(max_workers=DOWNLOAD_PROCESSES) as ex:\n", + " futures = {ex.submit(_load_subset, fileID, url, start_date): i for i, (fileID, url, start_date) in enumerate(items)}\n", + " for fut in tqdm(as_completed(futures), total=len(futures), desc='Subsetting CSLC'):\n", + " i = futures[fut]\n", + " results[i] = fut.result()\n", + "_t1 = time.perf_counter()\n", + "print(f\"Subset/download time: {_t1 - _t0:.1f} s\")\n", + "\n", + "for (fileID, start_date), res in zip(zip(cslc_df.fileID, cslc_df.startTime), results):\n", + " cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox = res\n", + " cslc_stack.append(cslc)\n", + " cslc_dates.append(pd.to_datetime(sensing_start).date())\n", + " if subset_bbox is not None:\n", + " bbox = subset_bbox\n", + " else:\n", + " cslc_poly = wkt.loads(bounding_polygon)\n", + " bbox = [cslc_poly.bounds[0], cslc_poly.bounds[2], cslc_poly.bounds[1], cslc_poly.bounds[3]]\n", + " bbox_stack.append(bbox)\n", + " xcoor_stack.append(xcoor)\n", + " ycoor_stack.append(ycoor)\n" + ] + }, + { + "cell_type": "markdown", + "id": "f5e97e45", + "metadata": {}, + "source": [ + "## 7. Generate the interferograms, compute for the coherence, save the files as GeoTiffs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57c74ac4", + "metadata": {}, + "outputs": [], + "source": [ + "def colorize(array=[], cmap='RdBu', cmin=[], cmax=[]):\n", + " normed_data = (array - cmin) / (cmax - cmin) \n", + " cm = plt.cm.get_cmap(cmap)\n", + " return cm(normed_data) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25c56034", + "metadata": {}, + "outputs": [], + "source": [ + "def goldstein_filter(ifg_cpx, alpha=0.5, pad=32, edge_trim=16):\n", + " # Goldstein filter with padding + taper + mask to reduce edge effects\n", + " mask = np.isfinite(ifg_cpx)\n", + " data = np.nan_to_num(ifg_cpx, nan=0.0)\n", + " if pad and pad > 0:\n", + " data = np.pad(data, ((pad, pad), (pad, pad)), mode=\"reflect\")\n", + " mask = np.pad(mask, ((pad, pad), (pad, pad)), mode=\"constant\", constant_values=False)\n", + " # Apply 2D Hann window (taper)\n", + " wy = np.hanning(data.shape[0])\n", + " wx = np.hanning(data.shape[1])\n", + " window = wy[:, None] * wx[None, :]\n", + " f = np.fft.fft2(data * window)\n", + " s = np.abs(f)\n", + " s = s / (s.max() + 1e-8)\n", + " f_filt = f * (s ** alpha)\n", + " out = np.fft.ifft2(f_filt)\n", + " if pad and pad > 0:\n", + " out = out[pad:-pad, pad:-pad]\n", + " mask = mask[pad:-pad, pad:-pad]\n", + " # restore NaNs outside valid mask\n", + " out[~mask] = np.nan\n", + " if edge_trim and edge_trim > 0:\n", + " out[:edge_trim, :] = np.nan\n", + " out[-edge_trim:, :] = np.nan\n", + " out[:, :edge_trim] = np.nan\n", + " out[:, -edge_trim:] = np.nan\n", + " return out\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82bda5ae", + "metadata": {}, + "outputs": [], + "source": [ + "def rasterWrite(outtif,arr,transform,epsg,dtype='float32'):\n", + " #writing geotiff using rasterio\n", + " \n", + " new_dataset = rasterio.open(outtif, 'w', driver='GTiff',\n", + " height = arr.shape[0], width = arr.shape[1],\n", + " count=1, dtype=dtype,\n", + " crs=CRS.from_epsg(epsg),\n", + " transform=transform,nodata=np.nan)\n", + " new_dataset.write(arr, 1)\n", + " new_dataset.close() " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b47f82a5", + "metadata": {}, + "outputs": [], + "source": [ + "## Build date pairs per burstID\n", + "cslc_dates = cslc_df[[\"startTime\"]]\n", + "burstID = cslc_df.operaBurstID.drop_duplicates(ignore_index=True)\n", + "n_unique_burstID = len(burstID)\n", + "\n", + "def _lag_list(lag):\n", + " if lag is None:\n", + " return []\n", + " if isinstance(lag, (list, tuple, set)):\n", + " return sorted({int(x) for x in lag})\n", + " return [int(lag)]\n", + "\n", + "pair_lags = _lag_list(pair_t_span_days)\n", + "pair_indices = [] # list of (ref_idx, sec_idx) in cslc_df order\n", + "\n", + "for bid, group in cslc_df.groupby('operaBurstID'):\n", + " group = group.sort_values('startTime')\n", + " idx = group.index.to_list()\n", + " dates = group['startTime'].to_list()\n", + "\n", + " burst_pairs = []\n", + " if pair_mode == 'all':\n", + " for i in range(len(idx)):\n", + " for j in range(i+1, len(idx)):\n", + " delta = (dates[j] - dates[i]).days\n", + " if delta < min_t_span_days:\n", + " continue\n", + " if max_t_span_days is not None and delta > max_t_span_days:\n", + " continue\n", + " burst_pairs.append((idx[i], idx[j]))\n", + " elif pair_mode == 't_span':\n", + " for i in range(len(idx)):\n", + " for j in range(i+1, len(idx)):\n", + " delta = (dates[j] - dates[i]).days\n", + " if delta in pair_lags and delta >= min_t_span_days and (max_t_span_days is None or delta <= max_t_span_days):\n", + " burst_pairs.append((idx[i], idx[j]))\n", + " else:\n", + " raise ValueError(\"pair_mode must be 'all' or 't_span'\")\n", + "\n", + " if max_pairs_per_burst is not None:\n", + " burst_pairs = burst_pairs[:int(max_pairs_per_burst)]\n", + "\n", + " pair_indices.extend(burst_pairs)\n", + "\n", + "if max_pairs_total is not None:\n", + " pair_indices = pair_indices[:int(max_pairs_total)]\n", + "\n", + "# Sort pairs by date, then burstID (so same dates group together)\n", + "def _pair_sort_key(pair):\n", + " ref_idx, sec_idx = pair\n", + " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", + " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", + " burst = cslc_df.operaBurstID.iloc[ref_idx]\n", + " return (ref_date, sec_date, burst)\n", + "pair_indices = sorted(pair_indices, key=_pair_sort_key)\n", + "print(f'Pair count: {len(pair_indices)}')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6852f778", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import warnings\n", + "from numpy.lib.stride_tricks import sliding_window_view\n", + "\n", + "\n", + "\n", + "\n", + "def take_looks(arr, row_looks, col_looks, func_type=\"nanmean\", edge_strategy=\"cutoff\"):\n", + " if row_looks == 1 and col_looks == 1:\n", + " return arr\n", + " if arr.ndim != 2:\n", + " raise ValueError(\"take_looks expects 2D array\")\n", + " rows, cols = arr.shape\n", + " if edge_strategy == \"cutoff\":\n", + " rows = (rows // row_looks) * row_looks\n", + " cols = (cols // col_looks) * col_looks\n", + " arr = arr[:rows, :cols]\n", + " elif edge_strategy == \"pad\":\n", + " pad_r = (-rows) % row_looks\n", + " pad_c = (-cols) % col_looks\n", + " if pad_r or pad_c:\n", + " arr = np.pad(arr, ((0, pad_r), (0, pad_c)), mode=\"constant\", constant_values=np.nan)\n", + " rows, cols = arr.shape\n", + " else:\n", + " raise ValueError(\"edge_strategy must be 'cutoff' or 'pad'\")\n", + "\n", + " new_rows = rows // row_looks\n", + " new_cols = cols // col_looks\n", + " func = getattr(np, func_type)\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\", category=RuntimeWarning)\n", + " return func(arr.reshape(new_rows, row_looks, new_cols, col_looks), axis=(1, 3))\n", + "\n", + "\n", + "def _multilook(arr, looks_y=1, looks_x=1):\n", + " return take_looks(arr, looks_y, looks_x, func_type=\"nanmean\", edge_strategy=\"cutoff\")\n", + "\n", + "def _box_mean(arr, win):\n", + " pad = win // 2\n", + " arr_p = np.pad(arr, pad_width=pad, mode='reflect')\n", + " windows = sliding_window_view(arr_p, (win, win))\n", + " return windows.mean(axis=(-2, -1))\n", + "\n", + " return _box_mean(arr, win)\n", + "\n", + "def lee_filter(img, win=5):\n", + " mean = _box_mean(img, win)\n", + " mean_sq = _box_mean(img**2, win)\n", + " var = mean_sq - mean**2\n", + " noise_var = np.nanmedian(var)\n", + " w = var / (var + noise_var + 1e-8)\n", + " return mean + w * (img - mean)\n", + "\n", + "def goldstein(phase, alpha, psize=32):\n", + " \"\"\"Apply the Goldstein adaptive filter to the given data.\"\"\"\n", + " def apply_pspec(data):\n", + " if alpha < 0:\n", + " raise ValueError(f\"alpha must be >= 0, got {alpha = }\")\n", + " weight = np.power(np.abs(data) ** 2, alpha / 2)\n", + " data = weight * data\n", + " return data\n", + "\n", + " def make_weight(nxp, nyp):\n", + " wx = 1.0 - np.abs(np.arange(nxp // 2) - (nxp / 2.0 - 1.0)) / (nxp / 2.0 - 1.0)\n", + " wy = 1.0 - np.abs(np.arange(nyp // 2) - (nyp / 2.0 - 1.0)) / (nyp / 2.0 - 1.0)\n", + " quadrant = np.outer(wy, wx)\n", + " weight = np.block(\n", + " [\n", + " [quadrant, np.flip(quadrant, axis=1)],\n", + " [np.flip(quadrant, axis=0), np.flip(np.flip(quadrant, axis=0), axis=1)],\n", + " ]\n", + " )\n", + " return weight\n", + "\n", + " def patch_goldstein_filter(data, weight, psize):\n", + " data = np.fft.fft2(data, s=(psize, psize))\n", + " data = apply_pspec(data)\n", + " data = np.fft.ifft2(data, s=(psize, psize))\n", + " return weight * data\n", + "\n", + " def apply_goldstein_filter(data):\n", + " out = np.zeros(data.shape, dtype=np.complex64)\n", + " empty_mask = np.isnan(data) | (np.angle(data) == 0)\n", + " if np.all(empty_mask):\n", + " return data\n", + " weight_matrix = make_weight(psize, psize)\n", + " for i in range(0, data.shape[0] - psize, psize // 2):\n", + " for j in range(0, data.shape[1] - psize, psize // 2):\n", + " data_window = data[i : i + psize, j : j + psize]\n", + " weight_window = weight_matrix[: data_window.shape[0], : data_window.shape[1]]\n", + " filtered_window = patch_goldstein_filter(data_window, weight_window, psize)\n", + " slice_i = slice(i, min(i + psize, out.shape[0]))\n", + " slice_j = slice(j, min(j + psize, out.shape[1]))\n", + " out[slice_i, slice_j] += filtered_window[: slice_i.stop - slice_i.start, : slice_j.stop - slice_j.start]\n", + " out[empty_mask] = 0\n", + " return out\n", + "\n", + " if np.iscomplexobj(phase):\n", + " return apply_goldstein_filter(phase)\n", + " else:\n", + " return apply_goldstein_filter(np.exp(1j * phase))\n", + "\n", + " phase = reference * np.conjugate(secondary)\n", + " amp = np.sqrt((reference * np.conjugate(reference)) * (secondary * np.conjugate(secondary)))\n", + " nan_mask = np.isnan(phase)\n", + " ifg[nan_mask] = np.nan\n", + " ifg_cpx = np.exp(1j * np.nan_to_num(np.angle(phase/amp)))\n", + " zero_mask = phase == 0\n", + " coh[nan_mask] = np.nan\n", + " coh[zero_mask] = 0\n", + " return ifg, coh, amp\n", + "\n", + "def calc_ifg_coh_filtered(reference, secondary, goldstein_alpha=0.5, coh_win=5, looks_y=1, looks_x=1):\n", + " reference = _multilook(reference, looks_y, looks_x)\n", + " secondary = _multilook(secondary, looks_y, looks_x)\n", + " phase = reference * np.conjugate(secondary)\n", + " amp = np.sqrt((reference * np.conjugate(reference)) * (secondary * np.conjugate(secondary)))\n", + " nan_mask = np.isnan(phase)\n", + " ifg_cpx = np.exp(1j * np.nan_to_num(np.angle(phase/amp)))\n", + " ifg_cpx_f = goldstein(ifg_cpx, alpha=goldstein_alpha, psize=32)\n", + " ifg = np.angle(ifg_cpx_f)\n", + " ifg[nan_mask] = np.nan\n", + " coh = np.abs(_box_mean(ifg_cpx, coh_win))\n", + " coh = np.clip(coh, 0, 1)\n", + " coh = lee_filter(coh, win=coh_win)\n", + " coh = np.clip(coh, 0, 1)\n", + " zero_mask = phase == 0\n", + " coh[nan_mask] = np.nan\n", + " coh[zero_mask] = 0\n", + " return ifg, coh, amp\n", + "\n", + "def calc_ifg_coh(reference, secondary, looks_y=1, looks_x=1):\n", + " return calc_ifg_coh_filtered(reference, secondary, looks_y=looks_y, looks_x=looks_x)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2775556c", + "metadata": {}, + "outputs": [], + "source": [ + "## For each date-pair, calculate the ifg, coh. Save the results as GeoTiffs.\n", + "for ref_idx, sec_idx in pair_indices:\n", + " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", + " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", + " print(f\"Reference: {ref_date} Secondary: {sec_date}\")\n", + "\n", + " # Calculate ifg, coh, amp\n", + " if \"calc_ifg_coh_filtered\" not in globals():\n", + " raise RuntimeError(\"calc_ifg_coh_filtered is not defined. Run the filter definition cell first.\")\n", + " looks_y, looks_x = MULTILOOK\n", + " ifg, coh, amp = calc_ifg_coh_filtered(cslc_stack[ref_idx], cslc_stack[sec_idx], looks_y=looks_y, looks_x=looks_x)\n", + "\n", + " # Save each interferogram as GeoTiff (no per-burst plotting)\n", + " transform = from_origin(xcoor_stack[ref_idx][0], ycoor_stack[ref_idx][0], dx, np.abs(dy))\n" + ] + }, + { + "cell_type": "markdown", + "id": "d57d1a71", + "metadata": {}, + "source": [ + "## 8. Merge the burst-wise interferograms and coherence and save as GeoTiff." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e98baf1f", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_merge(old_data, new_data, old_nodata, new_nodata, **kwargs): \n", + " mask = np.logical_and(~old_nodata, ~new_nodata)\n", + " old_data[mask] = new_data[mask]\n", + " mask = np.logical_and(old_nodata, ~new_nodata)\n", + " old_data[mask] = new_data[mask]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "897e44dd", + "metadata": {}, + "outputs": [], + "source": [ + "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", + "# Merge burst-wise interferograms per date-pair (in-memory)\n", + "from rasterio.io import MemoryFile\n", + "\n", + "from rasterio.warp import calculate_default_transform, reproject, Resampling\n", + "\n", + "\n", + "from rasterio.warp import reproject, Resampling\n", + "from pathlib import Path\n", + "\n", + "def _load_water_mask_match(mask_path, shape, transform, crs):\n", + " if mask_path is None or not Path(mask_path).exists():\n", + " return None\n", + " with rasterio.open(mask_path) as src:\n", + " src_mask = src.read(1)\n", + " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", + " return src_mask\n", + " dst = np.zeros(shape, dtype=src_mask.dtype)\n", + " reproject(\n", + " source=src_mask,\n", + " destination=dst,\n", + " src_transform=src.transform,\n", + " src_crs=src.crs,\n", + " dst_transform=transform,\n", + " dst_crs=crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " return dst\n", + "\n", + "def _apply_water_mask(arr, mask):\n", + " # mask: 1 = keep land, 0 = water\n", + " return np.where(mask == 0, np.nan, arr)\n", + "\n", + "\n", + "from affine import Affine\n", + "\n", + "def _trim_nan_border(arr, transform):\n", + " data = arr[0] if arr.ndim == 3 else arr\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if not mask.any():\n", + " return arr, transform\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " data = data[r0:r1, c0:c1]\n", + " if arr.ndim == 3:\n", + " arr = data[None, ...]\n", + " else:\n", + " arr = data\n", + " new_transform = transform * Affine.translation(c0, r0)\n", + " return arr, new_transform\n", + "\n", + "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", + " import os\n", + " os.makedirs(os.path.dirname(out_path), exist_ok=True)\n", + " dst_crs = 'EPSG:4326'\n", + " dst_transform, width, height = calculate_default_transform(\n", + " f'EPSG:{epsg}', dst_crs, mosaic.shape[2], mosaic.shape[1], *rasterio.transform.array_bounds(mosaic.shape[1], mosaic.shape[2], transform)\n", + " )\n", + " dest = np.zeros((1, height, width), dtype=mosaic.dtype)\n", + " reproject(\n", + " source=mosaic,\n", + " destination=dest,\n", + " src_transform=transform,\n", + " src_crs=f'EPSG:{epsg}',\n", + " dst_transform=dst_transform,\n", + " dst_crs=dst_crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " out_meta = {\n", + " 'driver': 'GTiff',\n", + " 'height': height,\n", + " 'width': width,\n", + " 'count': 1,\n", + " 'dtype': mosaic.dtype,\n", + " 'crs': dst_crs,\n", + " 'transform': dst_transform,\n", + " }\n", + " with rasterio.open(out_path, 'w', **out_meta) as dst:\n", + " dst.write(dest)\n", + "\n", + "# Group pair indices by date tag\n", + "pairs_by_tag = {}\n", + "for r, s in pair_indices:\n", + " ref_date = cslc_dates.iloc[r].values[0]\n", + " sec_date = cslc_dates.iloc[s].values[0]\n", + " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", + " pairs_by_tag.setdefault(tag, []).append((r, s))\n", + "\n", + "for tag, pairs in pairs_by_tag.items():\n", + " srcs = []\n", + " for r, s in pairs:\n", + " looks_y, looks_x = MULTILOOK\n", + " ifg, coh, amp = calc_ifg_coh(cslc_stack[r], cslc_stack[s], looks_y=looks_y, looks_x=looks_x)\n", + " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", + " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", + " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", + " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", + " mem = MemoryFile()\n", + " ds = mem.open(\n", + " driver='GTiff', height=ifg.shape[0], width=ifg.shape[1], count=1, dtype=ifg.dtype,\n", + " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " )\n", + " ds.write(ifg, 1)\n", + " srcs.append(ds)\n", + " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", + " dest, output_transform = _trim_nan_border(dest, output_transform)\n", + " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", + " if mask is not None:\n", + " dest[0] = _apply_water_mask(dest[0], mask)\n", + " out_meta = srcs[0].meta.copy()\n", + " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", + " out_path = f\"{savedir}/tifs/merged_ifg_{tag}.tif\"\n", + " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", + " dest1.write(dest)\n", + " if SAVE_WGS84:\n", + " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_ifg_WGS84_{tag}.tif\"\n", + " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", + " for ds in srcs:\n", + " ds.close()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c41924c", + "metadata": {}, + "outputs": [], + "source": [ + "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", + "# Merge burst-wise coherence per date-pair (in-memory)\n", + "from rasterio.io import MemoryFile\n", + "\n", + "from rasterio.warp import calculate_default_transform, reproject, Resampling\n", + "\n", + "\n", + "from rasterio.warp import reproject, Resampling\n", + "\n", + "def _load_water_mask_match(mask_path, shape, transform, crs):\n", + " with rasterio.open(mask_path) as src:\n", + " src_mask = src.read(1)\n", + " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", + " return src_mask\n", + " dst = np.zeros(shape, dtype=src_mask.dtype)\n", + " reproject(\n", + " source=src_mask,\n", + " destination=dst,\n", + " src_transform=src.transform,\n", + " src_crs=src.crs,\n", + " dst_transform=transform,\n", + " dst_crs=crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " return dst\n", + "\n", + "def _apply_water_mask(arr, mask):\n", + " # mask: 1 = keep land, 0 = water\n", + " return np.where(mask == 0, np.nan, arr)\n", + "\n", + "\n", + "from affine import Affine\n", + "\n", + "def _trim_nan_border(arr, transform):\n", + " data = arr[0] if arr.ndim == 3 else arr\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if not mask.any():\n", + " return arr, transform\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " data = data[r0:r1, c0:c1]\n", + " if arr.ndim == 3:\n", + " arr = data[None, ...]\n", + " else:\n", + " arr = data\n", + " new_transform = transform * Affine.translation(c0, r0)\n", + " return arr, new_transform\n", + "\n", + "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", + " import os\n", + " os.makedirs(os.path.dirname(out_path), exist_ok=True)\n", + " dst_crs = 'EPSG:4326'\n", + " dst_transform, width, height = calculate_default_transform(\n", + " f'EPSG:{epsg}', dst_crs, mosaic.shape[2], mosaic.shape[1], *rasterio.transform.array_bounds(mosaic.shape[1], mosaic.shape[2], transform)\n", + " )\n", + " dest = np.zeros((1, height, width), dtype=mosaic.dtype)\n", + " reproject(\n", + " source=mosaic,\n", + " destination=dest,\n", + " src_transform=transform,\n", + " src_crs=f'EPSG:{epsg}',\n", + " dst_transform=dst_transform,\n", + " dst_crs=dst_crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " out_meta = {\n", + " 'driver': 'GTiff',\n", + " 'height': height,\n", + " 'width': width,\n", + " 'count': 1,\n", + " 'dtype': mosaic.dtype,\n", + " 'crs': dst_crs,\n", + " 'transform': dst_transform,\n", + " }\n", + " with rasterio.open(out_path, 'w', **out_meta) as dst:\n", + " dst.write(dest)\n", + "\n", + "# Group pair indices by date tag\n", + "pairs_by_tag = {}\n", + "for r, s in pair_indices:\n", + " ref_date = cslc_dates.iloc[r].values[0]\n", + " sec_date = cslc_dates.iloc[s].values[0]\n", + " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", + " pairs_by_tag.setdefault(tag, []).append((r, s))\n", + "\n", + "for tag, pairs in pairs_by_tag.items():\n", + " srcs = []\n", + " for r, s in pairs:\n", + " looks_y, looks_x = MULTILOOK\n", + " ifg, coh, amp = calc_ifg_coh(cslc_stack[r], cslc_stack[s], looks_y=looks_y, looks_x=looks_x)\n", + " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", + " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", + " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", + " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", + " mem = MemoryFile()\n", + " ds = mem.open(\n", + " driver='GTiff', height=coh.shape[0], width=coh.shape[1], count=1, dtype=coh.dtype,\n", + " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " )\n", + " ds.write(coh, 1)\n", + " srcs.append(ds)\n", + " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", + " dest, output_transform = _trim_nan_border(dest, output_transform)\n", + " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", + " dest[0] = _apply_water_mask(dest[0], mask)\n", + " out_meta = srcs[0].meta.copy()\n", + " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", + " out_path = f\"{savedir}/tifs/merged_coh_{tag}.tif\"\n", + " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", + " dest1.write(dest)\n", + " if SAVE_WGS84:\n", + " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_coh_WGS84_{tag}.tif\"\n", + " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", + " for ds in srcs:\n", + " ds.close()\n" + ] + }, + { + "cell_type": "markdown", + "id": "858ea831", + "metadata": {}, + "source": [ + "## 9. Read the merged GeoTiff and Visualize using `matplotlib`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9cb2a341", + "metadata": {}, + "outputs": [], + "source": [ + "# Read merged IFG/COH files and plot paired grids\n", + "import glob\n", + "import math\n", + "import pandas as pd\n", + "import os\n", + "\n", + "\n", + "# Output dir for per-pair PNGs\n", + "pair_png_dir = f\"{savedir}/pairs_png\"\n", + "os.makedirs(pair_png_dir, exist_ok=True)\n", + "\n", + "ifg_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_ifg_*.tif\"))\n", + "coh_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\"))\n", + "\n", + "ifg_map = {p.split('merged_ifg_')[-1].replace('.tif',''): p for p in ifg_paths}\n", + "coh_map = {p.split('merged_coh_')[-1].replace('.tif',''): p for p in coh_paths}\n", + "\n", + "\n", + "\n", + "def _prep_da(path):\n", + " da = rioxarray.open_rasterio(path)[0]\n", + " if REPROJECT_FOR_DISPLAY:\n", + " da = da.rio.reproject(\"EPSG:4326\")\n", + " data = da.values\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if mask.any():\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " # Trim NaN borders so edges don't show padding\n", + " da = da.isel(y=slice(r0, r1), x=slice(c0, c1))\n", + " return da\n", + "\n", + "pair_tags = sorted(set(ifg_map).intersection(coh_map))\n", + "# Filter pairs by current date range\n", + "date_start_day = dateStart.date()\n", + "date_end_day = dateEnd.date()\n", + "pair_tags = [t for t in pair_tags if (date_start_day <= pd.to_datetime(t.split('-')[0], format='%Y%m%d').date() <= date_end_day and date_start_day <= pd.to_datetime(t.split('-')[1], format='%Y%m%d').date() <= date_end_day)]\n", + "\n", + "if not pair_tags:\n", + " print('No matching IFG/COH pairs found')\n", + "else:\n", + " # Save ALL pairs as PNGs\n", + " for tag in pair_tags:\n", + " fig, axes = plt.subplots(1, 2, figsize=(10, 4), constrained_layout=True)\n", + " ax_ifg, ax_coh = axes\n", + "\n", + " # IFG\n", + " merged_ifg = _prep_da(ifg_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", + " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", + " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", + " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", + " ax_ifg.set_xticks([])\n", + " ax_ifg.set_yticks([])\n", + " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", + "\n", + " # COH\n", + " merged_coh = _prep_da(coh_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", + " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=0, vmax=1.0)\n", + " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", + " ax_coh.set_xticks([])\n", + " ax_coh.set_yticks([])\n", + " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')\n", + "\n", + " out_png = os.path.join(pair_png_dir, f\"pair_{tag}.png\")\n", + " fig.savefig(out_png, dpi=150)\n", + " plt.close(fig)\n", + "\n", + " # Display only last 5 pairs in notebook\n", + " display_tags = pair_tags[-5:]\n", + " n = len(display_tags)\n", + " ncols = 2\n", + " nrows = math.ceil(n / 1) # one pair per row\n", + " fig, axes = plt.subplots(nrows, ncols, figsize=(6*ncols, 3*nrows), constrained_layout=True)\n", + " if nrows == 1:\n", + " axes = [axes]\n", + "\n", + " for i, tag in enumerate(display_tags):\n", + " ax_ifg, ax_coh = axes[i]\n", + "\n", + " # IFG\n", + " merged_ifg = _prep_da(ifg_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", + " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", + " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", + " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", + " ax_ifg.set_xticks([])\n", + " ax_ifg.set_yticks([])\n", + " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", + "\n", + " # COH\n", + " merged_coh = _prep_da(coh_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", + " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=0, vmax=1.0)\n", + " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", + " ax_coh.set_xticks([])\n", + " ax_coh.set_yticks([])\n", + " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')\n" + ] + }, + { + "cell_type": "markdown", + "id": "ea1e4f24", + "metadata": {}, + "source": [ + "## 9.5 Merge and plot amplitude mosaics (per date)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7f478f4", + "metadata": {}, + "outputs": [], + "source": [ + "import h5py\n", + "import numpy as np\n", + "import glob\n", + "import math\n", + "import rasterio\n", + "from rasterio.transform import from_origin\n", + "from rasterio.crs import CRS\n", + "from rasterio.warp import calculate_default_transform, reproject, Resampling\n", + "\n", + "\n", + "from rasterio.warp import reproject, Resampling\n", + "\n", + "def _load_water_mask_match(mask_path, shape, transform, crs):\n", + " with rasterio.open(mask_path) as src:\n", + " src_mask = src.read(1)\n", + " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", + " return src_mask\n", + " dst = np.zeros(shape, dtype=src_mask.dtype)\n", + " reproject(\n", + " source=src_mask,\n", + " destination=dst,\n", + " src_transform=src.transform,\n", + " src_crs=src.crs,\n", + " dst_transform=transform,\n", + " dst_crs=crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " return dst\n", + "\n", + "def _apply_water_mask(arr, mask):\n", + " # mask: 1 = keep land, 0 = water\n", + " return np.where(mask == 0, np.nan, arr)\n", + "\n", + "\n", + "from affine import Affine\n", + "\n", + "def _trim_nan_border(arr, transform):\n", + " data = arr[0] if arr.ndim == 3 else arr\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if not mask.any():\n", + " return arr, transform\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " data = data[r0:r1, c0:c1]\n", + " if arr.ndim == 3:\n", + " arr = data[None, ...]\n", + " else:\n", + " arr = data\n", + " new_transform = transform * Affine.translation(c0, r0)\n", + " return arr, new_transform\n", + "\n", + "# Build per-date amplitude mosaics directly from subset H5 (no per-burst GeoTIFFs)\n", + "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", + "\n", + "date_tags = sorted(cslc_df.startTime.astype(str).str.replace('-', '').unique())\n", + "\n", + "def _save_mosaic_utm(out_path, mosaic, transform, epsg):\n", + " with rasterio.open(\n", + " out_path, \"w\", driver=\"GTiff\", height=mosaic.shape[0], width=mosaic.shape[1],\n", + " count=1, dtype=mosaic.dtype, crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " ) as dst_ds:\n", + " dst_ds.write(mosaic, 1)\n", + "\n", + "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", + " dst_crs = \"EPSG:4326\"\n", + " src_crs = CRS.from_epsg(epsg)\n", + " height, width = mosaic.shape\n", + " dst_transform, dst_width, dst_height = calculate_default_transform(\n", + " src_crs, dst_crs, width, height, *rasterio.transform.array_bounds(height, width, transform)\n", + " )\n", + " dst = np.empty((dst_height, dst_width), dtype=mosaic.dtype)\n", + " reproject(\n", + " source=mosaic,\n", + " destination=dst,\n", + " src_transform=transform,\n", + " src_crs=src_crs,\n", + " dst_transform=dst_transform,\n", + " dst_crs=dst_crs,\n", + " resampling=Resampling.bilinear,\n", + " )\n", + " with rasterio.open(\n", + " out_path, \"w\", driver=\"GTiff\", height=dst_height, width=dst_width, count=1,\n", + " dtype=dst.dtype, crs=dst_crs, transform=dst_transform, nodata=np.nan\n", + " ) as dst_ds:\n", + " dst_ds.write(dst, 1)\n", + "\n", + "# Mosaicking helper (in memory)\n", + "def _mosaic_arrays(arrays, transforms, epsg):\n", + " # Convert arrays to in-memory rasterio datasets via MemoryFile\n", + " from rasterio.io import MemoryFile\n", + " srcs = []\n", + " for arr, transform in zip(arrays, transforms):\n", + " mem = MemoryFile()\n", + " ds = mem.open(\n", + " driver='GTiff', height=arr.shape[0], width=arr.shape[1], count=1, dtype=arr.dtype,\n", + " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " )\n", + " ds.write(arr, 1)\n", + " srcs.append(ds)\n", + " dest, out_transform = merge.merge(srcs, method=custom_merge)\n", + " for ds in srcs:\n", + " ds.close()\n", + " return dest[0], out_transform\n", + "\n", + "looks_y, looks_x = MULTILOOK\n", + "for date_tag in date_tags:\n", + " # collect subset H5 files for this date\n", + " rows = cslc_df[cslc_df.startTime.astype(str).str.replace('-', '') == date_tag]\n", + " arrays = []\n", + " transforms = []\n", + " epsg = None\n", + " for fileID in rows.fileID:\n", + " subset_path = f\"{savedir}/subset_cslc/{fileID}.h5\"\n", + " with h5py.File(subset_path, 'r') as h5:\n", + " cslc = h5['/data/VV'][:]\n", + " xcoor = h5['/data/x_coordinates'][:]\n", + " ycoor = h5['/data/y_coordinates'][:]\n", + " dx = int(h5['/data/x_spacing'][()])\n", + " dy = int(h5['/data/y_spacing'][()])\n", + " epsg = int(h5['/data/projection'][()])\n", + " power_ml = _multilook(np.abs(cslc)**2, looks_y, looks_x)\n", + " amp = 10*np.log10(power_ml)\n", + " dy_signed = (ycoor[1] - ycoor[0]) if len(ycoor) > 1 else -dy\n", + " x0 = xcoor[0] + (looks_x - 1) * dx / 2\n", + " y0 = ycoor[0] + (looks_y - 1) * dy_signed / 2\n", + " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", + " arrays.append(amp)\n", + " transforms.append(transform)\n", + "\n", + " if not arrays:\n", + " continue\n", + " mosaic, out_transform = _mosaic_arrays(arrays, transforms, epsg)\n", + " out_path_utm = f\"{savedir}/tifs/merged_amp_{date_tag}.tif\"\n", + " mosaic, out_transform = _trim_nan_border(mosaic, out_transform)\n", + " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " mask = _load_water_mask_match(WATER_MASK_PATH, mosaic.shape, out_transform, CRS.from_epsg(epsg))\n", + " mosaic = _apply_water_mask(mosaic, mask)\n", + " _save_mosaic_utm(out_path_utm, mosaic, out_transform, epsg)\n", + " if SAVE_WGS84:\n", + " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_amp_WGS84_{date_tag}.tif\"\n", + " _save_mosaic_utm_to_wgs84(out_path_wgs84, mosaic, out_transform, epsg)\n", + "\n", + "# Plot merged amplitude mosaics in a grid (native CRS from saved GeoTIFFs)\n", + "\n", + "# Output dir for amplitude PNGs\n", + "amp_png_dir = f\"{savedir}/amp_png\"\n", + "os.makedirs(amp_png_dir, exist_ok=True)\n", + "\n", + "paths = sorted(glob.glob(f\"{savedir}/tifs/merged_amp_*.tif\"))\n", + "paths = [p for p in paths if 'WGS84' not in p]\n", + "all_vals = []\n", + "for p in paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " all_vals.append(da.values.ravel())\n", + "if all_vals:\n", + " all_vals = np.concatenate(all_vals)\n", + " gmin = np.nanpercentile(all_vals, 2)\n", + " gmax = np.nanpercentile(all_vals, 90)\n", + "else:\n", + " gmin, gmax = None, None\n", + "n = len(paths)\n", + "if n == 0:\n", + " print('No merged amplitude files found')\n", + "else:\n", + " # Save ALL amplitude PNGs\n", + " for path in paths:\n", + " src = rioxarray.open_rasterio(path)\n", + " amp = src[0]\n", + " minlon, minlat, maxlon, maxlat = amp.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " fig, ax = plt.subplots(figsize=(5,4))\n", + " im = ax.imshow(amp.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", + " tag = path.split('merged_amp_')[-1].replace('.tif','')\n", + " ax.set_title(f\"AMP_{tag}\", fontsize=10)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " fig.colorbar(im, ax=ax, orientation='vertical', fraction=0.046, pad=0.02)\n", + " out_png = os.path.join(amp_png_dir, f\"amp_{tag}.png\")\n", + " fig.savefig(out_png, dpi=150)\n", + " plt.close(fig)\n", + "\n", + " # Show only last 5 in notebook\n", + " display_paths = paths[-5:]\n", + " n = len(display_paths)\n", + " ncols = 3\n", + " nrows = math.ceil(n / ncols)\n", + " fig, axes = plt.subplots(nrows, ncols, figsize=(4*ncols, 3*nrows), constrained_layout=True)\n", + " axes = axes.ravel()\n", + " for ax, path in zip(axes, display_paths):\n", + " src = rioxarray.open_rasterio(path)\n", + " amp = src[0]\n", + " minlon, minlat, maxlon, maxlat = amp.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " im = ax.imshow(amp.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", + " tag = path.split('merged_amp_')[-1].replace('.tif','')\n", + " ax.set_title(f\"AMP_{tag}\", fontsize=10)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " for ax in axes[n:]:\n", + " ax.axis('off')\n", + " fig.colorbar(im, ax=axes.tolist(), orientation='vertical', fraction=0.02, pad=0.02)\n" + ] + }, + { + "cell_type": "markdown", + "id": "df66a59f", + "metadata": {}, + "source": [ + "## 10. Monthly mean coherence calendar (per year)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c31a947a", + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "import pandas as pd\n", + "import numpy as np\n", + "import rioxarray\n", + "import xarray as xr\n", + "import matplotlib.pyplot as plt\n", + "\n", + "\n", + "\n", + "# Build an index of merged coherence files by midpoint year-month\n", + "records = []\n", + "for path in sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\")):\n", + " tag = path.split('merged_coh_')[-1].replace('.tif','')\n", + " try:\n", + " ref_str, sec_str = tag.split('-')\n", + " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", + " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", + " mid_date = ref_date + (sec_date - ref_date) / 2\n", + " except Exception:\n", + " continue\n", + " records.append({\"path\": path, \"mid_date\": mid_date})\n", + "\n", + "df_paths = pd.DataFrame(records)\n", + "if df_paths.empty:\n", + " print('No merged coherence files found for calendar')\n", + " raise SystemExit\n", + "\n", + "# Apply current date range using midpoint date\n", + "date_start_day = dateStart.date()\n", + "date_end_day = dateEnd.date()\n", + "df_paths = df_paths[(df_paths['mid_date'].dt.date >= date_start_day) & (df_paths['mid_date'].dt.date <= date_end_day)]\n", + "\n", + "# Calendar year labeling\n", + "if USE_WATER_YEAR:\n", + " # Water year starts Oct (10) and ends Sep (9)\n", + " df_paths['year'] = df_paths['mid_date'].dt.year + (df_paths['mid_date'].dt.month >= 10).astype(int)\n", + " month_order = [10,11,12,1,2,3,4,5,6,7,8,9]\n", + " month_labels = ['Oct','Nov','Dec','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep']\n", + "else:\n", + " df_paths['year'] = df_paths['mid_date'].dt.year\n", + " month_order = list(range(1,13))\n", + " month_labels = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']\n", + "\n", + "years = sorted(df_paths['year'].unique())\n", + "\n", + "# Contrast stretch for low coherence (red = low)\n", + "# norm = mcolors.PowerNorm(gamma=0.3, vmin=0, vmax=1)\n", + "\n", + "# One row per year, 12 columns\n", + "fig, axes = plt.subplots(len(years), 12, figsize=(24, 2.5*len(years)), constrained_layout=True)\n", + "if len(years) == 1:\n", + " axes = np.array([axes])\n", + "\n", + "for row_idx, y in enumerate(years):\n", + " # pick a template for consistent grid within the year (first available file)\n", + " year_paths = df_paths[df_paths['year'] == y]['path'].tolist()\n", + " if not year_paths:\n", + " continue\n", + " template = rioxarray.open_rasterio(year_paths[0])[0]\n", + "\n", + " for col_idx, m in enumerate(month_order):\n", + " ax = axes[row_idx, col_idx]\n", + " month_paths = df_paths[(df_paths['year'] == y) & (df_paths['mid_date'].dt.month == m)]['path'].tolist()\n", + " if USE_WATER_YEAR:\n", + " year_for_month = y - 1 if m in (10, 11, 12) else y\n", + " else:\n", + " year_for_month = y\n", + " title = f\"{month_labels[col_idx]} {year_for_month}\"\n", + " if not month_paths:\n", + " ax.set_title(title, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " # keep a visible box for empty months\n", + " for spine in ax.spines.values():\n", + " spine.set_visible(True)\n", + " spine.set_linewidth(0.8)\n", + " spine.set_color('0.5')\n", + " continue\n", + " stacks = []\n", + " for p in month_paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " da = da.rio.reproject_match(template)\n", + " stacks.append(da)\n", + " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", + " minlon, minlat, maxlon, maxlat = da_month.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " im = ax.imshow(da_month.values, cmap='gray', vmin=0, vmax=1, origin='upper', extent=bbox, interpolation='none')\n", + " ax.set_title(title, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " for spine in ax.spines.values():\n", + " spine.set_visible(True)\n", + " spine.set_linewidth(0.8)\n", + " spine.set_color('0.5')\n", + " # left-side year label\n", + " if USE_WATER_YEAR:\n", + " label = f\"WY {y}\"\n", + " else:\n", + " label = str(y)\n", + " axes[row_idx, 0].set_ylabel(label, rotation=90, labelpad=6, fontsize=9)\n", + " axes[row_idx, 0].yaxis.set_label_coords(-0.06, 0.5)\n", + "\n", + "fig.colorbar(im, ax=axes, orientation='vertical', fraction=0.02, pad=0.02, label='Mean coherence')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "02e68ad2", + "metadata": {}, + "outputs": [], + "source": [ + "# Debug: list midpoint dates and their counts\n", + "df_paths[['mid_date']].sort_values('mid_date')\n", + "df_paths['mid_date'].dt.to_period('M').value_counts().sort_index()\n" + ] + }, + { + "cell_type": "markdown", + "id": "169216f0", + "metadata": {}, + "source": [ + "## 11. Create GIF animations (Amplitude + Coherence)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "73ba2684", + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "import os\n", + "import numpy as np\n", + "import pandas as pd\n", + "import imageio.v2 as imageio\n", + "import matplotlib.pyplot as plt\n", + "import rioxarray\n", + "import xarray as xr\n", + "\n", + "# Output folders\n", + "gif_dir = f\"{savedir}/gifs\"\n", + "os.makedirs(gif_dir, exist_ok=True)\n", + "\n", + "def _global_bounds(paths):\n", + " bounds = []\n", + " for p in paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " minx, miny, maxx, maxy = da.rio.bounds()\n", + " bounds.append((minx, miny, maxx, maxy))\n", + " minx = min(b[0] for b in bounds)\n", + " miny = min(b[1] for b in bounds)\n", + " maxx = max(b[2] for b in bounds)\n", + " maxy = max(b[3] for b in bounds)\n", + " return [minx, maxx, miny, maxy]\n", + "\n", + "def _render_frames(tif_paths, out_dir, cmap, vmin=None, vmax=None, title_prefix=\"\", extent=None, cbar_label=None, cbar_ticks=None):\n", + " os.makedirs(out_dir, exist_ok=True)\n", + " frames = []\n", + " for p in tif_paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " if extent is None:\n", + " minlon, minlat, maxlon, maxlat = da.rio.bounds()\n", + " extent = [minlon, maxlon, minlat, maxlat]\n", + " fig, ax = plt.subplots(figsize=(6,4))\n", + " im = ax.imshow(da.values, cmap=cmap, origin='upper', extent=extent, vmin=vmin, vmax=vmax)\n", + " tag = os.path.basename(p).replace('.tif','')\n", + " ax.set_title(f\"{title_prefix}{tag}\", fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", + " if cbar_label:\n", + " cb.set_label(cbar_label)\n", + " if cbar_ticks is not None:\n", + " cb.set_ticks(cbar_ticks)\n", + " frame_path = os.path.join(out_dir, f\"{tag}.png\")\n", + " fig.savefig(frame_path, dpi=150)\n", + " plt.close(fig)\n", + " frames.append(frame_path)\n", + " return frames\n", + "\n", + "def _pad_frames(frame_paths):\n", + " imgs = [imageio.imread(f) for f in frame_paths]\n", + " max_h = max(im.shape[0] for im in imgs)\n", + " max_w = max(im.shape[1] for im in imgs)\n", + " padded = []\n", + " for im in imgs:\n", + " pad_h = max_h - im.shape[0]\n", + " pad_w = max_w - im.shape[1]\n", + " padded.append(np.pad(im, ((0, pad_h), (0, pad_w), (0, 0)), mode='edge'))\n", + " return padded\n", + "\n", + "# Amplitude GIF (uses merged amplitude mosaics)\n", + "amp_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_amp_*.tif\"))\n", + "amp_paths = [p for p in amp_paths if 'WGS84' not in p]\n", + "if amp_paths:\n", + " _vals = []\n", + " for p in amp_paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " _vals.append(da.values.ravel())\n", + " _vals = np.concatenate(_vals)\n", + " amp_vmin = np.nanpercentile(_vals, 1)\n", + " amp_vmax = np.nanpercentile(_vals, 99)\n", + " amp_extent = _global_bounds(amp_paths)\n", + " amp_frames = _render_frames(\n", + " amp_paths, f\"{gif_dir}/amp_frames\", cmap=\"gray\", vmin=amp_vmin, vmax=amp_vmax,\n", + " title_prefix=\"AMP_\", extent=amp_extent, cbar_label=\"Amplitude (dB)\"\n", + " )\n", + " amp_gif = f\"{gif_dir}/amplitude.gif\"\n", + " amp_imgs = _pad_frames(amp_frames)\n", + " imageio.mimsave(amp_gif, amp_imgs, duration=0.8)\n", + " print(f\"Wrote {amp_gif}\")\n", + "else:\n", + " print('No merged amplitude files found for GIF')\n", + "\n", + "# Coherence GIF (uses merged coherence mosaics)\n", + "coh_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\"))\n", + "coh_paths = [p for p in coh_paths if 'WGS84' not in p]\n", + "if coh_paths:\n", + " coh_extent = _global_bounds(coh_paths)\n", + " coh_frames = _render_frames(\n", + " coh_paths, f\"{gif_dir}/coh_frames\", cmap='gray', vmin=0, vmax=1,\n", + " title_prefix='COH_', extent=coh_extent, cbar_label='Coherence', cbar_ticks=[0,0.5,1]\n", + " )\n", + " coh_gif = f\"{gif_dir}/coherence.gif\"\n", + " coh_imgs = _pad_frames(coh_frames)\n", + " imageio.mimsave(coh_gif, coh_imgs, duration=0.8)\n", + " print(f\"Wrote {coh_gif}\")\n", + "else:\n", + " print('No merged coherence files found for GIF')\n", + "\n", + "\n", + "# Monthly mean coherence GIF (same monthly averaging as calendar)\n", + "if coh_paths:\n", + " records = []\n", + " for path in sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\")):\n", + " tag = path.split('merged_coh_')[-1].replace('.tif','')\n", + " try:\n", + " ref_str, sec_str = tag.split('-')\n", + " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", + " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", + " mid_date = ref_date + (sec_date - ref_date) / 2\n", + " except Exception:\n", + " continue\n", + " records.append({\"path\": path, \"mid_date\": mid_date})\n", + "\n", + " df_paths = pd.DataFrame(records)\n", + " if df_paths.empty:\n", + " print('No merged coherence files found for monthly GIF')\n", + " else:\n", + " # Apply current date range using midpoint date\n", + " date_start_day = dateStart.date()\n", + " date_end_day = dateEnd.date()\n", + " df_paths = df_paths[(df_paths['mid_date'].dt.date >= date_start_day) & (df_paths['mid_date'].dt.date <= date_end_day)]\n", + "\n", + " # Month/year label logic\n", + " if USE_WATER_YEAR:\n", + " df_paths['year'] = df_paths['mid_date'].dt.year + (df_paths['mid_date'].dt.month >= 10).astype(int)\n", + " month_order = [10,11,12,1,2,3,4,5,6,7,8,9]\n", + " month_labels = ['Oct','Nov','Dec','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep']\n", + " else:\n", + " df_paths['year'] = df_paths['mid_date'].dt.year\n", + " month_order = list(range(1,13))\n", + " month_labels = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']\n", + "\n", + " # Frame output\n", + " monthly_dir = f\"{gif_dir}/coh_monthly_frames\"\n", + " os.makedirs(monthly_dir, exist_ok=True)\n", + " monthly_frames = []\n", + "\n", + " years = sorted(df_paths['year'].unique())\n", + " coh_extent = _global_bounds(coh_paths)\n", + "\n", + " for y in years:\n", + " year_paths = df_paths[df_paths['year'] == y]['path'].tolist()\n", + " if not year_paths:\n", + " continue\n", + " template = rioxarray.open_rasterio(year_paths[0])[0]\n", + "\n", + " for col_idx, m in enumerate(month_order):\n", + " month_paths = df_paths[(df_paths['year'] == y) & (df_paths['mid_date'].dt.month == m)]['path'].tolist()\n", + " if not month_paths:\n", + " continue\n", + "\n", + " stacks = []\n", + " for p in month_paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " da = da.rio.reproject_match(template)\n", + " stacks.append(da)\n", + " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", + "\n", + " if USE_WATER_YEAR:\n", + " year_for_month = y - 1 if m in (10, 11, 12) else y\n", + " else:\n", + " year_for_month = y\n", + " title = f\"{month_labels[col_idx]} {year_for_month}\"\n", + "\n", + " fig, ax = plt.subplots(figsize=(6,4))\n", + " im = ax.imshow(da_month.values, cmap='gray', vmin=0, vmax=1, origin='upper', extent=coh_extent, interpolation='none')\n", + " ax.set_title(title, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", + " cb.set_label('Mean coherence')\n", + " cb.set_ticks([0, 0.5, 1])\n", + " tag = f\"{year_for_month}_{m:02d}\"\n", + " frame_path = os.path.join(monthly_dir, f\"{tag}.png\")\n", + " fig.savefig(frame_path, dpi=150)\n", + " plt.close(fig)\n", + " monthly_frames.append(frame_path)\n", + "\n", + " if monthly_frames:\n", + " monthly_imgs = _pad_frames(monthly_frames)\n", + " monthly_gif = f\"{gif_dir}/coherence_monthly.gif\"\n", + " imageio.mimsave(monthly_gif, monthly_imgs, duration=0.8)\n", + " print(f\"Wrote {monthly_gif}\")\n", + " else:\n", + " print('No monthly coherence frames created (no data in range)')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "615c3b85", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "opera_cslc", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/CSLC/Landslides/environment_opera_cslc_landslides.yml b/CSLC/Landslides/environment_opera_cslc_landslides.yml new file mode 100644 index 0000000..57d3b77 --- /dev/null +++ b/CSLC/Landslides/environment_opera_cslc_landslides.yml @@ -0,0 +1,29 @@ +name: opera_cslc_landslides +channels: + - conda-forge +dependencies: + - python>=3.10 + - asf_search + - affine + - cartopy + - folium + - gdal + - geopandas + - h5py + - imageio + - ipykernel + - jupyterlab + - matplotlib + - numpy + - opera-utils + - pandas + - pyproj + - rasterio + - requests + - rioxarray + - shapely + - tqdm + - xarray + - pip + - pip: + - watermark From 2b5bcce89b1e61358fc50024ca6e855adeac8bb9 Mon Sep 17 00:00:00 2001 From: Al Handwerger Date: Fri, 30 Jan 2026 16:45:01 -0800 Subject: [PATCH 2/7] Refactor example usage in Jupyter notebook --- CSLC/Landslides/CSLC-S1_for_landslides.ipynb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb index c582c3c..abc1731 100644 --- a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb +++ b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb @@ -414,9 +414,8 @@ "\n", " return out_path\n", "\n", - "# Example usage:\n", - "# WATER_MASK_PATH = build_worldcover_water_mask(aoi, f\"{savedir}/water_mask/water_mask_esa_wc2021.tif\")\n", - "# APPLY_WATER_MASK = True\n" + "WATER_MASK_PATH = build_worldcover_water_mask(aoi, f\"{savedir}/water_mask/water_mask_esa_wc2021.tif\")\n", + "APPLY_WATER_MASK = True\n" ] }, { From 41355ac22144710281e660aed2fac47757f86a21 Mon Sep 17 00:00:00 2001 From: Al Handwerger Date: Fri, 30 Jan 2026 16:46:23 -0800 Subject: [PATCH 3/7] Set TARGET_PIXEL_M to 30 in landslides notebook --- CSLC/Landslides/CSLC-S1_for_landslides.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb index abc1731..bec92fb 100644 --- a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb +++ b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb @@ -295,7 +295,7 @@ "# Multilooking (spatial averaging)\n", "# Set either MULTILOOK (looks_y, looks_x) OR TARGET_PIXEL_M (meters). TARGET overrides MULTILOOK.\n", "MULTILOOK = (1, 1) # e.g., (3, 6) for 30m from (dy=10m, dx=5m)\n", - "TARGET_PIXEL_M = None # e.g., 30.0 or 90.0\n", + "TARGET_PIXEL_M = 30 # e.g., 30.0 or 90.0\n", "\n", "REPROJECT_FOR_DISPLAY = False # set True to reproject plots to EPSG:4326\n", "SAVE_WGS84 = False # set True to save WGS84 GeoTIFF mosaics\n", From 3b70f36116a12def1dd4e4516f82b32deb79ad5d Mon Sep 17 00:00:00 2001 From: Al Handwerger Date: Mon, 2 Feb 2026 15:34:23 -0800 Subject: [PATCH 4/7] Delete CSLC/Landslides directory --- CSLC/Landslides/CSLC-S1_for_landslides.ipynb | 2056 ----------------- .../environment_opera_cslc_landslides.yml | 29 - 2 files changed, 2085 deletions(-) delete mode 100644 CSLC/Landslides/CSLC-S1_for_landslides.ipynb delete mode 100644 CSLC/Landslides/environment_opera_cslc_landslides.yml diff --git a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb deleted file mode 100644 index bec92fb..0000000 --- a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb +++ /dev/null @@ -1,2056 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "36315bdf", - "metadata": {}, - "source": [ - "# Generate wrapped interferograms and coherence maps using OPERA CSLC-S1\n", - "\n", - "--- \n", - "\n", - "This notebook:\n", - "- Searches OPERA CSLC-S1 products for your AOI + date range\n", - "- Subsets CSLCs **before download** using opera-utils\n", - "- Builds interferograms/coherence and merges bursts\n", - "- Exports mosaics and visualizations\n", - "\n", - "**Quick start**\n", - "1) Set parameters in the next cell (AOI, date range, pairing)\n", - "2) Run cells top-to-bottom\n", - "3) Outputs land in `savedir/`\n", - "\n", - "**Key toggles**\n", - "- `REPROJECT_FOR_DISPLAY`: reproject plots to WGS84 when True\n", - "- `SAVE_WGS84`: save WGS84 GeoTIFF mosaics when True\n", - "- `DOWNLOAD_WITH_PROGRESS`: show progress bar for downloads\n", - "- `USE_WATER_YEAR`: Oct–Sep calendar layout when True\n", - "- `pair_mode` / `t_span`: control IFG pairing (all vs fixed separation)\n", - "\n", - "**Outputs**\n", - "- Subset CSLC H5: `savedir/subset_cslc/*.h5`\n", - "- Mosaics (IFG/COH, native CRS): `savedir/tifs/merged_ifg_*`, `merged_coh_*`\n", - "- WGS84 mosaics: `savedir/tifs/WGS84/merged_ifg_WGS84_*`, `merged_coh_WGS84_*`, `merged_amp_WGS84_*`\n", - "- Amplitude mosaics (native CRS): `savedir/tifs/merged_amp_*.tif`\n", - "- GIFs: `savedir/gifs/*.gif`\n" - ] - }, - { - "cell_type": "markdown", - "id": "2706d0ca-c490-4a68-bbd7-c3ff1b2adfb9", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Data Used in the Example: \n", - "\n", - "- **10 meter (Northing) x 5 meter (Easting) North America OPERA Coregistered Single Look Complex from Sentinel-1 products**\n", - " - This dataset contains Level-2 OPERA coregistered single-look-complex (CSLC) data from Sentinel-1 (S1). The data in this example are geocoded CSLC-S1 data covering Palos Verdes landslides, California, USA. \n", - " \n", - " - The OPERA project is generating geocoded burst-wise CSLC-S1 products over North America which includes USA and US Territories within 200 km from the US border, Canada, and all mainland countries from the southern US border down to and including Panama. Each pixel within a burst SLC is represented by a complex number and contains both the amplitude and phase information. The CSLC-S1 products are distributed over projected map coordinates using the Universal Transverse Mercator (UTM) projection with spacing in the X- and Y-directions of 5 m and 10 m, respectively. Each OPERA CSLC-S1 product is distributed as a HDF5 file following the CF-1.8 convention with separate groups containing the data raster layers, the low-resolution correction layers, and relevant product metadata.\n", - "\n", - " - For more information about the OPERA project and other products please visit our website at https://www.jpl.nasa.gov/go/opera .\n", - "\n", - "Please refer to the [OPERA Product Specification Document](https://d2pn8kiwq2w21t.cloudfront.net/documents/OPERA_CSLC-S1_ProductSpec_v1.0.0_D-108278_Initial_2023-09-11_URS321269.pdf) for details about the CSLC-S1 product.\n", - "\n", - "*Prepared by Al Handwerger and M. Grace Bato*\n", - "\n", - "---" - ] - }, - { - "cell_type": "markdown", - "id": "9b2e135b", - "metadata": {}, - "source": [ - "## 0. Setup your conda environment\n", - "\n", - "Assuming you have conda installed. Open your terminal and run the following:\n", - "```\n", - "\n", - "# Create the OPERA CSLC environment\n", - "conda env create -f environment_opera_cslc_landslides.yml\n", - "conda activate opera_cslc_landslides\n", - "python -m ipykernel install --user --name opera_cslc_landslides\n", - "\n", - "```\n", - "\n", - "--- " - ] - }, - { - "cell_type": "markdown", - "id": "e0d4309e", - "metadata": {}, - "source": [ - "## 1. Load Python modules" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "84a1468d-0aaf-4f06-9875-6b753d94ad42", - "metadata": {}, - "outputs": [], - "source": [ - "## Load necessary modules\n", - "%load_ext watermark\n", - "\n", - "import asf_search as asf\n", - "import geopandas as gpd\n", - "import pandas as pd\n", - "\n", - "import numpy as np\n", - "from netrc import netrc\n", - "from subprocess import Popen\n", - "from platform import system\n", - "from getpass import getpass\n", - "import folium\n", - "import datetime as dt\n", - "from shapely.geometry import box\n", - "from shapely.geometry import Point\n", - "import shapely.wkt as wkt\n", - "import rioxarray\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib.patches as patches\n", - "import cartopy.crs as ccrs\n", - "import xarray as xr\n", - "import rasterio\n", - "from rasterio.transform import from_origin\n", - "from rasterio import merge\n", - "from rasterio.crs import CRS\n", - "\n", - "import os, sys\n", - "proj_dir = os.path.join(sys.prefix, \"share\", \"proj\")\n", - "os.environ[\"PROJ_LIB\"] = proj_dir # for older PROJ\n", - "os.environ[\"PROJ_DATA\"] = proj_dir # for newer PROJ\n", - "\n", - "\n", - "%watermark --iversions\n", - "\n", - "import os" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9952139c", - "metadata": {}, - "outputs": [], - "source": [ - "# Environment check\n", - "import sys\n", - "import importlib\n", - "\n", - "REQUIRED_PKGS = [\n", - " 'asf_search','cartopy','folium','geopandas','h5py','imageio','matplotlib','numpy','pandas',\n", - " 'pyproj','rasterio','rioxarray','shapely','xarray','opera_utils','tqdm','rich'\n", - "]\n", - "missing = []\n", - "for pkg in REQUIRED_PKGS:\n", - " try:\n", - " importlib.import_module(pkg)\n", - " except Exception:\n", - " missing.append(pkg)\n", - "\n", - "if missing:\n", - " raise ImportError(\"Missing packages: \" + ', '.join(missing) + \". Activate opera_cslc env or install from environment_opera_cslc.yml\")\n", - "\n", - "print(f\"Python: {sys.executable}\")\n", - "print('Environment check OK')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1faa1ef0-3d5e-424e-b602-3392b720af6d", - "metadata": {}, - "outputs": [], - "source": [ - "## Load plotting module\n", - "import matplotlib.pyplot as plt\n", - "%matplotlib inline\n", - "%config InlineBackend.figure_format='retina'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "49952732", - "metadata": {}, - "outputs": [], - "source": [ - "## Load pandas and setup config to expand the display of the database\n", - "import pandas as pd\n", - "# pd.set_option('display.max_rows', None)\n", - "pd.set_option('display.max_columns', None)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "137538f3", - "metadata": {}, - "outputs": [], - "source": [ - "def _maybe_reproject(da):\n", - " if REPROJECT_FOR_DISPLAY:\n", - " return da.rio.reproject(\"EPSG:4326\")\n", - " return da\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c30a7294-742e-4ac6-bc9b-82cc493e1e6c", - "metadata": {}, - "outputs": [], - "source": [ - "## Avoid lots of these warnings printing to notebook from asf_search\n", - "import warnings\n", - "warnings.filterwarnings('ignore')" - ] - }, - { - "cell_type": "markdown", - "id": "32ef4ad4", - "metadata": {}, - "source": [ - "## 2. Set up your NASA Earthdata Login Credentials" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0b6ad5ac-9089-4377-be63-ddf1731263f2", - "metadata": {}, - "outputs": [], - "source": [ - "urs = 'urs.earthdata.nasa.gov'\n", - "prompts = ['Enter NASA Earthdata Login Username: ',\n", - " 'Enter NASA Earthdata Login Password: ']\n", - "\n", - "netrc_name = \"_netrc\" if system() == \"Windows\" else \".netrc\"\n", - "netrc_path = os.path.expanduser(f\"~/{netrc_name}\")\n", - "\n", - "def write_netrc():\n", - " username = getpass(prompt=prompts[0])\n", - " password = getpass(prompt=prompts[1])\n", - " with open(netrc_path, 'a') as f:\n", - " f.write(f\"\\nmachine {urs}\\n\")\n", - " f.write(f\"login {username}\\n\")\n", - " f.write(f\"password {password}\\n\")\n", - " os.chmod(netrc_path, 0o600)\n", - "\n", - "def has_urs_credentials():\n", - " try:\n", - " creds = netrc(netrc_path).authenticators(urs)\n", - " return creds is not None\n", - " except (FileNotFoundError, NetrcParseError):\n", - " return False\n", - "\n", - "if not has_urs_credentials():\n", - " if not os.path.exists(netrc_path):\n", - " open(netrc_path, 'w').close()\n", - " write_netrc()\n", - "\n", - "import os\n", - "os.environ[\"GDAL_HTTP_NETRC\"] = \"YES\"\n", - "os.environ[\"GDAL_HTTP_NETRC_FILE\"] = netrc_path\n" - ] - }, - { - "cell_type": "markdown", - "id": "18311b89", - "metadata": {}, - "source": [ - "## 3. Enter user-defined parameters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d5eeba6e", - "metadata": {}, - "outputs": [], - "source": [ - "# User parameters (edit these)\n", - "# AOI is a WKT polygon in EPSG:4326\n", - "## Enter user-defined parameters\n", - "SITE_NAME = \"Palos_Verdes_Landslides\" # used for output folder naming\n", - "aoi = \"POLYGON((-118.3955 33.7342,-118.3464 33.7342,-118.3464 33.7616,-118.3955 33.7616,-118.3955 33.7342))\"\n", - "orbitPass = \"DESCENDING\"\n", - "pathNumber = 71\n", - "# Optional burst selection before download\n", - "# Use subswath (e.g., 'IW2', 'IW3') or specific OPERA burst ID (e.g., 'T071_151230_IW3')\n", - "BURST_SUBSWATH = 'IW3' # e.g., 'IW2' or ['IW2', 'IW3'] or None \n", - "BURST_ID = None # e.g., 'T071_151230_IW3' or list of burst IDs\n", - "dateStart = dt.datetime.fromisoformat('2017-10-01 00:00:00') #'YYYY-MM-DD HH:MM:SS'\n", - "dateEnd = dt.datetime.fromisoformat('2017-12-01 23:59:59') #'YYYY-MM-DD HH:MM:SS'\n", - "\n", - "# Pairing options\n", - "pair_mode = 't_span' # 'all' or 't_span'\n", - "pair_t_span_days = 12 # int or list of ints (e.g., [12, 24])\n", - "\n", - "# Multilooking (spatial averaging)\n", - "# Set either MULTILOOK (looks_y, looks_x) OR TARGET_PIXEL_M (meters). TARGET overrides MULTILOOK.\n", - "MULTILOOK = (1, 1) # e.g., (3, 6) for 30m from (dy=10m, dx=5m)\n", - "TARGET_PIXEL_M = 30 # e.g., 30.0 or 90.0\n", - "\n", - "REPROJECT_FOR_DISPLAY = False # set True to reproject plots to EPSG:4326\n", - "SAVE_WGS84 = False # set True to save WGS84 GeoTIFF mosaics\n", - "\n", - "\n", - "\n", - "DOWNLOAD_WITH_PROGRESS = True # set True for per-file progress bar\n", - "\n", - "import os\n", - "\n", - "min_t_span_days = 12 # minimum separation (days)\n", - "max_t_span_days = 12 # maximum separation (days) or None\n", - "max_pairs_per_burst = None # int or None\n", - "max_pairs_total = None # int or None\n", - "\n", - "# Calendar settings\n", - "USE_WATER_YEAR = True # True: Oct–Sep, False: Jan–Dec\n", - "\n", - "DOWNLOAD_PROCESSES = min(8, max(2, (os.cpu_count() or 4) // 2))\n", - "# DOWNLOAD_BATCH_SIZE = 5\n", - "\n", - "\n", - "# Normalize name for filesystem (letters/numbers/_/- only)\n", - "import re\n", - "site_slug = re.sub(r\"[^A-Za-z0-9_-]+\", \"\", SITE_NAME)\n", - "orbit_code = orbitPass[0].upper() # 'A' or 'D'\n", - "savedir = f'./{site_slug}_{orbit_code}{pathNumber:03d}/'\n", - "\n", - "# Water mask options\n", - "# in params cell\n", - "WATER_MASK_PATH = f\"{savedir}/water_mask/water_mask_esa_wc2021.tif\"\n", - "APPLY_WATER_MASK = True\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6f072529", - "metadata": {}, - "outputs": [], - "source": [ - "# ESA WorldCover 2021 water mask (GDAL-only)\n", - "from pathlib import Path\n", - "import geopandas as gpd\n", - "import shapely.wkt\n", - "import rasterio\n", - "from osgeo import gdal\n", - "\n", - "ESA_WC_GRID_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/esa_worldcover_grid.fgb\"\n", - "ESA_WC_BASE_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/v200/2021/map\"\n", - "# ESA WorldCover class codes: 80 = Permanent water bodies\n", - "ESA_WC_WATER_CLASSES = {80}\n", - "\n", - "\n", - "def build_worldcover_water_mask(aoi_wkt, out_path, target_res_deg=None):\n", - " # Create a binary land mask from ESA WorldCover (1=land, 0=water).\n", - " out_path = Path(out_path)\n", - " out_path.parent.mkdir(parents=True, exist_ok=True)\n", - " if out_path.exists():\n", - " return out_path\n", - "\n", - "\n", - " aoi_geom = shapely.wkt.loads(aoi_wkt)\n", - " # Load tile grid and select intersecting tiles\n", - " grid = gpd.read_file(ESA_WC_GRID_URL)\n", - " grid = grid.to_crs(\"EPSG:4326\")\n", - " # Find tile id column (varies by grid version)\n", - " tile_col = next((c for c in grid.columns if 'tile' in c.lower()), None)\n", - " if tile_col is None:\n", - " raise RuntimeError(f\"No tile column found in grid columns: {list(grid.columns)}\")\n", - " tiles = grid[grid.intersects(aoi_geom)][tile_col].tolist()\n", - " if not tiles:\n", - " raise RuntimeError(\"No WorldCover tiles intersect AOI\")\n", - "\n", - " print(f\"Selected tiles: {tiles}\")\n", - "\n", - " tile_urls = [\n", - " f\"{ESA_WC_BASE_URL}/ESA_WorldCover_10m_2021_v200_{t}_Map.tif\"\n", - " for t in tiles\n", - " ]\n", - "\n", - " # Quick URL check for first tile\n", - " first_url = tile_urls[0]\n", - " try:\n", - " _ = gdal.Open(first_url)\n", - " except Exception as e:\n", - " raise RuntimeError(f\"GDAL cannot open first tile URL: {first_url}\\n{e}\")\n", - "\n", - " vrt_path = out_path.with_suffix(\".vrt\")\n", - " gdal.BuildVRT(str(vrt_path), tile_urls)\n", - "\n", - " minx, miny, maxx, maxy = aoi_geom.bounds\n", - " warp_kwargs = dict(\n", - " format=\"GTiff\",\n", - " outputBounds=[minx, miny, maxx, maxy],\n", - " multithread=True,\n", - " )\n", - " if target_res_deg is not None:\n", - " warp_kwargs.update(dict(xRes=target_res_deg, yRes=target_res_deg, targetAlignedPixels=True))\n", - "\n", - " tmp_map = out_path.with_name(out_path.stem + \"_map.tif\")\n", - " warp_ds = gdal.Warp(str(tmp_map), str(vrt_path), **warp_kwargs)\n", - " if warp_ds is None:\n", - " raise RuntimeError(\"GDAL Warp returned None. Check network access/URL.\")\n", - " warp_ds = None\n", - "\n", - " with rasterio.open(tmp_map) as src:\n", - " data = src.read(1)\n", - " profile = src.profile\n", - "\n", - " mask = (~np.isin(data, list(ESA_WC_WATER_CLASSES))).astype(\"uint8\")\n", - " profile.update(dtype=\"uint8\", count=1, nodata=0)\n", - "\n", - " with rasterio.open(out_path, \"w\", **profile) as dst:\n", - " dst.write(mask, 1)\n", - "\n", - " return out_path\n", - "\n", - "WATER_MASK_PATH = build_worldcover_water_mask(aoi, f\"{savedir}/water_mask/water_mask_esa_wc2021.tif\")\n", - "APPLY_WATER_MASK = True\n" - ] - }, - { - "cell_type": "markdown", - "id": "98916bef", - "metadata": {}, - "source": [ - "## 4. Query OPERA CSLCs using `asf_search`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5e33d72e-2e75-4099-93d6-5cd058643e35", - "metadata": {}, - "outputs": [], - "source": [ - "## Search for OPERA CSLC data in ASF DAAC\n", - "try:\n", - " search_params = dict(\n", - " intersectsWith= aoi,\n", - " dataset='OPERA-S1',\n", - " processingLevel='CSLC',\n", - " flightDirection = orbitPass,\n", - " start=dateStart,\n", - " end=dateEnd)\n", - "\n", - " ## Return results\n", - " results = asf.search(**search_params)\n", - " print(f\"Length of Results: {len(results)}\")\n", - "\n", - "except TypeError:\n", - " search_params = dict(\n", - " intersectsWith= aoi.wkt,\n", - " dataset='OPERA-S1',\n", - " processingLevel='CSLC',\n", - " flightDirection = orbitPass,\n", - " start=dateStart,\n", - " end=dateEnd)\n", - "\n", - " ## Return results\n", - " results = asf.search(**search_params)\n", - " print(f\"Length of Results: {len(results)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b2f5b5df", - "metadata": {}, - "outputs": [], - "source": [ - "## Save the results in a geopandas dataframe\n", - "gf = gpd.GeoDataFrame.from_features(results.geojson(), crs='EPSG:4326')\n", - "\n", - "## Filter data based on specified track number\n", - "gf = gf[gf.pathNumber==pathNumber]\n", - "# gf = gf[gf.pgeVersion==\"2.1.1\"] \n", - "gf" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fe3a0963-1d13-4b99-955a-b2c0fa45f1e3", - "metadata": {}, - "outputs": [], - "source": [ - "# Get only relevant metadata\n", - "cslc_df = gf[['operaBurstID', 'fileID', 'startTime', 'stopTime', 'url', 'geometry', 'pgeVersion']]\n", - "cslc_df['startTime'] = pd.to_datetime(cslc_df.startTime).dt.date\n", - "cslc_df['stopTime'] = pd.to_datetime(cslc_df.stopTime).dt.date\n", - "\n", - "# Extract production time from fileID (2nd date token)\n", - "def _prod_time_from_fileid(file_id):\n", - " # Example: OPERA_L2_CSLC-S1_..._20221122T161650Z_20240504T081640Z_...\n", - " parts = str(file_id).split('_')\n", - " return parts[5] if len(parts) > 5 else None\n", - "\n", - "cslc_df['productionTime'] = pd.to_datetime(cslc_df['fileID'].apply(_prod_time_from_fileid), format='%Y%m%dT%H%M%SZ', errors='coerce')\n", - "\n", - "# Keep newest duplicate by productionTime (fallback to pgeVersion, stopTime)\n", - "cslc_df = cslc_df.sort_values(by=['operaBurstID', 'startTime', 'productionTime', 'pgeVersion', 'stopTime'])\n", - "cslc_df = cslc_df.drop_duplicates(subset=['operaBurstID', 'startTime'], keep='last', ignore_index=True)\n", - "\n", - "import re\n", - "\n", - "def _subswath_from_fileid(file_id):\n", - " # Example: ...-IW2_... -> IW2\n", - " m = re.search(r\"-IW[1-3]_\", str(file_id))\n", - " return m.group(0)[1:4] if m else None\n", - "\n", - "cslc_df['burstSubswath'] = cslc_df['fileID'].apply(_subswath_from_fileid)\n", - "\n", - "# Optional filtering by subswath or specific burst IDs\n", - "if BURST_SUBSWATH:\n", - " if isinstance(BURST_SUBSWATH, (list, tuple, set)):\n", - " subswaths = {str(s).upper() for s in BURST_SUBSWATH}\n", - " else:\n", - " subswaths = {str(BURST_SUBSWATH).upper()}\n", - " cslc_df = cslc_df[cslc_df['burstSubswath'].str.upper().isin(subswaths)]\n", - "\n", - "if BURST_ID:\n", - " if isinstance(BURST_ID, (list, tuple, set)):\n", - " burst_ids = {str(b) for b in BURST_ID}\n", - " else:\n", - " burst_ids = {str(BURST_ID)}\n", - " cslc_df = cslc_df[cslc_df['operaBurstID'].isin(burst_ids)]\n", - "cslc_df\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "76cb1007", - "metadata": {}, - "outputs": [], - "source": [ - "import shapely.wkt as wkt\n", - "import geopandas as gpd\n", - "\n", - "aoi_geom = wkt.loads(aoi)\n", - "aoi_gdf = gpd.GeoDataFrame(geometry=[aoi_geom], crs=\"EPSG:4326\")\n", - "\n", - "m = cslc_df[['operaBurstID', 'geometry']].explore(\n", - " zoom=9,\n", - " tiles=\"Esri.WorldImagery\",\n", - " style_kwds={\"fill\": True, \"fillColor\": \"blue\", \"fillOpacity\": 0.1, \"weight\": 2},\n", - " name=\"CSLC footprints\",\n", - ")\n", - "\n", - "aoi_gdf.explore(\n", - " m=m,\n", - " color=\"red\",\n", - " style_kwds={\"fill\": True, \"fillColor\": \"blue\", \"fillOpacity\": 0.1, \"weight\": 2},\n", - " name=\"AOI\",\n", - ")\n", - "\n", - "m\n" - ] - }, - { - "cell_type": "markdown", - "id": "9ca8a611", - "metadata": {}, - "source": [ - "## 5. Download the CSLC-S1 locally" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c0cde1fc", - "metadata": {}, - "outputs": [], - "source": [ - "## Download step skipped: CSLC subsets are streamed via opera-utils\n", - "print('Skipping full CSLC downloads; using opera-utils HTTP subsetting.')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aea6a5b7", - "metadata": {}, - "outputs": [], - "source": [ - "# Sort the CSLC-S1 by burstID and date\n", - "cslc_df = cslc_df.sort_values(by=[\"operaBurstID\", \"startTime\"], ignore_index=True)\n", - "cslc_df\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa4801fc", - "metadata": {}, - "outputs": [], - "source": [ - "# Enforce date range on dataframe (useful when re-running with narrower dates)\n", - "date_start_day = dateStart.date()\n", - "date_end_day = dateEnd.date()\n", - "cslc_df = cslc_df[(cslc_df['startTime'] >= date_start_day) & (cslc_df['startTime'] <= date_end_day)]\n", - "cslc_df = cslc_df.reset_index(drop=True)\n", - "cslc_df\n" - ] - }, - { - "cell_type": "markdown", - "id": "48add9c3", - "metadata": {}, - "source": [ - "## 6. Read each CSLC-S1 and stack them together\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "01891a96", - "metadata": {}, - "outputs": [], - "source": [ - "cslc_stack = []; cslc_dates = []; bbox_stack = []; xcoor_stack = []; ycoor_stack = []\n", - "\n", - "import os\n", - "import time\n", - "import numpy as np\n", - "import tempfile\n", - "import requests\n", - "import xarray as xr\n", - "import h5py\n", - "from tqdm.auto import tqdm\n", - "from pathlib import Path\n", - "from pyproj import Transformer\n", - "from shapely import wkt\n", - "from shapely.ops import transform as shp_transform\n", - "from opera_utils.credentials import get_earthdata_username_password\n", - "from opera_utils.disp._remote import open_file\n", - "from opera_utils.disp._utils import _get_netcdf_encoding\n", - "\n", - "subset_dir = f\"{savedir}/subset_cslc\"\n", - "os.makedirs(subset_dir, exist_ok=True)\n", - "\n", - "def _extract_subset(input_obj, outpath, rows, cols, chunks=(1,256,256)):\n", - " X0, X1 = (cols.start, cols.stop) if cols is not None else (None, None)\n", - " Y0, Y1 = (rows.start, rows.stop) if rows is not None else (None, None)\n", - " ds = xr.open_dataset(input_obj, engine=\"h5netcdf\", group=\"data\")\n", - " subset = ds.isel(y_coordinates=slice(Y0, Y1), x_coordinates=slice(X0, X1))\n", - " subset.to_netcdf(\n", - " outpath,\n", - " engine=\"h5netcdf\",\n", - " group=\"data\",\n", - " encoding=_get_netcdf_encoding(subset, chunks=chunks),\n", - " )\n", - " for group in (\"metadata\", \"identification\"):\n", - " with h5py.File(input_obj) as hf, h5py.File(outpath, \"a\") as dest_hf:\n", - " hf.copy(group, dest_hf, name=group)\n", - " with h5py.File(outpath, \"a\") as hf:\n", - " ctype = h5py.h5t.py_create(np.complex64)\n", - " ctype.commit(hf[\"/\"].id, np.bytes_(\"complex64\"))\n", - "\n", - "def _subset_h5_to_disk(url, aoi_wkt, out_dir):\n", - " outpath = Path(out_dir) / Path(url).name\n", - " if outpath.exists():\n", - " return outpath\n", - "\n", - " # determine row/col slices by reading coords\n", - " with open_file(url) as in_f:\n", - " ds = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", - " xcoor = ds[\"x_coordinates\"].values\n", - " ycoor = ds[\"y_coordinates\"].values\n", - " epsg = int(ds[\"projection\"].values)\n", - "\n", - " aoi_geom = wkt.loads(aoi_wkt)\n", - " if epsg != 4326:\n", - " transformer = Transformer.from_crs('EPSG:4326', f'EPSG:{epsg}', always_xy=True)\n", - " aoi_geom = shp_transform(transformer.transform, aoi_geom)\n", - " minx, miny, maxx, maxy = aoi_geom.bounds\n", - " x_mask = (xcoor >= minx) & (xcoor <= maxx)\n", - " y_mask = (ycoor >= miny) & (ycoor <= maxy)\n", - " if not x_mask.any() or not y_mask.any():\n", - " raise ValueError('AOI does not intersect this CSLC extent')\n", - " ix = np.where(x_mask)[0]\n", - " iy = np.where(y_mask)[0]\n", - " rows = slice(iy.min(), iy.max()+1)\n", - " cols = slice(ix.min(), ix.max()+1)\n", - "\n", - " if url.startswith('s3://'):\n", - " with open_file(url) as in_f:\n", - " _extract_subset(in_f, outpath, rows, cols)\n", - " else:\n", - " # HTTPS: download to temp then subset\n", - " with tempfile.NamedTemporaryFile(suffix='.h5') as tf:\n", - " if url.startswith('http'):\n", - " session = requests.Session()\n", - " username, password = get_earthdata_username_password()\n", - " session.auth = (username, password)\n", - " resp = session.get(url)\n", - " resp.raise_for_status()\n", - " tf.write(resp.content)\n", - " tf.flush()\n", - " _extract_subset(tf.name, outpath, rows, cols)\n", - " return outpath\n", - "\n", - "def _load_subset(file_id, url, start_date):\n", - " outpath = _subset_h5_to_disk(url, aoi, subset_dir)\n", - " # now read subset locally with h5py (fast)\n", - " with h5py.File(outpath, 'r') as h5:\n", - " cslc = h5['/data/VV'][:]\n", - " xcoor = h5['/data/x_coordinates'][:]\n", - " ycoor = h5['/data/y_coordinates'][:]\n", - " dx = int(h5['/data/x_spacing'][()])\n", - " dy = int(h5['/data/y_spacing'][()])\n", - " epsg = int(h5['/data/projection'][()])\n", - " sensing_start = h5['/metadata/processing_information/input_burst_metadata/sensing_start'][()].astype(str)\n", - " sensing_stop = h5['/metadata/processing_information/input_burst_metadata/sensing_stop'][()].astype(str)\n", - " dims = h5['/metadata/processing_information/input_burst_metadata/shape'][:]\n", - " bounding_polygon = h5['/identification/bounding_polygon'][()].astype(str)\n", - " orbit_direction = h5['/identification/orbit_pass_direction'][()].astype(str)\n", - " center_lon, center_lat = h5['/metadata/processing_information/input_burst_metadata/center']\n", - " wavelength = h5['/metadata/processing_information/input_burst_metadata/wavelength'][()].astype(str)\n", - " subset_bbox = [float(xcoor.min()), float(xcoor.max()), float(ycoor.min()), float(ycoor.max())]\n", - " return cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox\n", - "\n", - "# Subset with progress (parallel)\n", - "from concurrent.futures import ThreadPoolExecutor, as_completed\n", - "\n", - "items = list(zip(cslc_df.fileID, cslc_df.url, cslc_df.startTime))\n", - "# Diagnostic: check pixel spacing before multilooking\n", - "with open_file(items[0][1]) as in_f:\n", - " ds0 = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", - " dx0 = float(ds0[\"x_spacing\"].values)\n", - " dy0 = float(ds0[\"y_spacing\"].values)\n", - "print(f\"Pixel spacing (dx, dy) = ({dx0}, {dy0})\")\n", - "\n", - "results = [None] * len(items)\n", - "_t0 = time.perf_counter()\n", - "with ThreadPoolExecutor(max_workers=DOWNLOAD_PROCESSES) as ex:\n", - " futures = {ex.submit(_load_subset, fileID, url, start_date): i for i, (fileID, url, start_date) in enumerate(items)}\n", - " for fut in tqdm(as_completed(futures), total=len(futures), desc='Subsetting CSLC'):\n", - " i = futures[fut]\n", - " results[i] = fut.result()\n", - "_t1 = time.perf_counter()\n", - "print(f\"Subset/download time: {_t1 - _t0:.1f} s\")\n", - "\n", - "for (fileID, start_date), res in zip(zip(cslc_df.fileID, cslc_df.startTime), results):\n", - " cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox = res\n", - " cslc_stack.append(cslc)\n", - " cslc_dates.append(pd.to_datetime(sensing_start).date())\n", - " if subset_bbox is not None:\n", - " bbox = subset_bbox\n", - " else:\n", - " cslc_poly = wkt.loads(bounding_polygon)\n", - " bbox = [cslc_poly.bounds[0], cslc_poly.bounds[2], cslc_poly.bounds[1], cslc_poly.bounds[3]]\n", - " bbox_stack.append(bbox)\n", - " xcoor_stack.append(xcoor)\n", - " ycoor_stack.append(ycoor)\n" - ] - }, - { - "cell_type": "markdown", - "id": "f5e97e45", - "metadata": {}, - "source": [ - "## 7. Generate the interferograms, compute for the coherence, save the files as GeoTiffs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57c74ac4", - "metadata": {}, - "outputs": [], - "source": [ - "def colorize(array=[], cmap='RdBu', cmin=[], cmax=[]):\n", - " normed_data = (array - cmin) / (cmax - cmin) \n", - " cm = plt.cm.get_cmap(cmap)\n", - " return cm(normed_data) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25c56034", - "metadata": {}, - "outputs": [], - "source": [ - "def goldstein_filter(ifg_cpx, alpha=0.5, pad=32, edge_trim=16):\n", - " # Goldstein filter with padding + taper + mask to reduce edge effects\n", - " mask = np.isfinite(ifg_cpx)\n", - " data = np.nan_to_num(ifg_cpx, nan=0.0)\n", - " if pad and pad > 0:\n", - " data = np.pad(data, ((pad, pad), (pad, pad)), mode=\"reflect\")\n", - " mask = np.pad(mask, ((pad, pad), (pad, pad)), mode=\"constant\", constant_values=False)\n", - " # Apply 2D Hann window (taper)\n", - " wy = np.hanning(data.shape[0])\n", - " wx = np.hanning(data.shape[1])\n", - " window = wy[:, None] * wx[None, :]\n", - " f = np.fft.fft2(data * window)\n", - " s = np.abs(f)\n", - " s = s / (s.max() + 1e-8)\n", - " f_filt = f * (s ** alpha)\n", - " out = np.fft.ifft2(f_filt)\n", - " if pad and pad > 0:\n", - " out = out[pad:-pad, pad:-pad]\n", - " mask = mask[pad:-pad, pad:-pad]\n", - " # restore NaNs outside valid mask\n", - " out[~mask] = np.nan\n", - " if edge_trim and edge_trim > 0:\n", - " out[:edge_trim, :] = np.nan\n", - " out[-edge_trim:, :] = np.nan\n", - " out[:, :edge_trim] = np.nan\n", - " out[:, -edge_trim:] = np.nan\n", - " return out\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82bda5ae", - "metadata": {}, - "outputs": [], - "source": [ - "def rasterWrite(outtif,arr,transform,epsg,dtype='float32'):\n", - " #writing geotiff using rasterio\n", - " \n", - " new_dataset = rasterio.open(outtif, 'w', driver='GTiff',\n", - " height = arr.shape[0], width = arr.shape[1],\n", - " count=1, dtype=dtype,\n", - " crs=CRS.from_epsg(epsg),\n", - " transform=transform,nodata=np.nan)\n", - " new_dataset.write(arr, 1)\n", - " new_dataset.close() " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b47f82a5", - "metadata": {}, - "outputs": [], - "source": [ - "## Build date pairs per burstID\n", - "cslc_dates = cslc_df[[\"startTime\"]]\n", - "burstID = cslc_df.operaBurstID.drop_duplicates(ignore_index=True)\n", - "n_unique_burstID = len(burstID)\n", - "\n", - "def _lag_list(lag):\n", - " if lag is None:\n", - " return []\n", - " if isinstance(lag, (list, tuple, set)):\n", - " return sorted({int(x) for x in lag})\n", - " return [int(lag)]\n", - "\n", - "pair_lags = _lag_list(pair_t_span_days)\n", - "pair_indices = [] # list of (ref_idx, sec_idx) in cslc_df order\n", - "\n", - "for bid, group in cslc_df.groupby('operaBurstID'):\n", - " group = group.sort_values('startTime')\n", - " idx = group.index.to_list()\n", - " dates = group['startTime'].to_list()\n", - "\n", - " burst_pairs = []\n", - " if pair_mode == 'all':\n", - " for i in range(len(idx)):\n", - " for j in range(i+1, len(idx)):\n", - " delta = (dates[j] - dates[i]).days\n", - " if delta < min_t_span_days:\n", - " continue\n", - " if max_t_span_days is not None and delta > max_t_span_days:\n", - " continue\n", - " burst_pairs.append((idx[i], idx[j]))\n", - " elif pair_mode == 't_span':\n", - " for i in range(len(idx)):\n", - " for j in range(i+1, len(idx)):\n", - " delta = (dates[j] - dates[i]).days\n", - " if delta in pair_lags and delta >= min_t_span_days and (max_t_span_days is None or delta <= max_t_span_days):\n", - " burst_pairs.append((idx[i], idx[j]))\n", - " else:\n", - " raise ValueError(\"pair_mode must be 'all' or 't_span'\")\n", - "\n", - " if max_pairs_per_burst is not None:\n", - " burst_pairs = burst_pairs[:int(max_pairs_per_burst)]\n", - "\n", - " pair_indices.extend(burst_pairs)\n", - "\n", - "if max_pairs_total is not None:\n", - " pair_indices = pair_indices[:int(max_pairs_total)]\n", - "\n", - "# Sort pairs by date, then burstID (so same dates group together)\n", - "def _pair_sort_key(pair):\n", - " ref_idx, sec_idx = pair\n", - " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", - " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", - " burst = cslc_df.operaBurstID.iloc[ref_idx]\n", - " return (ref_date, sec_date, burst)\n", - "pair_indices = sorted(pair_indices, key=_pair_sort_key)\n", - "print(f'Pair count: {len(pair_indices)}')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6852f778", - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "import warnings\n", - "from numpy.lib.stride_tricks import sliding_window_view\n", - "\n", - "\n", - "\n", - "\n", - "def take_looks(arr, row_looks, col_looks, func_type=\"nanmean\", edge_strategy=\"cutoff\"):\n", - " if row_looks == 1 and col_looks == 1:\n", - " return arr\n", - " if arr.ndim != 2:\n", - " raise ValueError(\"take_looks expects 2D array\")\n", - " rows, cols = arr.shape\n", - " if edge_strategy == \"cutoff\":\n", - " rows = (rows // row_looks) * row_looks\n", - " cols = (cols // col_looks) * col_looks\n", - " arr = arr[:rows, :cols]\n", - " elif edge_strategy == \"pad\":\n", - " pad_r = (-rows) % row_looks\n", - " pad_c = (-cols) % col_looks\n", - " if pad_r or pad_c:\n", - " arr = np.pad(arr, ((0, pad_r), (0, pad_c)), mode=\"constant\", constant_values=np.nan)\n", - " rows, cols = arr.shape\n", - " else:\n", - " raise ValueError(\"edge_strategy must be 'cutoff' or 'pad'\")\n", - "\n", - " new_rows = rows // row_looks\n", - " new_cols = cols // col_looks\n", - " func = getattr(np, func_type)\n", - " with warnings.catch_warnings():\n", - " warnings.simplefilter(\"ignore\", category=RuntimeWarning)\n", - " return func(arr.reshape(new_rows, row_looks, new_cols, col_looks), axis=(1, 3))\n", - "\n", - "\n", - "def _multilook(arr, looks_y=1, looks_x=1):\n", - " return take_looks(arr, looks_y, looks_x, func_type=\"nanmean\", edge_strategy=\"cutoff\")\n", - "\n", - "def _box_mean(arr, win):\n", - " pad = win // 2\n", - " arr_p = np.pad(arr, pad_width=pad, mode='reflect')\n", - " windows = sliding_window_view(arr_p, (win, win))\n", - " return windows.mean(axis=(-2, -1))\n", - "\n", - " return _box_mean(arr, win)\n", - "\n", - "def lee_filter(img, win=5):\n", - " mean = _box_mean(img, win)\n", - " mean_sq = _box_mean(img**2, win)\n", - " var = mean_sq - mean**2\n", - " noise_var = np.nanmedian(var)\n", - " w = var / (var + noise_var + 1e-8)\n", - " return mean + w * (img - mean)\n", - "\n", - "def goldstein(phase, alpha, psize=32):\n", - " \"\"\"Apply the Goldstein adaptive filter to the given data.\"\"\"\n", - " def apply_pspec(data):\n", - " if alpha < 0:\n", - " raise ValueError(f\"alpha must be >= 0, got {alpha = }\")\n", - " weight = np.power(np.abs(data) ** 2, alpha / 2)\n", - " data = weight * data\n", - " return data\n", - "\n", - " def make_weight(nxp, nyp):\n", - " wx = 1.0 - np.abs(np.arange(nxp // 2) - (nxp / 2.0 - 1.0)) / (nxp / 2.0 - 1.0)\n", - " wy = 1.0 - np.abs(np.arange(nyp // 2) - (nyp / 2.0 - 1.0)) / (nyp / 2.0 - 1.0)\n", - " quadrant = np.outer(wy, wx)\n", - " weight = np.block(\n", - " [\n", - " [quadrant, np.flip(quadrant, axis=1)],\n", - " [np.flip(quadrant, axis=0), np.flip(np.flip(quadrant, axis=0), axis=1)],\n", - " ]\n", - " )\n", - " return weight\n", - "\n", - " def patch_goldstein_filter(data, weight, psize):\n", - " data = np.fft.fft2(data, s=(psize, psize))\n", - " data = apply_pspec(data)\n", - " data = np.fft.ifft2(data, s=(psize, psize))\n", - " return weight * data\n", - "\n", - " def apply_goldstein_filter(data):\n", - " out = np.zeros(data.shape, dtype=np.complex64)\n", - " empty_mask = np.isnan(data) | (np.angle(data) == 0)\n", - " if np.all(empty_mask):\n", - " return data\n", - " weight_matrix = make_weight(psize, psize)\n", - " for i in range(0, data.shape[0] - psize, psize // 2):\n", - " for j in range(0, data.shape[1] - psize, psize // 2):\n", - " data_window = data[i : i + psize, j : j + psize]\n", - " weight_window = weight_matrix[: data_window.shape[0], : data_window.shape[1]]\n", - " filtered_window = patch_goldstein_filter(data_window, weight_window, psize)\n", - " slice_i = slice(i, min(i + psize, out.shape[0]))\n", - " slice_j = slice(j, min(j + psize, out.shape[1]))\n", - " out[slice_i, slice_j] += filtered_window[: slice_i.stop - slice_i.start, : slice_j.stop - slice_j.start]\n", - " out[empty_mask] = 0\n", - " return out\n", - "\n", - " if np.iscomplexobj(phase):\n", - " return apply_goldstein_filter(phase)\n", - " else:\n", - " return apply_goldstein_filter(np.exp(1j * phase))\n", - "\n", - " phase = reference * np.conjugate(secondary)\n", - " amp = np.sqrt((reference * np.conjugate(reference)) * (secondary * np.conjugate(secondary)))\n", - " nan_mask = np.isnan(phase)\n", - " ifg[nan_mask] = np.nan\n", - " ifg_cpx = np.exp(1j * np.nan_to_num(np.angle(phase/amp)))\n", - " zero_mask = phase == 0\n", - " coh[nan_mask] = np.nan\n", - " coh[zero_mask] = 0\n", - " return ifg, coh, amp\n", - "\n", - "def calc_ifg_coh_filtered(reference, secondary, goldstein_alpha=0.5, coh_win=5, looks_y=1, looks_x=1):\n", - " reference = _multilook(reference, looks_y, looks_x)\n", - " secondary = _multilook(secondary, looks_y, looks_x)\n", - " phase = reference * np.conjugate(secondary)\n", - " amp = np.sqrt((reference * np.conjugate(reference)) * (secondary * np.conjugate(secondary)))\n", - " nan_mask = np.isnan(phase)\n", - " ifg_cpx = np.exp(1j * np.nan_to_num(np.angle(phase/amp)))\n", - " ifg_cpx_f = goldstein(ifg_cpx, alpha=goldstein_alpha, psize=32)\n", - " ifg = np.angle(ifg_cpx_f)\n", - " ifg[nan_mask] = np.nan\n", - " coh = np.abs(_box_mean(ifg_cpx, coh_win))\n", - " coh = np.clip(coh, 0, 1)\n", - " coh = lee_filter(coh, win=coh_win)\n", - " coh = np.clip(coh, 0, 1)\n", - " zero_mask = phase == 0\n", - " coh[nan_mask] = np.nan\n", - " coh[zero_mask] = 0\n", - " return ifg, coh, amp\n", - "\n", - "def calc_ifg_coh(reference, secondary, looks_y=1, looks_x=1):\n", - " return calc_ifg_coh_filtered(reference, secondary, looks_y=looks_y, looks_x=looks_x)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2775556c", - "metadata": {}, - "outputs": [], - "source": [ - "## For each date-pair, calculate the ifg, coh. Save the results as GeoTiffs.\n", - "for ref_idx, sec_idx in pair_indices:\n", - " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", - " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", - " print(f\"Reference: {ref_date} Secondary: {sec_date}\")\n", - "\n", - " # Calculate ifg, coh, amp\n", - " if \"calc_ifg_coh_filtered\" not in globals():\n", - " raise RuntimeError(\"calc_ifg_coh_filtered is not defined. Run the filter definition cell first.\")\n", - " looks_y, looks_x = MULTILOOK\n", - " ifg, coh, amp = calc_ifg_coh_filtered(cslc_stack[ref_idx], cslc_stack[sec_idx], looks_y=looks_y, looks_x=looks_x)\n", - "\n", - " # Save each interferogram as GeoTiff (no per-burst plotting)\n", - " transform = from_origin(xcoor_stack[ref_idx][0], ycoor_stack[ref_idx][0], dx, np.abs(dy))\n" - ] - }, - { - "cell_type": "markdown", - "id": "d57d1a71", - "metadata": {}, - "source": [ - "## 8. Merge the burst-wise interferograms and coherence and save as GeoTiff." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e98baf1f", - "metadata": {}, - "outputs": [], - "source": [ - "def custom_merge(old_data, new_data, old_nodata, new_nodata, **kwargs): \n", - " mask = np.logical_and(~old_nodata, ~new_nodata)\n", - " old_data[mask] = new_data[mask]\n", - " mask = np.logical_and(old_nodata, ~new_nodata)\n", - " old_data[mask] = new_data[mask]\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "897e44dd", - "metadata": {}, - "outputs": [], - "source": [ - "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", - "# Merge burst-wise interferograms per date-pair (in-memory)\n", - "from rasterio.io import MemoryFile\n", - "\n", - "from rasterio.warp import calculate_default_transform, reproject, Resampling\n", - "\n", - "\n", - "from rasterio.warp import reproject, Resampling\n", - "from pathlib import Path\n", - "\n", - "def _load_water_mask_match(mask_path, shape, transform, crs):\n", - " if mask_path is None or not Path(mask_path).exists():\n", - " return None\n", - " with rasterio.open(mask_path) as src:\n", - " src_mask = src.read(1)\n", - " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", - " return src_mask\n", - " dst = np.zeros(shape, dtype=src_mask.dtype)\n", - " reproject(\n", - " source=src_mask,\n", - " destination=dst,\n", - " src_transform=src.transform,\n", - " src_crs=src.crs,\n", - " dst_transform=transform,\n", - " dst_crs=crs,\n", - " resampling=Resampling.nearest,\n", - " )\n", - " return dst\n", - "\n", - "def _apply_water_mask(arr, mask):\n", - " # mask: 1 = keep land, 0 = water\n", - " return np.where(mask == 0, np.nan, arr)\n", - "\n", - "\n", - "from affine import Affine\n", - "\n", - "def _trim_nan_border(arr, transform):\n", - " data = arr[0] if arr.ndim == 3 else arr\n", - " mask = np.isfinite(data) & (data != 0)\n", - " if not mask.any():\n", - " return arr, transform\n", - " rows = np.where(mask.any(axis=1))[0]\n", - " cols = np.where(mask.any(axis=0))[0]\n", - " r0, r1 = rows[0], rows[-1] + 1\n", - " c0, c1 = cols[0], cols[-1] + 1\n", - " data = data[r0:r1, c0:c1]\n", - " if arr.ndim == 3:\n", - " arr = data[None, ...]\n", - " else:\n", - " arr = data\n", - " new_transform = transform * Affine.translation(c0, r0)\n", - " return arr, new_transform\n", - "\n", - "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", - " import os\n", - " os.makedirs(os.path.dirname(out_path), exist_ok=True)\n", - " dst_crs = 'EPSG:4326'\n", - " dst_transform, width, height = calculate_default_transform(\n", - " f'EPSG:{epsg}', dst_crs, mosaic.shape[2], mosaic.shape[1], *rasterio.transform.array_bounds(mosaic.shape[1], mosaic.shape[2], transform)\n", - " )\n", - " dest = np.zeros((1, height, width), dtype=mosaic.dtype)\n", - " reproject(\n", - " source=mosaic,\n", - " destination=dest,\n", - " src_transform=transform,\n", - " src_crs=f'EPSG:{epsg}',\n", - " dst_transform=dst_transform,\n", - " dst_crs=dst_crs,\n", - " resampling=Resampling.nearest,\n", - " )\n", - " out_meta = {\n", - " 'driver': 'GTiff',\n", - " 'height': height,\n", - " 'width': width,\n", - " 'count': 1,\n", - " 'dtype': mosaic.dtype,\n", - " 'crs': dst_crs,\n", - " 'transform': dst_transform,\n", - " }\n", - " with rasterio.open(out_path, 'w', **out_meta) as dst:\n", - " dst.write(dest)\n", - "\n", - "# Group pair indices by date tag\n", - "pairs_by_tag = {}\n", - "for r, s in pair_indices:\n", - " ref_date = cslc_dates.iloc[r].values[0]\n", - " sec_date = cslc_dates.iloc[s].values[0]\n", - " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", - " pairs_by_tag.setdefault(tag, []).append((r, s))\n", - "\n", - "for tag, pairs in pairs_by_tag.items():\n", - " srcs = []\n", - " for r, s in pairs:\n", - " looks_y, looks_x = MULTILOOK\n", - " ifg, coh, amp = calc_ifg_coh(cslc_stack[r], cslc_stack[s], looks_y=looks_y, looks_x=looks_x)\n", - " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", - " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", - " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", - " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", - " mem = MemoryFile()\n", - " ds = mem.open(\n", - " driver='GTiff', height=ifg.shape[0], width=ifg.shape[1], count=1, dtype=ifg.dtype,\n", - " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", - " )\n", - " ds.write(ifg, 1)\n", - " srcs.append(ds)\n", - " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", - " dest, output_transform = _trim_nan_border(dest, output_transform)\n", - " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", - " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", - " if mask is not None:\n", - " dest[0] = _apply_water_mask(dest[0], mask)\n", - " out_meta = srcs[0].meta.copy()\n", - " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", - " out_path = f\"{savedir}/tifs/merged_ifg_{tag}.tif\"\n", - " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", - " dest1.write(dest)\n", - " if SAVE_WGS84:\n", - " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_ifg_WGS84_{tag}.tif\"\n", - " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", - " for ds in srcs:\n", - " ds.close()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7c41924c", - "metadata": {}, - "outputs": [], - "source": [ - "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", - "# Merge burst-wise coherence per date-pair (in-memory)\n", - "from rasterio.io import MemoryFile\n", - "\n", - "from rasterio.warp import calculate_default_transform, reproject, Resampling\n", - "\n", - "\n", - "from rasterio.warp import reproject, Resampling\n", - "\n", - "def _load_water_mask_match(mask_path, shape, transform, crs):\n", - " with rasterio.open(mask_path) as src:\n", - " src_mask = src.read(1)\n", - " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", - " return src_mask\n", - " dst = np.zeros(shape, dtype=src_mask.dtype)\n", - " reproject(\n", - " source=src_mask,\n", - " destination=dst,\n", - " src_transform=src.transform,\n", - " src_crs=src.crs,\n", - " dst_transform=transform,\n", - " dst_crs=crs,\n", - " resampling=Resampling.nearest,\n", - " )\n", - " return dst\n", - "\n", - "def _apply_water_mask(arr, mask):\n", - " # mask: 1 = keep land, 0 = water\n", - " return np.where(mask == 0, np.nan, arr)\n", - "\n", - "\n", - "from affine import Affine\n", - "\n", - "def _trim_nan_border(arr, transform):\n", - " data = arr[0] if arr.ndim == 3 else arr\n", - " mask = np.isfinite(data) & (data != 0)\n", - " if not mask.any():\n", - " return arr, transform\n", - " rows = np.where(mask.any(axis=1))[0]\n", - " cols = np.where(mask.any(axis=0))[0]\n", - " r0, r1 = rows[0], rows[-1] + 1\n", - " c0, c1 = cols[0], cols[-1] + 1\n", - " data = data[r0:r1, c0:c1]\n", - " if arr.ndim == 3:\n", - " arr = data[None, ...]\n", - " else:\n", - " arr = data\n", - " new_transform = transform * Affine.translation(c0, r0)\n", - " return arr, new_transform\n", - "\n", - "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", - " import os\n", - " os.makedirs(os.path.dirname(out_path), exist_ok=True)\n", - " dst_crs = 'EPSG:4326'\n", - " dst_transform, width, height = calculate_default_transform(\n", - " f'EPSG:{epsg}', dst_crs, mosaic.shape[2], mosaic.shape[1], *rasterio.transform.array_bounds(mosaic.shape[1], mosaic.shape[2], transform)\n", - " )\n", - " dest = np.zeros((1, height, width), dtype=mosaic.dtype)\n", - " reproject(\n", - " source=mosaic,\n", - " destination=dest,\n", - " src_transform=transform,\n", - " src_crs=f'EPSG:{epsg}',\n", - " dst_transform=dst_transform,\n", - " dst_crs=dst_crs,\n", - " resampling=Resampling.nearest,\n", - " )\n", - " out_meta = {\n", - " 'driver': 'GTiff',\n", - " 'height': height,\n", - " 'width': width,\n", - " 'count': 1,\n", - " 'dtype': mosaic.dtype,\n", - " 'crs': dst_crs,\n", - " 'transform': dst_transform,\n", - " }\n", - " with rasterio.open(out_path, 'w', **out_meta) as dst:\n", - " dst.write(dest)\n", - "\n", - "# Group pair indices by date tag\n", - "pairs_by_tag = {}\n", - "for r, s in pair_indices:\n", - " ref_date = cslc_dates.iloc[r].values[0]\n", - " sec_date = cslc_dates.iloc[s].values[0]\n", - " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", - " pairs_by_tag.setdefault(tag, []).append((r, s))\n", - "\n", - "for tag, pairs in pairs_by_tag.items():\n", - " srcs = []\n", - " for r, s in pairs:\n", - " looks_y, looks_x = MULTILOOK\n", - " ifg, coh, amp = calc_ifg_coh(cslc_stack[r], cslc_stack[s], looks_y=looks_y, looks_x=looks_x)\n", - " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", - " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", - " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", - " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", - " mem = MemoryFile()\n", - " ds = mem.open(\n", - " driver='GTiff', height=coh.shape[0], width=coh.shape[1], count=1, dtype=coh.dtype,\n", - " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", - " )\n", - " ds.write(coh, 1)\n", - " srcs.append(ds)\n", - " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", - " dest, output_transform = _trim_nan_border(dest, output_transform)\n", - " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", - " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", - " dest[0] = _apply_water_mask(dest[0], mask)\n", - " out_meta = srcs[0].meta.copy()\n", - " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", - " out_path = f\"{savedir}/tifs/merged_coh_{tag}.tif\"\n", - " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", - " dest1.write(dest)\n", - " if SAVE_WGS84:\n", - " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_coh_WGS84_{tag}.tif\"\n", - " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", - " for ds in srcs:\n", - " ds.close()\n" - ] - }, - { - "cell_type": "markdown", - "id": "858ea831", - "metadata": {}, - "source": [ - "## 9. Read the merged GeoTiff and Visualize using `matplotlib`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9cb2a341", - "metadata": {}, - "outputs": [], - "source": [ - "# Read merged IFG/COH files and plot paired grids\n", - "import glob\n", - "import math\n", - "import pandas as pd\n", - "import os\n", - "\n", - "\n", - "# Output dir for per-pair PNGs\n", - "pair_png_dir = f\"{savedir}/pairs_png\"\n", - "os.makedirs(pair_png_dir, exist_ok=True)\n", - "\n", - "ifg_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_ifg_*.tif\"))\n", - "coh_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\"))\n", - "\n", - "ifg_map = {p.split('merged_ifg_')[-1].replace('.tif',''): p for p in ifg_paths}\n", - "coh_map = {p.split('merged_coh_')[-1].replace('.tif',''): p for p in coh_paths}\n", - "\n", - "\n", - "\n", - "def _prep_da(path):\n", - " da = rioxarray.open_rasterio(path)[0]\n", - " if REPROJECT_FOR_DISPLAY:\n", - " da = da.rio.reproject(\"EPSG:4326\")\n", - " data = da.values\n", - " mask = np.isfinite(data) & (data != 0)\n", - " if mask.any():\n", - " rows = np.where(mask.any(axis=1))[0]\n", - " cols = np.where(mask.any(axis=0))[0]\n", - " r0, r1 = rows[0], rows[-1] + 1\n", - " c0, c1 = cols[0], cols[-1] + 1\n", - " # Trim NaN borders so edges don't show padding\n", - " da = da.isel(y=slice(r0, r1), x=slice(c0, c1))\n", - " return da\n", - "\n", - "pair_tags = sorted(set(ifg_map).intersection(coh_map))\n", - "# Filter pairs by current date range\n", - "date_start_day = dateStart.date()\n", - "date_end_day = dateEnd.date()\n", - "pair_tags = [t for t in pair_tags if (date_start_day <= pd.to_datetime(t.split('-')[0], format='%Y%m%d').date() <= date_end_day and date_start_day <= pd.to_datetime(t.split('-')[1], format='%Y%m%d').date() <= date_end_day)]\n", - "\n", - "if not pair_tags:\n", - " print('No matching IFG/COH pairs found')\n", - "else:\n", - " # Save ALL pairs as PNGs\n", - " for tag in pair_tags:\n", - " fig, axes = plt.subplots(1, 2, figsize=(10, 4), constrained_layout=True)\n", - " ax_ifg, ax_coh = axes\n", - "\n", - " # IFG\n", - " merged_ifg = _prep_da(ifg_map[tag])\n", - " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", - " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", - " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", - " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", - " ax_ifg.set_xticks([])\n", - " ax_ifg.set_yticks([])\n", - " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", - "\n", - " # COH\n", - " merged_coh = _prep_da(coh_map[tag])\n", - " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", - " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=0, vmax=1.0)\n", - " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", - " ax_coh.set_xticks([])\n", - " ax_coh.set_yticks([])\n", - " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')\n", - "\n", - " out_png = os.path.join(pair_png_dir, f\"pair_{tag}.png\")\n", - " fig.savefig(out_png, dpi=150)\n", - " plt.close(fig)\n", - "\n", - " # Display only last 5 pairs in notebook\n", - " display_tags = pair_tags[-5:]\n", - " n = len(display_tags)\n", - " ncols = 2\n", - " nrows = math.ceil(n / 1) # one pair per row\n", - " fig, axes = plt.subplots(nrows, ncols, figsize=(6*ncols, 3*nrows), constrained_layout=True)\n", - " if nrows == 1:\n", - " axes = [axes]\n", - "\n", - " for i, tag in enumerate(display_tags):\n", - " ax_ifg, ax_coh = axes[i]\n", - "\n", - " # IFG\n", - " merged_ifg = _prep_da(ifg_map[tag])\n", - " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", - " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", - " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", - " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", - " ax_ifg.set_xticks([])\n", - " ax_ifg.set_yticks([])\n", - " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", - "\n", - " # COH\n", - " merged_coh = _prep_da(coh_map[tag])\n", - " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", - " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=0, vmax=1.0)\n", - " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", - " ax_coh.set_xticks([])\n", - " ax_coh.set_yticks([])\n", - " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')\n" - ] - }, - { - "cell_type": "markdown", - "id": "ea1e4f24", - "metadata": {}, - "source": [ - "## 9.5 Merge and plot amplitude mosaics (per date)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e7f478f4", - "metadata": {}, - "outputs": [], - "source": [ - "import h5py\n", - "import numpy as np\n", - "import glob\n", - "import math\n", - "import rasterio\n", - "from rasterio.transform import from_origin\n", - "from rasterio.crs import CRS\n", - "from rasterio.warp import calculate_default_transform, reproject, Resampling\n", - "\n", - "\n", - "from rasterio.warp import reproject, Resampling\n", - "\n", - "def _load_water_mask_match(mask_path, shape, transform, crs):\n", - " with rasterio.open(mask_path) as src:\n", - " src_mask = src.read(1)\n", - " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", - " return src_mask\n", - " dst = np.zeros(shape, dtype=src_mask.dtype)\n", - " reproject(\n", - " source=src_mask,\n", - " destination=dst,\n", - " src_transform=src.transform,\n", - " src_crs=src.crs,\n", - " dst_transform=transform,\n", - " dst_crs=crs,\n", - " resampling=Resampling.nearest,\n", - " )\n", - " return dst\n", - "\n", - "def _apply_water_mask(arr, mask):\n", - " # mask: 1 = keep land, 0 = water\n", - " return np.where(mask == 0, np.nan, arr)\n", - "\n", - "\n", - "from affine import Affine\n", - "\n", - "def _trim_nan_border(arr, transform):\n", - " data = arr[0] if arr.ndim == 3 else arr\n", - " mask = np.isfinite(data) & (data != 0)\n", - " if not mask.any():\n", - " return arr, transform\n", - " rows = np.where(mask.any(axis=1))[0]\n", - " cols = np.where(mask.any(axis=0))[0]\n", - " r0, r1 = rows[0], rows[-1] + 1\n", - " c0, c1 = cols[0], cols[-1] + 1\n", - " data = data[r0:r1, c0:c1]\n", - " if arr.ndim == 3:\n", - " arr = data[None, ...]\n", - " else:\n", - " arr = data\n", - " new_transform = transform * Affine.translation(c0, r0)\n", - " return arr, new_transform\n", - "\n", - "# Build per-date amplitude mosaics directly from subset H5 (no per-burst GeoTIFFs)\n", - "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", - "\n", - "date_tags = sorted(cslc_df.startTime.astype(str).str.replace('-', '').unique())\n", - "\n", - "def _save_mosaic_utm(out_path, mosaic, transform, epsg):\n", - " with rasterio.open(\n", - " out_path, \"w\", driver=\"GTiff\", height=mosaic.shape[0], width=mosaic.shape[1],\n", - " count=1, dtype=mosaic.dtype, crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", - " ) as dst_ds:\n", - " dst_ds.write(mosaic, 1)\n", - "\n", - "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", - " dst_crs = \"EPSG:4326\"\n", - " src_crs = CRS.from_epsg(epsg)\n", - " height, width = mosaic.shape\n", - " dst_transform, dst_width, dst_height = calculate_default_transform(\n", - " src_crs, dst_crs, width, height, *rasterio.transform.array_bounds(height, width, transform)\n", - " )\n", - " dst = np.empty((dst_height, dst_width), dtype=mosaic.dtype)\n", - " reproject(\n", - " source=mosaic,\n", - " destination=dst,\n", - " src_transform=transform,\n", - " src_crs=src_crs,\n", - " dst_transform=dst_transform,\n", - " dst_crs=dst_crs,\n", - " resampling=Resampling.bilinear,\n", - " )\n", - " with rasterio.open(\n", - " out_path, \"w\", driver=\"GTiff\", height=dst_height, width=dst_width, count=1,\n", - " dtype=dst.dtype, crs=dst_crs, transform=dst_transform, nodata=np.nan\n", - " ) as dst_ds:\n", - " dst_ds.write(dst, 1)\n", - "\n", - "# Mosaicking helper (in memory)\n", - "def _mosaic_arrays(arrays, transforms, epsg):\n", - " # Convert arrays to in-memory rasterio datasets via MemoryFile\n", - " from rasterio.io import MemoryFile\n", - " srcs = []\n", - " for arr, transform in zip(arrays, transforms):\n", - " mem = MemoryFile()\n", - " ds = mem.open(\n", - " driver='GTiff', height=arr.shape[0], width=arr.shape[1], count=1, dtype=arr.dtype,\n", - " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", - " )\n", - " ds.write(arr, 1)\n", - " srcs.append(ds)\n", - " dest, out_transform = merge.merge(srcs, method=custom_merge)\n", - " for ds in srcs:\n", - " ds.close()\n", - " return dest[0], out_transform\n", - "\n", - "looks_y, looks_x = MULTILOOK\n", - "for date_tag in date_tags:\n", - " # collect subset H5 files for this date\n", - " rows = cslc_df[cslc_df.startTime.astype(str).str.replace('-', '') == date_tag]\n", - " arrays = []\n", - " transforms = []\n", - " epsg = None\n", - " for fileID in rows.fileID:\n", - " subset_path = f\"{savedir}/subset_cslc/{fileID}.h5\"\n", - " with h5py.File(subset_path, 'r') as h5:\n", - " cslc = h5['/data/VV'][:]\n", - " xcoor = h5['/data/x_coordinates'][:]\n", - " ycoor = h5['/data/y_coordinates'][:]\n", - " dx = int(h5['/data/x_spacing'][()])\n", - " dy = int(h5['/data/y_spacing'][()])\n", - " epsg = int(h5['/data/projection'][()])\n", - " power_ml = _multilook(np.abs(cslc)**2, looks_y, looks_x)\n", - " amp = 10*np.log10(power_ml)\n", - " dy_signed = (ycoor[1] - ycoor[0]) if len(ycoor) > 1 else -dy\n", - " x0 = xcoor[0] + (looks_x - 1) * dx / 2\n", - " y0 = ycoor[0] + (looks_y - 1) * dy_signed / 2\n", - " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", - " arrays.append(amp)\n", - " transforms.append(transform)\n", - "\n", - " if not arrays:\n", - " continue\n", - " mosaic, out_transform = _mosaic_arrays(arrays, transforms, epsg)\n", - " out_path_utm = f\"{savedir}/tifs/merged_amp_{date_tag}.tif\"\n", - " mosaic, out_transform = _trim_nan_border(mosaic, out_transform)\n", - " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", - " mask = _load_water_mask_match(WATER_MASK_PATH, mosaic.shape, out_transform, CRS.from_epsg(epsg))\n", - " mosaic = _apply_water_mask(mosaic, mask)\n", - " _save_mosaic_utm(out_path_utm, mosaic, out_transform, epsg)\n", - " if SAVE_WGS84:\n", - " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_amp_WGS84_{date_tag}.tif\"\n", - " _save_mosaic_utm_to_wgs84(out_path_wgs84, mosaic, out_transform, epsg)\n", - "\n", - "# Plot merged amplitude mosaics in a grid (native CRS from saved GeoTIFFs)\n", - "\n", - "# Output dir for amplitude PNGs\n", - "amp_png_dir = f\"{savedir}/amp_png\"\n", - "os.makedirs(amp_png_dir, exist_ok=True)\n", - "\n", - "paths = sorted(glob.glob(f\"{savedir}/tifs/merged_amp_*.tif\"))\n", - "paths = [p for p in paths if 'WGS84' not in p]\n", - "all_vals = []\n", - "for p in paths:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " all_vals.append(da.values.ravel())\n", - "if all_vals:\n", - " all_vals = np.concatenate(all_vals)\n", - " gmin = np.nanpercentile(all_vals, 2)\n", - " gmax = np.nanpercentile(all_vals, 90)\n", - "else:\n", - " gmin, gmax = None, None\n", - "n = len(paths)\n", - "if n == 0:\n", - " print('No merged amplitude files found')\n", - "else:\n", - " # Save ALL amplitude PNGs\n", - " for path in paths:\n", - " src = rioxarray.open_rasterio(path)\n", - " amp = src[0]\n", - " minlon, minlat, maxlon, maxlat = amp.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " fig, ax = plt.subplots(figsize=(5,4))\n", - " im = ax.imshow(amp.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", - " tag = path.split('merged_amp_')[-1].replace('.tif','')\n", - " ax.set_title(f\"AMP_{tag}\", fontsize=10)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " fig.colorbar(im, ax=ax, orientation='vertical', fraction=0.046, pad=0.02)\n", - " out_png = os.path.join(amp_png_dir, f\"amp_{tag}.png\")\n", - " fig.savefig(out_png, dpi=150)\n", - " plt.close(fig)\n", - "\n", - " # Show only last 5 in notebook\n", - " display_paths = paths[-5:]\n", - " n = len(display_paths)\n", - " ncols = 3\n", - " nrows = math.ceil(n / ncols)\n", - " fig, axes = plt.subplots(nrows, ncols, figsize=(4*ncols, 3*nrows), constrained_layout=True)\n", - " axes = axes.ravel()\n", - " for ax, path in zip(axes, display_paths):\n", - " src = rioxarray.open_rasterio(path)\n", - " amp = src[0]\n", - " minlon, minlat, maxlon, maxlat = amp.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " im = ax.imshow(amp.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", - " tag = path.split('merged_amp_')[-1].replace('.tif','')\n", - " ax.set_title(f\"AMP_{tag}\", fontsize=10)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " for ax in axes[n:]:\n", - " ax.axis('off')\n", - " fig.colorbar(im, ax=axes.tolist(), orientation='vertical', fraction=0.02, pad=0.02)\n" - ] - }, - { - "cell_type": "markdown", - "id": "df66a59f", - "metadata": {}, - "source": [ - "## 10. Monthly mean coherence calendar (per year)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c31a947a", - "metadata": {}, - "outputs": [], - "source": [ - "import glob\n", - "import pandas as pd\n", - "import numpy as np\n", - "import rioxarray\n", - "import xarray as xr\n", - "import matplotlib.pyplot as plt\n", - "\n", - "\n", - "\n", - "# Build an index of merged coherence files by midpoint year-month\n", - "records = []\n", - "for path in sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\")):\n", - " tag = path.split('merged_coh_')[-1].replace('.tif','')\n", - " try:\n", - " ref_str, sec_str = tag.split('-')\n", - " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", - " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", - " mid_date = ref_date + (sec_date - ref_date) / 2\n", - " except Exception:\n", - " continue\n", - " records.append({\"path\": path, \"mid_date\": mid_date})\n", - "\n", - "df_paths = pd.DataFrame(records)\n", - "if df_paths.empty:\n", - " print('No merged coherence files found for calendar')\n", - " raise SystemExit\n", - "\n", - "# Apply current date range using midpoint date\n", - "date_start_day = dateStart.date()\n", - "date_end_day = dateEnd.date()\n", - "df_paths = df_paths[(df_paths['mid_date'].dt.date >= date_start_day) & (df_paths['mid_date'].dt.date <= date_end_day)]\n", - "\n", - "# Calendar year labeling\n", - "if USE_WATER_YEAR:\n", - " # Water year starts Oct (10) and ends Sep (9)\n", - " df_paths['year'] = df_paths['mid_date'].dt.year + (df_paths['mid_date'].dt.month >= 10).astype(int)\n", - " month_order = [10,11,12,1,2,3,4,5,6,7,8,9]\n", - " month_labels = ['Oct','Nov','Dec','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep']\n", - "else:\n", - " df_paths['year'] = df_paths['mid_date'].dt.year\n", - " month_order = list(range(1,13))\n", - " month_labels = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']\n", - "\n", - "years = sorted(df_paths['year'].unique())\n", - "\n", - "# Contrast stretch for low coherence (red = low)\n", - "# norm = mcolors.PowerNorm(gamma=0.3, vmin=0, vmax=1)\n", - "\n", - "# One row per year, 12 columns\n", - "fig, axes = plt.subplots(len(years), 12, figsize=(24, 2.5*len(years)), constrained_layout=True)\n", - "if len(years) == 1:\n", - " axes = np.array([axes])\n", - "\n", - "for row_idx, y in enumerate(years):\n", - " # pick a template for consistent grid within the year (first available file)\n", - " year_paths = df_paths[df_paths['year'] == y]['path'].tolist()\n", - " if not year_paths:\n", - " continue\n", - " template = rioxarray.open_rasterio(year_paths[0])[0]\n", - "\n", - " for col_idx, m in enumerate(month_order):\n", - " ax = axes[row_idx, col_idx]\n", - " month_paths = df_paths[(df_paths['year'] == y) & (df_paths['mid_date'].dt.month == m)]['path'].tolist()\n", - " if USE_WATER_YEAR:\n", - " year_for_month = y - 1 if m in (10, 11, 12) else y\n", - " else:\n", - " year_for_month = y\n", - " title = f\"{month_labels[col_idx]} {year_for_month}\"\n", - " if not month_paths:\n", - " ax.set_title(title, fontsize=9)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " # keep a visible box for empty months\n", - " for spine in ax.spines.values():\n", - " spine.set_visible(True)\n", - " spine.set_linewidth(0.8)\n", - " spine.set_color('0.5')\n", - " continue\n", - " stacks = []\n", - " for p in month_paths:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " da = da.rio.reproject_match(template)\n", - " stacks.append(da)\n", - " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", - " minlon, minlat, maxlon, maxlat = da_month.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " im = ax.imshow(da_month.values, cmap='gray', vmin=0, vmax=1, origin='upper', extent=bbox, interpolation='none')\n", - " ax.set_title(title, fontsize=9)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " for spine in ax.spines.values():\n", - " spine.set_visible(True)\n", - " spine.set_linewidth(0.8)\n", - " spine.set_color('0.5')\n", - " # left-side year label\n", - " if USE_WATER_YEAR:\n", - " label = f\"WY {y}\"\n", - " else:\n", - " label = str(y)\n", - " axes[row_idx, 0].set_ylabel(label, rotation=90, labelpad=6, fontsize=9)\n", - " axes[row_idx, 0].yaxis.set_label_coords(-0.06, 0.5)\n", - "\n", - "fig.colorbar(im, ax=axes, orientation='vertical', fraction=0.02, pad=0.02, label='Mean coherence')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "02e68ad2", - "metadata": {}, - "outputs": [], - "source": [ - "# Debug: list midpoint dates and their counts\n", - "df_paths[['mid_date']].sort_values('mid_date')\n", - "df_paths['mid_date'].dt.to_period('M').value_counts().sort_index()\n" - ] - }, - { - "cell_type": "markdown", - "id": "169216f0", - "metadata": {}, - "source": [ - "## 11. Create GIF animations (Amplitude + Coherence)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "73ba2684", - "metadata": {}, - "outputs": [], - "source": [ - "import glob\n", - "import os\n", - "import numpy as np\n", - "import pandas as pd\n", - "import imageio.v2 as imageio\n", - "import matplotlib.pyplot as plt\n", - "import rioxarray\n", - "import xarray as xr\n", - "\n", - "# Output folders\n", - "gif_dir = f\"{savedir}/gifs\"\n", - "os.makedirs(gif_dir, exist_ok=True)\n", - "\n", - "def _global_bounds(paths):\n", - " bounds = []\n", - " for p in paths:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " minx, miny, maxx, maxy = da.rio.bounds()\n", - " bounds.append((minx, miny, maxx, maxy))\n", - " minx = min(b[0] for b in bounds)\n", - " miny = min(b[1] for b in bounds)\n", - " maxx = max(b[2] for b in bounds)\n", - " maxy = max(b[3] for b in bounds)\n", - " return [minx, maxx, miny, maxy]\n", - "\n", - "def _render_frames(tif_paths, out_dir, cmap, vmin=None, vmax=None, title_prefix=\"\", extent=None, cbar_label=None, cbar_ticks=None):\n", - " os.makedirs(out_dir, exist_ok=True)\n", - " frames = []\n", - " for p in tif_paths:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " if extent is None:\n", - " minlon, minlat, maxlon, maxlat = da.rio.bounds()\n", - " extent = [minlon, maxlon, minlat, maxlat]\n", - " fig, ax = plt.subplots(figsize=(6,4))\n", - " im = ax.imshow(da.values, cmap=cmap, origin='upper', extent=extent, vmin=vmin, vmax=vmax)\n", - " tag = os.path.basename(p).replace('.tif','')\n", - " ax.set_title(f\"{title_prefix}{tag}\", fontsize=9)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", - " if cbar_label:\n", - " cb.set_label(cbar_label)\n", - " if cbar_ticks is not None:\n", - " cb.set_ticks(cbar_ticks)\n", - " frame_path = os.path.join(out_dir, f\"{tag}.png\")\n", - " fig.savefig(frame_path, dpi=150)\n", - " plt.close(fig)\n", - " frames.append(frame_path)\n", - " return frames\n", - "\n", - "def _pad_frames(frame_paths):\n", - " imgs = [imageio.imread(f) for f in frame_paths]\n", - " max_h = max(im.shape[0] for im in imgs)\n", - " max_w = max(im.shape[1] for im in imgs)\n", - " padded = []\n", - " for im in imgs:\n", - " pad_h = max_h - im.shape[0]\n", - " pad_w = max_w - im.shape[1]\n", - " padded.append(np.pad(im, ((0, pad_h), (0, pad_w), (0, 0)), mode='edge'))\n", - " return padded\n", - "\n", - "# Amplitude GIF (uses merged amplitude mosaics)\n", - "amp_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_amp_*.tif\"))\n", - "amp_paths = [p for p in amp_paths if 'WGS84' not in p]\n", - "if amp_paths:\n", - " _vals = []\n", - " for p in amp_paths:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " _vals.append(da.values.ravel())\n", - " _vals = np.concatenate(_vals)\n", - " amp_vmin = np.nanpercentile(_vals, 1)\n", - " amp_vmax = np.nanpercentile(_vals, 99)\n", - " amp_extent = _global_bounds(amp_paths)\n", - " amp_frames = _render_frames(\n", - " amp_paths, f\"{gif_dir}/amp_frames\", cmap=\"gray\", vmin=amp_vmin, vmax=amp_vmax,\n", - " title_prefix=\"AMP_\", extent=amp_extent, cbar_label=\"Amplitude (dB)\"\n", - " )\n", - " amp_gif = f\"{gif_dir}/amplitude.gif\"\n", - " amp_imgs = _pad_frames(amp_frames)\n", - " imageio.mimsave(amp_gif, amp_imgs, duration=0.8)\n", - " print(f\"Wrote {amp_gif}\")\n", - "else:\n", - " print('No merged amplitude files found for GIF')\n", - "\n", - "# Coherence GIF (uses merged coherence mosaics)\n", - "coh_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\"))\n", - "coh_paths = [p for p in coh_paths if 'WGS84' not in p]\n", - "if coh_paths:\n", - " coh_extent = _global_bounds(coh_paths)\n", - " coh_frames = _render_frames(\n", - " coh_paths, f\"{gif_dir}/coh_frames\", cmap='gray', vmin=0, vmax=1,\n", - " title_prefix='COH_', extent=coh_extent, cbar_label='Coherence', cbar_ticks=[0,0.5,1]\n", - " )\n", - " coh_gif = f\"{gif_dir}/coherence.gif\"\n", - " coh_imgs = _pad_frames(coh_frames)\n", - " imageio.mimsave(coh_gif, coh_imgs, duration=0.8)\n", - " print(f\"Wrote {coh_gif}\")\n", - "else:\n", - " print('No merged coherence files found for GIF')\n", - "\n", - "\n", - "# Monthly mean coherence GIF (same monthly averaging as calendar)\n", - "if coh_paths:\n", - " records = []\n", - " for path in sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\")):\n", - " tag = path.split('merged_coh_')[-1].replace('.tif','')\n", - " try:\n", - " ref_str, sec_str = tag.split('-')\n", - " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", - " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", - " mid_date = ref_date + (sec_date - ref_date) / 2\n", - " except Exception:\n", - " continue\n", - " records.append({\"path\": path, \"mid_date\": mid_date})\n", - "\n", - " df_paths = pd.DataFrame(records)\n", - " if df_paths.empty:\n", - " print('No merged coherence files found for monthly GIF')\n", - " else:\n", - " # Apply current date range using midpoint date\n", - " date_start_day = dateStart.date()\n", - " date_end_day = dateEnd.date()\n", - " df_paths = df_paths[(df_paths['mid_date'].dt.date >= date_start_day) & (df_paths['mid_date'].dt.date <= date_end_day)]\n", - "\n", - " # Month/year label logic\n", - " if USE_WATER_YEAR:\n", - " df_paths['year'] = df_paths['mid_date'].dt.year + (df_paths['mid_date'].dt.month >= 10).astype(int)\n", - " month_order = [10,11,12,1,2,3,4,5,6,7,8,9]\n", - " month_labels = ['Oct','Nov','Dec','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep']\n", - " else:\n", - " df_paths['year'] = df_paths['mid_date'].dt.year\n", - " month_order = list(range(1,13))\n", - " month_labels = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']\n", - "\n", - " # Frame output\n", - " monthly_dir = f\"{gif_dir}/coh_monthly_frames\"\n", - " os.makedirs(monthly_dir, exist_ok=True)\n", - " monthly_frames = []\n", - "\n", - " years = sorted(df_paths['year'].unique())\n", - " coh_extent = _global_bounds(coh_paths)\n", - "\n", - " for y in years:\n", - " year_paths = df_paths[df_paths['year'] == y]['path'].tolist()\n", - " if not year_paths:\n", - " continue\n", - " template = rioxarray.open_rasterio(year_paths[0])[0]\n", - "\n", - " for col_idx, m in enumerate(month_order):\n", - " month_paths = df_paths[(df_paths['year'] == y) & (df_paths['mid_date'].dt.month == m)]['path'].tolist()\n", - " if not month_paths:\n", - " continue\n", - "\n", - " stacks = []\n", - " for p in month_paths:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " da = da.rio.reproject_match(template)\n", - " stacks.append(da)\n", - " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", - "\n", - " if USE_WATER_YEAR:\n", - " year_for_month = y - 1 if m in (10, 11, 12) else y\n", - " else:\n", - " year_for_month = y\n", - " title = f\"{month_labels[col_idx]} {year_for_month}\"\n", - "\n", - " fig, ax = plt.subplots(figsize=(6,4))\n", - " im = ax.imshow(da_month.values, cmap='gray', vmin=0, vmax=1, origin='upper', extent=coh_extent, interpolation='none')\n", - " ax.set_title(title, fontsize=9)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", - " cb.set_label('Mean coherence')\n", - " cb.set_ticks([0, 0.5, 1])\n", - " tag = f\"{year_for_month}_{m:02d}\"\n", - " frame_path = os.path.join(monthly_dir, f\"{tag}.png\")\n", - " fig.savefig(frame_path, dpi=150)\n", - " plt.close(fig)\n", - " monthly_frames.append(frame_path)\n", - "\n", - " if monthly_frames:\n", - " monthly_imgs = _pad_frames(monthly_frames)\n", - " monthly_gif = f\"{gif_dir}/coherence_monthly.gif\"\n", - " imageio.mimsave(monthly_gif, monthly_imgs, duration=0.8)\n", - " print(f\"Wrote {monthly_gif}\")\n", - " else:\n", - " print('No monthly coherence frames created (no data in range)')\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "615c3b85", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "opera_cslc", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.14" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/CSLC/Landslides/environment_opera_cslc_landslides.yml b/CSLC/Landslides/environment_opera_cslc_landslides.yml deleted file mode 100644 index 57d3b77..0000000 --- a/CSLC/Landslides/environment_opera_cslc_landslides.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: opera_cslc_landslides -channels: - - conda-forge -dependencies: - - python>=3.10 - - asf_search - - affine - - cartopy - - folium - - gdal - - geopandas - - h5py - - imageio - - ipykernel - - jupyterlab - - matplotlib - - numpy - - opera-utils - - pandas - - pyproj - - rasterio - - requests - - rioxarray - - shapely - - tqdm - - xarray - - pip - - pip: - - watermark From a113a3f391a4c304afc1e1e84ccfd0801fa9b9bd Mon Sep 17 00:00:00 2001 From: Al Handwerger Date: Mon, 2 Feb 2026 21:28:59 -0800 Subject: [PATCH 5/7] CSLC notebook for landslide applications Creates pairs of wrapped intereferograms, coherence, and amplitudes, with animations. --- CSLC/Landslides/CSLC-S1_for_landslides.ipynb | 2368 ++++++++++++++++++ CSLC/Landslides/environment.yml | 32 + 2 files changed, 2400 insertions(+) create mode 100644 CSLC/Landslides/CSLC-S1_for_landslides.ipynb create mode 100644 CSLC/Landslides/environment.yml diff --git a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb new file mode 100644 index 0000000..93ac184 --- /dev/null +++ b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb @@ -0,0 +1,2368 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "36315bdf", + "metadata": {}, + "source": [ + "# Generate wrapped interferograms, coherence, and backscatter (dB) maps and animations using OPERA CSLC-S1\n", + "\n", + "--- \n", + "\n", + "This notebook:\n", + "- Searches OPERA CSLC-S1 products for your AOI + date range\n", + "- Subsets CSLCs **before download** using opera-utils\n", + "- Builds interferograms/coherence and merges bursts\n", + "- Exports mosaics and visualizations\n", + "\n", + "**Quick start**\n", + "1) Set parameters in the next cell (AOI, date range, pairing)\n", + "2) Run cells top-to-bottom\n", + "3) Outputs land in `savedir/`\n", + "\n", + "**Key toggles**\n", + "- `SAVE_WGS84`: save WGS84 GeoTIFF mosaics when True\n", + "- `DOWNLOAD_WITH_releaseOGRESS`: show progress bar for downloads\n", + "- `USE_WATER_YEAR`: Oct–Sep calendar layout when True\n", + "- `pair_mode` / `t_span`: control IFG pairing (all vs fixed separation)\n", + "\n", + "**Outputs**\n", + "- Subset CSLC H5: `savedir/subset_cslc/*.h5`\n", + "- Mosaics (IFG/COH, native CRS): `savedir/tifs/merged_ifg_*`, `merged_coh_*`\n", + "- WGS84 mosaics: `savedir/tifs/WGS84/merged_ifg_WGS84_*`, `merged_coh_WGS84_*`, `merged_bsc__WGS84_*` (backscatter, dB)\n", + "- Calibrated backscatter (dB) mosaics (native CRS; = sigma0 or beta0): `savedir/tifs/merged_bsc__*.tif`\n", + "- GIFs: `savedir/gifs/*.gif`\n", + "\n", + "### Data Used in the Example: \n", + "\n", + "- **10 meter (Northing) x 5 meter (Easting) North America OPERA Coregistered Single Look Complex from Sentinel-1 products**\n", + " - This dataset contains Level-2 OPERA coregistered single-look-complex (CSLC) data from Sentinel-1 (S1). The data in this example are geocoded CSLC-S1 data covering Palos Verdes landslides, California, USA. \n", + " \n", + " - The OPERA project is generating geocoded burst-wise CSLC-S1 products over North America which includes USA and US Territories within 200 km from the US border, Canada, and all mainland countries from the southern US border down to and including Panama. Each pixel within a burst SLC is represented by a complex number and contains both the amplitude and phase information. The CSLC-S1 products are distributed over projected map coordinates using the Universal Transverse Mercator (UTM) projection with spacing in the X- and Y-directions of 5 m and 10 m, respectively. Each OPERA CSLC-S1 product is distributed as a HDF5 file following the CF-1.8 convention with separate groups containing the data raster layers, the low-resolution correction layers, and relevant product metadata.\n", + "\n", + " - For more information about the OPERA project and other products please visit our website at https://www.jpl.nasa.gov/go/opera .\n", + "\n", + "Please refer to the [OPERA Product Specification Document](https://d2pn8kiwq2w21t.cloudfront.net/documents/OPERA_CSLC-S1_ProductSpec_v1.0.0_D-108278_Initial_2023-09-11_URS321269.pdf) for details about the CSLC-S1 product.\n", + "\n", + "*Prepared by Al Handwerger and M. Grace Bato*\n", + "\n", + "---\n", + "\n", + "## 0. Setup your conda environment\n", + "\n", + "Assuming you have conda installed. Open your terminal and run the following:\n", + "```\n", + "\n", + "# Create the OPERA CSLC environment\n", + "conda (or mamba) env create -f environment.yml\n", + "conda (or mamba) activate opera_cslc_slides\n", + "python -m ipykernel install --user --name opera_cslc_slides\n", + "\n", + "```\n", + "\n", + "---\n", + "\n", + "## 1. Load Python modules" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84a1468d-0aaf-4f06-9875-6b753d94ad42", + "metadata": {}, + "outputs": [], + "source": [ + "## Load necessary modules\n", + "%load_ext watermark\n", + "\n", + "import asf_search as asf\n", + "import cartopy.crs as ccrs\n", + "import cmcrameri.cm as cmc\n", + "import datetime as dt\n", + "import folium\n", + "import geopandas as gpd\n", + "import glob\n", + "import h5py\n", + "import imageio.v2 as imageio\n", + "import math\n", + "import matplotlib.colors as mcolors\n", + "import matplotlib.patches as patches\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from numpy.lib.stride_tricks import sliding_window_view\n", + "from numpy.typing import NDArray\n", + "import os\n", + "import pandas as pd\n", + "from pathlib import Path\n", + "from pyproj import Transformer\n", + "import rasterio\n", + "from rasterio import merge\n", + "from rasterio.crs import CRS\n", + "from rasterio.io import MemoryFile\n", + "from rasterio.transform import from_origin\n", + "from rasterio.warp import calculate_default_transform, reproject, Resampling\n", + "import requests\n", + "import re\n", + "import rioxarray\n", + "from shapely.geometry import box, Point\n", + "from shapely.ops import transform as shp_transform\n", + "import shapely\n", + "import shapely.wkt as wkt\n", + "from subprocess import Popen\n", + "from platform import system\n", + "import sys\n", + "import tempfile\n", + "import time\n", + "from tqdm.auto import tqdm\n", + "import warnings\n", + "\n", + "from affine import Affine\n", + "from concurrent.futures import ThreadPoolExecutor, as_completed\n", + "from getpass import getpass\n", + "from netrc import netrc\n", + "from opera_utils.credentials import get_earthdata_username_password\n", + "from opera_utils.disp._remote import open_file\n", + "from opera_utils.disp._utils import _get_netcdf_encoding\n", + "from osgeo import gdal\n", + "\n", + "proj_dir = os.path.join(sys.prefix, \"share\", \"proj\")\n", + "os.environ[\"PROJ_LIB\"] = proj_dir # for older PROJ\n", + "os.environ[\"PROJ_DATA\"] = proj_dir # for newer PROJ\n", + "\n", + "%watermark --iversions\n", + "\n", + "\n", + "import seaborn as sns\n", + "# ROCKET_CMAP = sns.color_palette('rocket', as_cmap=True)\n", + "from scipy.signal import convolve\n", + "from scipy.ndimage import distance_transform_edt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9952139c", + "metadata": {}, + "outputs": [], + "source": [ + "# Environment check\n", + "import sys\n", + "import importlib\n", + "\n", + "REQUIRED_PKGS = [\n", + " 'asf_search','cartopy','folium','geopandas','h5py','imageio','matplotlib','numpy','pandas',\n", + " 'pyproj','rasterio','rioxarray','shapely','xarray','opera_utils','tqdm','rich','cmcrameri','seaborn'\n", + "]\n", + "missing = []\n", + "for pkg in REQUIRED_PKGS:\n", + " try:\n", + " importlib.import_module(pkg)\n", + " except Exception:\n", + " missing.append(pkg)\n", + "\n", + "if missing:\n", + " raise ImportError(\"Missing packages: \" + ', '.join(missing) + \". Activate opera_cslc env or install from environment.yml\")\n", + "\n", + "print(f\"Python: {sys.executable}\")\n", + "# Colormap sanity check\n", + "import cmcrameri.cm as cmc\n", + "# _ = ROCKET_CMAP\n", + "\n", + "print('Environment check OK')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1faa1ef0-3d5e-424e-b602-3392b720af6d", + "metadata": {}, + "outputs": [], + "source": [ + "## Notebook display setup\n", + "%matplotlib inline\n", + "%config InlineBackend.figure_format='retina'\n", + "\n", + "# Pandas display\n", + "# pd.set_option('display.max_rows', None)\n", + "pd.set_option('display.max_columns', None)\n", + "\n", + "# Optional reprojection helper for display\n", + "\n", + "# Avoid lots of warnings printing to notebook from asf_search\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "id": "32ef4ad4", + "metadata": {}, + "source": [ + "## 2. Set up your NASA Earthdata Login Credentials" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b6ad5ac-9089-4377-be63-ddf1731263f2", + "metadata": {}, + "outputs": [], + "source": [ + "urs = 'urs.earthdata.nasa.gov'\n", + "prompts = ['Enter NASA Earthdata Login Username: ',\n", + " 'Enter NASA Earthdata Login Password: ']\n", + "\n", + "netrc_name = \"_netrc\" if system() == \"Windows\" else \".netrc\"\n", + "netrc_path = os.path.expanduser(f\"~/{netrc_name}\")\n", + "\n", + "def write_netrc():\n", + " username = getpass(prompt=prompts[0])\n", + " password = getpass(prompt=prompts[1])\n", + " with open(netrc_path, 'a') as f:\n", + " f.write(f\"\\nmachine {urs}\\n\")\n", + " f.write(f\"login {username}\\n\")\n", + " f.write(f\"password {password}\\n\")\n", + " os.chmod(netrc_path, 0o600)\n", + "\n", + "def has_urs_credentials():\n", + " try:\n", + " creds = netrc(netrc_path).authenticators(urs)\n", + " return creds is not None\n", + " except (FileNotFoundError, NetrcParseError):\n", + " return False\n", + "\n", + "if not has_urs_credentials():\n", + " if not os.path.exists(netrc_path):\n", + " open(netrc_path, 'w').close()\n", + " write_netrc()\n", + "\n", + "os.environ[\"GDAL_HTTP_NETRC\"] = \"YES\"\n", + "os.environ[\"GDAL_HTTP_NETRC_FILE\"] = netrc_path" + ] + }, + { + "cell_type": "markdown", + "id": "4f948345", + "metadata": {}, + "source": [ + "## 3. Enter user-defined parameters\n", + "\n", + "**Parameter guide (impact on downstream steps):**\n", + "- **AOI / orbit / path / burst filters**: control which CSLCs are found and subset; changing these changes *all* downstream data.\n", + "- **dateStart / dateEnd**: controls query window and calendar/GIF coverage.\n", + "- **pair_mode / pair_t_span_days**: controls which interferometric pairs are formed; impacts IFG/COH density.\n", + "- **MULTILOOK / TARGET_PIXEL_M**: sets spatial averaging; affects resolution and noise level in IFG/COH/BSC.\n", + "- **CALIBRATION_MODE**: choose backscatter calibration: `sigma0` or `beta0`.\n", + "- **COH_METHOD**: `standard` (default) or `phase_only` (previous behavior).\n", + "- **COH_APPLY_LEE**: apply Lee filter to coherence (optional smoothing).\n", + "- **IFG_APPLY_FILTER**: apply Goldstein filter to interferogram phase.\n", + "- **Note**: `IFG_APPLY_FILTER` only affects the **interferogram phase** (for display/plots). Coherence is always computed from the unfiltered data; use `COH_METHOD` to choose standard vs phase-only, and `COH_APPLY_LEE` for optional smoothing.\n", + "- **MERGE_BLEND_OVERLAP**: if `True`, blend overlap regions between bursts using a distance‑based feather to reduce seams.\n", + "- **MERGE_BLEND_EPS**: small constant to avoid divide‑by‑zero in blend weights. Increase slightly if you see artifacts.\n", + "- **APPLY_WATER_MASK / WATER_MASK_PATH**: masks out water before saving outputs.\n", + "- **SAVE_WGS84**: optional WGS84 GeoTIFFs for easy display.\n", + "- **SKIP_EXISTING_TIFS**: reuses existing GeoTIFFs to speed re-runs (delete old files to force regeneration).\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5eeba6e", + "metadata": {}, + "outputs": [], + "source": [ + "# User parameters (edit these)\n", + "# AOI is a WKT polygon in EPSG:4326\n", + "# check ASF Vertex to look for correct pass/path for your AOI https://search.asf.alaska.edu/#/?maxResults=250&dataset=OPERA-S1&polygon=POLYGON((-118.3955%2033.7342,-118.3464%2033.7342,-118.3464%2033.7616,-118.3955%2033.7616,-118.3955%2033.7342))&productTypes=CSLC&resultsLoaded=true&granule=OPERA_L2_CSLC-S1_T071-151230-IW3_20260128T135247Z_20260129T074245Z_S1A_VV_v1.1&zoom=10.065¢er=-118.699,33.446\n", + "## Enter user-defined parameters\n", + "SITE_NAME = \"Palos_Verdes_Landslides\" # used for output folder naming\n", + "aoi = \"POLYGON((-118.3955 33.7342,-118.3464 33.7342,-118.3464 33.7616,-118.3955 33.7616,-118.3955 33.7342))\"\n", + "orbitPass = \"DESCENDING\" # ASCENDING or DESCENDING\n", + "pathNumber = 71 #71 DESC and 64 ASC\n", + "# Optional burst selection before download\n", + "# Use subswath (e.g., 'IW2', 'IW3') or specific OPERA burst ID (e.g., 'T071_151230_IW3')\n", + "BURST_SUBSWATH = 'IW3' # e.g., 'IW2' or ['IW2', 'IW3'] or None \n", + "# BURST_SUBSWATH = 'IW3' # e.g., 'IW2' or ['IW2', 'IW3'] or None \n", + "\n", + "BURST_ID = None # e.g., 'T071_151230_IW3' or list of burst IDs\n", + "\n", + "dateStart = dt.datetime.fromisoformat('2024-10-01 00:00:00') #'YY/YY-MM-DD HH:MM:SS'\n", + "dateEnd = dt.datetime.fromisoformat('2025-09-30 23:59:59') #'YYYY-MM-DD HH:MM:SS'\n", + "\n", + "# Pairing options\n", + "pair_mode = 't_span' # 'all' or 't_span'\n", + "pair_t_span_days = [12] # int or list of ints (e.g., [6, 12, 24])\n", + "\n", + "# Seasonal filters (exclude pairs if either endpoint falls in these windows)\n", + "EXCLUDE_DATE_RANGES = [] # list of (start, end) like [(\"2023-12-15\",\"2024-03-31\")] or []\n", + "EXCLUDE_MONTHDAY_RANGES = [] # list of (\"MM-DD\",\"MM-DD\") e.g., [(\"12-15\",\"02-15\")], or []\n", + "\n", + "#backscatter\n", + "CALIBRATION_MODE = 'sigma0' # 'sigma0' or 'beta0'\n", + "CALIBRATION_FACTOR_IS_AMPLITUDE = True # False: LUT applies to power (default); True: LUT applies to amplitude, so use squared factor\n", + "\n", + "# Multilooking (spatial averaging)\n", + "# Set either MULTILOOK (looks_y, looks_x) OR TARGET_PIXEL_M (meters). TARGET overrides MULTILOOK.\n", + "MULTILOOK = (1, 1) # e.g., (3, 6) for 30m from (dy=10m, dx=5m)\n", + "TARGET_PIXEL_M = None # e.g., 30.0 meter or 90.0 meter or None\n", + "GOLDSTEIN_ALPHA = 0.5 # 0 (none) to 1 (strong)\n", + "COH_METHOD = 'standard' # 'standard' or 'phase_only'\n", + "COH_USE_GAMMA_NORM = True # True to apply gamma normalization in plots (coherence only)\n", + "COH_NORM_GAMMA = 0.5\n", + "COH_KERNEL = 'boxcar' # 'boxcar' or 'weighted'\n", + "COH_KERNEL_NUM_CONV = 3 # only used for weighted kernel\n", + "COH_APPLY_LEE = False # apply Lee filter to coherence\n", + "IFG_APPLY_FILTER = True # apply Goldstein filter to interferogram phase\n", + "\n", + "#for burst overlaps\n", + "MERGE_BLEND_OVERLAP = True # feather overlaps to reduce burst overlaps\n", + "MERGE_BLEND_EPS = 1e-6\n", + "\n", + "SAVE_WGS84 = False # set True to save WGS84 GeoTIFF mosaics\n", + "SKIP_EXISTING_TIFS = False # skip writing GeoTIFFs if they already exist\n", + "\n", + "DOWNLOAD_WITH_PROGRESS = True # set True for per-file progress bar\n", + "\n", + "\n", + "min_t_span_days = 0 # minimum separation (days)\n", + "max_t_span_days = 12 # maximum separation (days) or None\n", + "max_pairs_per_burst = None # int or None\n", + "max_pairs_total = None # int or None\n", + "\n", + "# Calendar settings\n", + "USE_WATER_YEAR = True # True: Oct–Sep, False: Jan–Dec\n", + "\n", + "DOWNLOAD_PROCESSES = min(8, max(2, (os.cpu_count() or 4) // 2))\n", + "# DOWNLOAD_BATCH_SIZE = 5\n", + "\n", + "\n", + "# Normalize name for filesystem (letters/numbers/_/- only)\n", + "site_slug = re.sub(r\"[^A-Za-z0-9_-]+\", \"\", SITE_NAME)\n", + "orbit_code = orbitPass[0].upper() # 'A' or 'D'\n", + "savedir = f'./{site_slug}_{orbit_code}{pathNumber:03d}/'\n", + "\n", + "# Water mask options\n", + "# in params cell\n", + "WATER_MASK_PATH = f\"{savedir}/water_mask/water_mask_esa_wc2021.tif\"\n", + "APPLY_WATER_MASK = True" + ] + }, + { + "cell_type": "markdown", + "id": "80842b39", + "metadata": {}, + "source": [ + "**AOI size guidance**\n", + "\n", + "This notebook is tuned for *small* AOIs (e.g., individual landslides). The main opera-utils release includes fixes for small AOI subsetting. Large AOIs can dramatically increase download time, disk usage, and RAM needs. If you use a large polygon, consider:\n", + "- Shorter date ranges or fewer bursts\n", + "- Coarser `TARGET_PIXEL_M` (more multilooking)\n", + "- Tiling the AOI into smaller polygons and mosaicking later\n", + "\n", + "If you see memory errors or very long runtimes, reduce the AOI size or date range.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89fa0047", + "metadata": {}, + "outputs": [], + "source": [ + "# Utilities (used by multiple sections)\n", + "\n", + "def _load_water_mask_match(mask_path, shape, transform, crs):\n", + " if mask_path is None or not Path(mask_path).exists():\n", + " return None\n", + " with rasterio.open(mask_path) as src:\n", + " src_mask = src.read(1)\n", + " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", + " return src_mask\n", + " dst = np.zeros(shape, dtype=src_mask.dtype)\n", + " reproject(\n", + " source=src_mask,\n", + " destination=dst,\n", + " src_transform=src.transform,\n", + " src_crs=src.crs,\n", + " dst_transform=transform,\n", + " dst_crs=crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " return dst\n", + "\n", + "\n", + "def _apply_water_mask(arr, mask):\n", + " # mask: 1 = keep land, 0 = water\n", + " return np.where(mask == 0, np.nan, arr)\n", + "\n", + "\n", + "def _trim_nan_border(arr, transform):\n", + " data = arr[0] if arr.ndim == 3 else arr\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if not mask.any():\n", + " return arr, transform\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " data = data[r0:r1, c0:c1]\n", + " if arr.ndim == 3:\n", + " arr = data[None, ...]\n", + " else:\n", + " arr = data\n", + " new_transform = transform * Affine.translation(c0, r0)\n", + " return arr, new_transform\n", + "\n", + "\n", + "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", + " dst_crs = 'EPSG:4326'\n", + " dst_transform, width, height = calculate_default_transform(\n", + " f'EPSG:{epsg}', dst_crs, mosaic.shape[2], mosaic.shape[1],\n", + " *rasterio.transform.array_bounds(mosaic.shape[1], mosaic.shape[2], transform)\n", + " )\n", + " dest = np.zeros((1, height, width), dtype=mosaic.dtype)\n", + " reproject(\n", + " source=mosaic,\n", + " destination=dest,\n", + " src_transform=transform,\n", + " src_crs=f'EPSG:{epsg}',\n", + " dst_transform=dst_transform,\n", + " dst_crs=dst_crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " out_meta = {\n", + " 'driver': 'GTiff',\n", + " 'height': height,\n", + " 'width': width,\n", + " 'count': 1,\n", + " 'dtype': mosaic.dtype,\n", + " 'crs': dst_crs,\n", + " 'transform': dst_transform,\n", + " }\n", + " with rasterio.open(out_path, 'w', **out_meta) as dst:\n", + " dst.write(dest)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f072529", + "metadata": {}, + "outputs": [], + "source": [ + "# ESA WorldCover 2021 water mask (GDAL-only)\n", + "\n", + "ESA_WC_GRID_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/esa_worldcover_grid.fgb\"\n", + "ESA_WC_BASE_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/v200/2021/map\"\n", + "# ESA WorldCover class codes: 80 = Permanent water bodies\n", + "ESA_WC_WATER_CLASSES = {80}\n", + "\n", + "\n", + "def build_worldcover_water_mask(aoi_wkt, out_path, target_res_deg=None):\n", + " # Create a binary land mask from ESA WorldCover (1=land, 0=water).\n", + " out_path = Path(out_path)\n", + " out_path.parent.mkdir(parents=True, exist_ok=True)\n", + " if out_path.exists():\n", + " return out_path\n", + "\n", + "\n", + " aoi_geom = shapely.wkt.loads(aoi_wkt)\n", + " # Load tile grid and select intersecting tiles\n", + " grid = gpd.read_file(ESA_WC_GRID_URL)\n", + " grid = grid.to_crs(\"EPSG:4326\")\n", + " # Find tile id column (varies by grid version)\n", + " tile_col = next((c for c in grid.columns if 'tile' in c.lower()), None)\n", + " if tile_col is None:\n", + " raise RuntimeError(f\"No tile column found in grid columns: {list(grid.columns)}\")\n", + " tiles = grid[grid.intersects(aoi_geom)][tile_col].tolist()\n", + " if not tiles:\n", + " raise RuntimeError(\"No WorldCover tiles intersect AOI\")\n", + "\n", + " print(f\"Selected tiles: {tiles}\")\n", + "\n", + " tile_urls = [\n", + " f\"{ESA_WC_BASE_URL}/ESA_WorldCover_10m_2021_v200_{t}_Map.tif\"\n", + " for t in tiles\n", + " ]\n", + "\n", + " # Quick URL check for first tile\n", + " first_url = tile_urls[0]\n", + " try:\n", + " _ = gdal.Open(first_url)\n", + " except Exception as e:\n", + " raise RuntimeError(f\"GDAL cannot open first tile URL: {first_url}\\n{e}\")\n", + "\n", + " vrt_path = out_path.with_suffix(\".vrt\")\n", + " gdal.BuildVRT(str(vrt_path), tile_urls)\n", + "\n", + " minx, miny, maxx, maxy = aoi_geom.bounds\n", + " warp_kwargs = dict(\n", + " format=\"GTiff\",\n", + " outputBounds=[minx, miny, maxx, maxy],\n", + " multithread=True,\n", + " )\n", + " if target_res_deg is not None:\n", + " warp_kwargs.update(dict(xRes=target_res_deg, yRes=target_res_deg, targetAlignedPixels=True))\n", + "\n", + " tmp_map = out_path.with_name(out_path.stem + \"_map.tif\")\n", + " warp_ds = gdal.Warp(str(tmp_map), str(vrt_path), **warp_kwargs)\n", + " if warp_ds is None:\n", + " raise RuntimeError(\"GDAL Warp returned None. Check network access/URL.\")\n", + " warp_ds = None\n", + "\n", + " with rasterio.open(tmp_map) as src:\n", + " data = src.read(1)\n", + " profile = src.profile\n", + "\n", + " mask = (~np.isin(data, list(ESA_WC_WATER_CLASSES))).astype(\"uint8\")\n", + " profile.update(dtype=\"uint8\", count=1, nodata=0)\n", + "\n", + " with rasterio.open(out_path, \"w\", **profile) as dst:\n", + " dst.write(mask, 1)\n", + "\n", + " return out_path\n", + "\n", + "\n", + "if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " WATER_MASK_PATH = build_worldcover_water_mask(aoi, WATER_MASK_PATH)" + ] + }, + { + "cell_type": "markdown", + "id": "98916bef", + "metadata": {}, + "source": [ + "## 4. Query OPERA CSLCs using `asf_search`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e33d72e-2e75-4099-93d6-5cd058643e35", + "metadata": {}, + "outputs": [], + "source": [ + "## Search for OPERA CSLC data in ASF DAAC\n", + "try:\n", + " search_params = dict(\n", + " intersectsWith= aoi,\n", + " dataset='OPERA-S1',\n", + " processingLevel='CSLC',\n", + " flightDirection = orbitPass,\n", + " start=dateStart,\n", + " end=dateEnd)\n", + "\n", + " ## Return results\n", + " results = asf.search(**search_params)\n", + " print(f\"Length of Results: {len(results)}\")\n", + "\n", + "except TypeError:\n", + " search_params = dict(\n", + " intersectsWith= aoi.wkt,\n", + " dataset='OPERA-S1',\n", + " processingLevel='CSLC',\n", + " flightDirection = orbitPass,\n", + " start=dateStart,\n", + " end=dateEnd)\n", + "\n", + " ## Return results\n", + " results = asf.search(**search_params)\n", + " print(f\"Length of Results: {len(results)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2f5b5df", + "metadata": {}, + "outputs": [], + "source": [ + "## Save the results in a geopandas dataframe\n", + "gf = gpd.GeoDataFrame.from_features(results.geojson(), crs='EPSG:4326')\n", + "\n", + "## Filter data based on specified track number\n", + "gf = gf[gf.pathNumber==pathNumber]\n", + "# gf = gf[gf.pgeVersion==\"2.1.1\"] " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe3a0963-1d13-4b99-955a-b2c0fa45f1e3", + "metadata": {}, + "outputs": [], + "source": [ + "# Get only relevant metadata\n", + "cslc_df = gf[['operaBurstID', 'fileID', 'startTime', 'stopTime', 'url', 'geometry', 'pgeVersion']]\n", + "cslc_df['startTime'] = pd.to_datetime(cslc_df.startTime).dt.date\n", + "cslc_df['stopTime'] = pd.to_datetime(cslc_df.stopTime).dt.date\n", + "\n", + "# Extract production time from fileID (2nd date token)\n", + "def _prod_time_from_fileid(file_id):\n", + " # Example: OPERA_L2_CSLC-S1_..._20221122T161650Z_20240504T081640Z_...\n", + " parts = str(file_id).split('_')\n", + " return parts[5] if len(parts) > 5 else None\n", + "\n", + "cslc_df['productionTime'] = pd.to_datetime(cslc_df['fileID'].apply(_prod_time_from_fileid), format='%Y%m%dT%H%M%SZ', errors='coerce')\n", + "\n", + "# Keep newest duplicate by productionTime (fallback to pgeVersion, stopTime)\n", + "cslc_df = cslc_df.sort_values(by=['operaBurstID', 'startTime', 'productionTime', 'pgeVersion', 'stopTime'])\n", + "cslc_df = cslc_df.drop_duplicates(subset=['operaBurstID', 'startTime'], keep='last', ignore_index=True)\n", + "\n", + "\n", + "def _subswath_from_fileid(file_id):\n", + " # Example: ...-IW2_... -> IW2\n", + " m = re.search(r\"-IW[1-3]_\", str(file_id))\n", + " return m.group(0)[1:4] if m else None\n", + "\n", + "cslc_df['burstSubswath'] = cslc_df['fileID'].apply(_subswath_from_fileid)\n", + "\n", + "# Optional filtering by subswath or specific burst IDs\n", + "if BURST_SUBSWATH:\n", + " if isinstance(BURST_SUBSWATH, (list, tuple, set)):\n", + " subswaths = {str(s).upper() for s in BURST_SUBSWATH}\n", + " else:\n", + " subswaths = {str(BURST_SUBSWATH).upper()}\n", + " cslc_df = cslc_df[cslc_df['burstSubswath'].str.upper().isin(subswaths)]\n", + "\n", + "if BURST_ID:\n", + " if isinstance(BURST_ID, (list, tuple, set)):\n", + " burst_ids = {str(b) for b in BURST_ID}\n", + " else:\n", + " burst_ids = {str(BURST_ID)}\n", + " cslc_df = cslc_df[cslc_df['operaBurstID'].isin(burst_ids)]\n", + "cslc_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76cb1007", + "metadata": {}, + "outputs": [], + "source": [ + "# Build AOI geometry\n", + "aoi_geom = wkt.loads(aoi)\n", + "aoi_gdf = gpd.GeoDataFrame(geometry=[aoi_geom], crs=\"EPSG:4326\")\n", + "\n", + "# Map center\n", + "centroid = aoi_gdf.geometry[0].centroid\n", + "m = folium.Map(location=[centroid.y, centroid.x], zoom_start=9, tiles=\"Esri.WorldImagery\")\n", + "\n", + "# Add CSLC footprints\n", + "folium.GeoJson(\n", + " data=cslc_df[['operaBurstID','geometry']].set_geometry('geometry').to_crs(\"EPSG:4326\").__geo_interface__,\n", + " name=\"CSLC footprints\",\n", + " style_function=lambda x: {\n", + " \"fillColor\": \"blue\",\n", + " \"color\": \"blue\",\n", + " \"weight\": 2,\n", + " \"fillOpacity\": 0.1,\n", + " },\n", + ").add_to(m)\n", + "\n", + "# Add AOI\n", + "folium.GeoJson(\n", + " data=aoi_gdf.__geo_interface__,\n", + " name=\"AOI\",\n", + " style_function=lambda x: {\n", + " \"fillColor\": \"red\",\n", + " \"color\": \"red\",\n", + " \"weight\": 2,\n", + " \"fillOpacity\": 0.1,\n", + " },\n", + ").add_to(m)\n", + "\n", + "folium.LayerControl().add_to(m)\n", + "\n", + "m" + ] + }, + { + "cell_type": "markdown", + "id": "9ca8a611", + "metadata": {}, + "source": [ + "## 5. Download the CSLC-S1 locally" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aea6a5b7", + "metadata": {}, + "outputs": [], + "source": [ + "## Download step skipped: CSLC subsets are streamed via opera-utils\n", + "print('Skipping full CSLC downloads; using opera-utils HTTP subsetting.')\n", + "\n", + "# Sort the CSLC-S1 by burstID and date\n", + "cslc_df = cslc_df.sort_values(by=[\"operaBurstID\", \"startTime\"], ignore_index=True)\n", + "cslc_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa4801fc", + "metadata": {}, + "outputs": [], + "source": [ + "# Enforce date range on dataframe (useful when re-running with narrower dates)\n", + "date_start_day = dateStart.date()\n", + "date_end_day = dateEnd.date()\n", + "cslc_df = cslc_df[(cslc_df['startTime'] >= date_start_day) & (cslc_df['startTime'] <= date_end_day)]\n", + "cslc_df = cslc_df.reset_index(drop=True)\n", + "cslc_df" + ] + }, + { + "cell_type": "markdown", + "id": "48add9c3", + "metadata": {}, + "source": [ + "## 6. Read each CSLC-S1 and stack them together\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01891a96", + "metadata": {}, + "outputs": [], + "source": [ + "cslc_stack = []; cslc_dates = []; bbox_stack = []; xcoor_stack = []; ycoor_stack = []\n", + "\n", + "\n", + "subset_dir = f\"{savedir}/subset_cslc\"\n", + "os.makedirs(subset_dir, exist_ok=True)\n", + "\n", + "def _extract_subset(input_obj, outpath, rows, cols, chunks=(1,256,256)):\n", + " X0, X1 = (cols.start, cols.stop) if cols is not None else (None, None)\n", + " Y0, Y1 = (rows.start, rows.stop) if rows is not None else (None, None)\n", + " ds = xr.open_dataset(input_obj, engine=\"h5netcdf\", group=\"data\")\n", + " if 'VV' not in ds.data_vars:\n", + " raise ValueError('Source missing VV data')\n", + " subset = ds.isel(y_coordinates=slice(Y0, Y1), x_coordinates=slice(X0, X1))\n", + " subset.to_netcdf(\n", + " outpath,\n", + " engine=\"h5netcdf\",\n", + " group=\"data\",\n", + " encoding=_get_netcdf_encoding(subset, chunks=chunks),\n", + " )\n", + " for group in (\"metadata\", \"identification\"):\n", + " with h5py.File(input_obj) as hf, h5py.File(outpath, \"a\") as dest_hf:\n", + " hf.copy(group, dest_hf, name=group)\n", + " with h5py.File(outpath, \"a\") as hf:\n", + " ctype = h5py.h5t.py_create(np.complex64)\n", + " ctype.commit(hf[\"/\"].id, np.bytes_(\"complex64\"))\n", + "\n", + "def _subset_h5_to_disk(url, aoi_wkt, out_dir):\n", + " outpath = Path(out_dir) / Path(url).name\n", + " if outpath.exists():\n", + " if outpath.stat().st_size < 100 * 1024:\n", + " outpath.unlink()\n", + " else:\n", + " return outpath\n", + "\n", + " # determine row/col slices by reading coords\n", + " with open_file(url) as in_f:\n", + " ds = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", + " xcoor = ds[\"x_coordinates\"].values\n", + " ycoor = ds[\"y_coordinates\"].values\n", + " epsg = int(ds[\"projection\"].values)\n", + "\n", + " aoi_geom = wkt.loads(aoi_wkt)\n", + " if epsg != 4326:\n", + " transformer = Transformer.from_crs('EPSG:4326', f'EPSG:{epsg}', always_xy=True)\n", + " aoi_geom = shp_transform(transformer.transform, aoi_geom)\n", + " minx, miny, maxx, maxy = aoi_geom.bounds\n", + " x_mask = (xcoor >= minx) & (xcoor <= maxx)\n", + " y_mask = (ycoor >= miny) & (ycoor <= maxy)\n", + " if not x_mask.any() or not y_mask.any():\n", + " raise ValueError('AOI does not intersect this CSLC extent')\n", + " ix = np.where(x_mask)[0]\n", + " iy = np.where(y_mask)[0]\n", + " rows = slice(iy.min(), iy.max()+1)\n", + " cols = slice(ix.min(), ix.max()+1)\n", + "\n", + " if url.startswith('s3://'):\n", + " with open_file(url) as in_f:\n", + " _extract_subset(in_f, outpath, rows, cols)\n", + " else:\n", + " # HTTPS: download to temp then subset\n", + " with tempfile.NamedTemporaryFile(suffix='.h5') as tf:\n", + " if url.startswith('http'):\n", + " session = requests.Session()\n", + " username, password = get_earthdata_username_password()\n", + " session.auth = (username, password)\n", + " resp = session.get(url, stream=True)\n", + " resp.raise_for_status()\n", + " content_type = resp.headers.get('Content-Type', '').lower()\n", + " if 'text/html' in content_type:\n", + " raise ValueError('Got HTML response instead of HDF5; check Earthdata login/auth')\n", + " for chunk in resp.iter_content(chunk_size=1024 * 1024):\n", + " if chunk:\n", + " tf.write(chunk)\n", + " tf.flush()\n", + " if tf.tell() < 100 * 1024:\n", + " raise ValueError('Downloaded file too small; likely auth/redirect issue')\n", + " _extract_subset(tf.name, outpath, rows, cols)\n", + " # validate output has VV; remove tiny/invalid outputs\n", + " try:\n", + " with h5py.File(outpath, 'r') as h5:\n", + " if '/data/VV' not in h5:\n", + " raise ValueError('Subset missing /data/VV')\n", + " if outpath.stat().st_size < 100 * 1024:\n", + " raise ValueError('Subset file too small')\n", + " except Exception:\n", + " if Path(outpath).exists():\n", + " Path(outpath).unlink()\n", + " raise\n", + " return outpath\n", + "\n", + "def _load_subset(file_id, url, start_date):\n", + " try:\n", + " outpath = _subset_h5_to_disk(url, aoi, subset_dir)\n", + " except FileNotFoundError:\n", + " return None # skip missing products\n", + " # now read subset locally with h5py (fast)\n", + " with h5py.File(outpath, 'r') as h5:\n", + " cslc = h5['/data/VV'][:]\n", + " xcoor = h5['/data/x_coordinates'][:]\n", + " ycoor = h5['/data/y_coordinates'][:]\n", + " dx = int(h5['/data/x_spacing'][()])\n", + " dy = int(h5['/data/y_spacing'][()])\n", + " epsg = int(h5['/data/projection'][()])\n", + " sensing_start = h5['/metadata/processing_information/input_burst_metadata/sensing_start'][()].astype(str)\n", + " sensing_stop = h5['/metadata/processing_information/input_burst_metadata/sensing_stop'][()].astype(str)\n", + " dims = h5['/metadata/processing_information/input_burst_metadata/shape'][:]\n", + " bounding_polygon = h5['/identification/bounding_polygon'][()].astype(str)\n", + " orbit_direction = h5['/identification/orbit_pass_direction'][()].astype(str)\n", + " center_lon, center_lat = h5['/metadata/processing_information/input_burst_metadata/center']\n", + " wavelength = h5['/metadata/processing_information/input_burst_metadata/wavelength'][()].astype(str)\n", + " subset_bbox = [float(xcoor.min()), float(xcoor.max()), float(ycoor.min()), float(ycoor.max())]\n", + " return cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox\n", + "\n", + "# Subset with progress (parallel)\n", + "\n", + "items = list(zip(cslc_df.fileID, cslc_df.url, cslc_df.startTime))\n", + "import xarray as xr\n", + "# Diagnostic: check pixel spacing before multilooking\n", + "with open_file(items[0][1]) as in_f:\n", + " ds0 = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", + " dx0 = float(ds0[\"x_spacing\"].values)\n", + " dy0 = float(ds0[\"y_spacing\"].values)\n", + "print(f\"Pixel spacing (dx, dy) = ({dx0}, {dy0})\")\n", + "\n", + "# Derive anisotropic coherence window from TARGET_PIXEL_M or MULTILOOK\n", + "def _odd_at_least_one(n):\n", + " n = max(1, int(round(n)))\n", + " return n if n % 2 == 1 else n + 1\n", + "\n", + "# Aim for ~60 m window; if multilook pixel size exceeds 60 m, use that instead\n", + "if TARGET_PIXEL_M is not None:\n", + " base_win_m = TARGET_PIXEL_M\n", + "else:\n", + " base_win_m = max(abs(dx0) * MULTILOOK[1], abs(dy0) * MULTILOOK[0])\n", + "\n", + "COH_WIN_M = max(60.0, base_win_m)\n", + "COH_WIN_X = _odd_at_least_one(COH_WIN_M / abs(dx0))\n", + "COH_WIN_Y = _odd_at_least_one(COH_WIN_M / abs(dy0))\n", + "print(f\"Using anisotropic COH_WIN (Y,X)=({COH_WIN_Y},{COH_WIN_X}) from COH_WIN_M={COH_WIN_M} m\")\n", + "\n", + "# Derive MULTILOOK from TARGET_PIXEL_M if provided\n", + "if TARGET_PIXEL_M is not None:\n", + " looks_x = max(1, int(round(TARGET_PIXEL_M / abs(dx0))))\n", + " looks_y = max(1, int(round(TARGET_PIXEL_M / abs(dy0))))\n", + " MULTILOOK = (looks_y, looks_x)\n", + " print(f\"Using MULTILOOK={MULTILOOK} for TARGET_PIXEL_M={TARGET_PIXEL_M} (dx={dx0}, dy={dy0})\")\n", + "\n", + "\n", + "results = [None] * len(items)\n", + "_t0 = time.perf_counter()\n", + "with ThreadPoolExecutor(max_workers=DOWNLOAD_PROCESSES) as ex:\n", + " futures = {ex.submit(_load_subset, fileID, url, start_date): i for i, (fileID, url, start_date) in enumerate(items)}\n", + " for fut in tqdm(as_completed(futures), total=len(futures), desc='Subsetting CSLC'):\n", + " i = futures[fut]\n", + " results[i] = fut.result()\n", + "_t1 = time.perf_counter()\n", + "print(f\"Subset/download time: {_t1 - _t0:.1f} s\")\n", + "\n", + "valid_idx = [i for i, res in enumerate(results) if res is not None]\n", + "if len(valid_idx) != len(results):\n", + " print(f\"Skipping {len(results) - len(valid_idx)} failed subsets\")\n", + " cslc_df = cslc_df.iloc[valid_idx].reset_index(drop=True)\n", + " results = [results[i] for i in valid_idx]\n", + "\n", + "for (fileID, start_date), res in zip(zip(cslc_df.fileID, cslc_df.startTime), results):\n", + " cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox = res\n", + " cslc_stack.append(cslc)\n", + " cslc_dates.append(pd.to_datetime(sensing_start).date())\n", + " if subset_bbox is not None:\n", + " bbox = subset_bbox\n", + " else:\n", + " cslc_poly = wkt.loads(bounding_polygon)\n", + " bbox = [cslc_poly.bounds[0], cslc_poly.bounds[2], cslc_poly.bounds[1], cslc_poly.bounds[3]]\n", + " bbox_stack.append(bbox)\n", + " xcoor_stack.append(xcoor)\n", + " ycoor_stack.append(ycoor)" + ] + }, + { + "cell_type": "markdown", + "id": "f5e97e45", + "metadata": {}, + "source": [ + "
\n", + "7. Generate the interferograms, compute for the coherence, save the files as GeoTiffs\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d87c1f9", + "metadata": {}, + "outputs": [], + "source": [ + "import h5py, os, glob\n", + "f = sorted(glob.glob(f\"{savedir}/subset_cslc/*.h5\"))[0]\n", + "print(f, os.path.getsize(f))\n", + "with h5py.File(f, \"r\") as h5:\n", + " print(list(h5[\"/data\"].keys()))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57c74ac4", + "metadata": {}, + "outputs": [], + "source": [ + "def colorize(array=[], cmap='RdBu', cmin=[], cmax=[]):\n", + " normed_data = (array - cmin) / (cmax - cmin) \n", + " cm = plt.cm.get_cmap(cmap)\n", + " return cm(normed_data) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25c56034", + "metadata": {}, + "outputs": [], + "source": [ + "def goldstein_filter(ifg_cpx, alpha=0.5, pad=32, edge_trim=16):\n", + " # Goldstein filter with padding + taper + mask to reduce edge effects\n", + " mask = np.isfinite(ifg_cpx)\n", + " data = np.nan_to_num(ifg_cpx, nan=0.0)\n", + " if pad and pad > 0:\n", + " data = np.pad(data, ((pad, pad), (pad, pad)), mode=\"reflect\")\n", + " mask = np.pad(mask, ((pad, pad), (pad, pad)), mode=\"constant\", constant_values=False)\n", + " # Apply 2D Hann window (taper)\n", + " wy = np.hanning(data.shape[0])\n", + " wx = np.hanning(data.shape[1])\n", + " window = wy[:, None] * wx[None, :]\n", + " f = np.fft.fft2(data * window)\n", + " s = np.abs(f)\n", + " s = s / (s.max() + 1e-8)\n", + " f_filt = f * (s ** alpha)\n", + " out = np.fft.ifft2(f_filt)\n", + " if pad and pad > 0:\n", + " out = out[pad:-pad, pad:-pad]\n", + " mask = mask[pad:-pad, pad:-pad]\n", + " # restore NaNs outside valid mask\n", + " out[~mask] = np.nan\n", + " if edge_trim and edge_trim > 0:\n", + " out[:edge_trim, :] = np.nan\n", + " out[-edge_trim:, :] = np.nan\n", + " out[:, :edge_trim] = np.nan\n", + " out[:, -edge_trim:] = np.nan\n", + " return out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82bda5ae", + "metadata": {}, + "outputs": [], + "source": [ + "def rasterWrite(outtif,arr,transform,epsg,dtype='float32'):\n", + " #writing geotiff using rasterio\n", + " \n", + " new_dataset = rasterio.open(outtif, 'w', driver='GTiff',\n", + " height = arr.shape[0], width = arr.shape[1],\n", + " count=1, dtype=dtype,\n", + " crs=CRS.from_epsg(epsg),\n", + " transform=transform,nodata=np.nan)\n", + " new_dataset.write(arr, 1)\n", + " new_dataset.close() " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b47f82a5", + "metadata": {}, + "outputs": [], + "source": [ + "## Build date pairs per burstID\n", + "cslc_dates = cslc_df[[\"startTime\"]]\n", + "burstID = cslc_df.operaBurstID.drop_duplicates(ignore_index=True)\n", + "n_unique_burstID = len(burstID)\n", + "\n", + "def _lag_list(lag):\n", + " if lag is None:\n", + " return []\n", + " if isinstance(lag, (list, tuple, set)):\n", + " return sorted({int(x) for x in lag})\n", + " return [int(lag)]\n", + "\n", + "pair_lags = _lag_list(pair_t_span_days)\n", + "pair_indices = [] # list of (ref_idx, sec_idx) in cslc_df order\n", + "\n", + "for bid, group in cslc_df.groupby('operaBurstID'):\n", + " group = group.sort_values('startTime')\n", + " idx = group.index.to_list()\n", + " dates = group['startTime'].to_list()\n", + "\n", + " burst_pairs = []\n", + " if pair_mode == 'all':\n", + " for i in range(len(idx)):\n", + " for j in range(i+1, len(idx)):\n", + " delta = (dates[j] - dates[i]).days\n", + " if delta < min_t_span_days:\n", + " continue\n", + " if max_t_span_days is not None and delta > max_t_span_days:\n", + " continue\n", + " burst_pairs.append((idx[i], idx[j]))\n", + " elif pair_mode == 't_span':\n", + " for i in range(len(idx)):\n", + " for j in range(i+1, len(idx)):\n", + " delta = (dates[j] - dates[i]).days\n", + " if delta in pair_lags and delta >= min_t_span_days and (max_t_span_days is None or delta <= max_t_span_days):\n", + " burst_pairs.append((idx[i], idx[j]))\n", + " else:\n", + " raise ValueError(\"pair_mode must be 'all' or 't_span'\")\n", + "\n", + " if max_pairs_per_burst is not None:\n", + " burst_pairs = burst_pairs[:int(max_pairs_per_burst)]\n", + "\n", + " pair_indices.extend(burst_pairs)\n", + "\n", + "if max_pairs_total is not None:\n", + " pair_indices = pair_indices[:int(max_pairs_total)]\n", + "\n", + "# Sort pairs by date, then burstID (so same dates group together)\n", + "def _pair_sort_key(pair):\n", + " ref_idx, sec_idx = pair\n", + " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", + " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", + " burst = cslc_df.operaBurstID.iloc[ref_idx]\n", + " return (ref_date, sec_date, burst)\n", + "pair_indices = sorted(pair_indices, key=_pair_sort_key)\n", + "print(f'Pair count: {len(pair_indices)}')\n", + "\n", + "# Seasonal filtering based on pair endpoints\n", + "from datetime import date\n", + "\n", + "# Normalize exclude ranges to date objects\n", + "_excl_ranges = []\n", + "for s, e in EXCLUDE_DATE_RANGES:\n", + " try:\n", + " s_d = pd.to_datetime(s).date()\n", + " e_d = pd.to_datetime(e).date()\n", + " except Exception:\n", + " continue\n", + " _excl_ranges.append((s_d, e_d))\n", + "\n", + "# Normalize month-day ranges\n", + "if EXCLUDE_MONTHDAY_RANGES:\n", + " if len(EXCLUDE_MONTHDAY_RANGES) == 2 and all(isinstance(x, str) for x in EXCLUDE_MONTHDAY_RANGES):\n", + " EXCLUDE_MONTHDAY_RANGES = [(EXCLUDE_MONTHDAY_RANGES[0], EXCLUDE_MONTHDAY_RANGES[1])]\n", + "\n", + "# Filter pairs: exclude if either endpoint falls in excluded months or ranges\n", + "filtered_pairs = []\n", + "for r, s in pair_indices:\n", + " d1 = pd.to_datetime(cslc_dates.iloc[r].values[0]).date()\n", + " d2 = pd.to_datetime(cslc_dates.iloc[s].values[0]).date()\n", + "\n", + " # Date-range exclusion\n", + " excluded = False\n", + " for rs, re in _excl_ranges:\n", + " if rs <= d1 <= re or rs <= d2 <= re:\n", + " excluded = True\n", + " break\n", + " if excluded:\n", + " continue\n", + "\n", + " # Month-day exclusion (recurring each year)\n", + " if EXCLUDE_MONTHDAY_RANGES:\n", + " md1 = (d1.month, d1.day)\n", + " md2 = (d2.month, d2.day)\n", + " for rng in EXCLUDE_MONTHDAY_RANGES:\n", + " if not (isinstance(rng, (list, tuple)) and len(rng) == 2):\n", + " continue\n", + " s_md, e_md = rng\n", + " try:\n", + " s_m, s_d = map(int, s_md.split('-'))\n", + " e_m, e_d = map(int, e_md.split('-'))\n", + " except Exception:\n", + " continue\n", + " start = (s_m, s_d)\n", + " end = (e_m, e_d)\n", + " # handle ranges that wrap year end (e.g., 12-15 to 02-15)\n", + " def _in_range(md):\n", + " if start <= end:\n", + " return start <= md <= end\n", + " return md >= start or md <= end\n", + " if _in_range(md1) or _in_range(md2):\n", + " excluded = True\n", + " break\n", + " if excluded:\n", + " continue\n", + "\n", + " filtered_pairs.append((r, s))\n", + "\n", + "pair_indices = filtered_pairs\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6852f778", + "metadata": {}, + "outputs": [], + "source": [ + "def take_looks(arr, row_looks, col_looks, func_type=\"nanmean\", edge_strategy=\"cutoff\"):\n", + " if row_looks == 1 and col_looks == 1:\n", + " return arr\n", + " if arr.ndim != 2:\n", + " raise ValueError(\"take_looks expects 2D array\")\n", + " rows, cols = arr.shape\n", + " if edge_strategy == \"cutoff\":\n", + " rows = (rows // row_looks) * row_looks\n", + " cols = (cols // col_looks) * col_looks\n", + " arr = arr[:rows, :cols]\n", + " elif edge_strategy == \"pad\":\n", + " pad_r = (-rows) % row_looks\n", + " pad_c = (-cols) % col_looks\n", + " if pad_r or pad_c:\n", + " arr = np.pad(arr, ((0, pad_r), (0, pad_c)), mode=\"constant\", constant_values=np.nan)\n", + " rows, cols = arr.shape\n", + " else:\n", + " raise ValueError(\"edge_strategy must be 'cutoff' or 'pad'\")\n", + "\n", + " new_rows = rows // row_looks\n", + " new_cols = cols // col_looks\n", + " func = getattr(np, func_type)\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\", category=RuntimeWarning)\n", + " return func(arr.reshape(new_rows, row_looks, new_cols, col_looks), axis=(1, 3))\n", + "\n", + "\n", + "def _multilook(arr, looks_y=1, looks_x=1):\n", + " return take_looks(arr, looks_y, looks_x, func_type=\"nanmean\", edge_strategy=\"cutoff\")\n", + "\n", + "\n", + "def _box_mean(arr, win_y, win_x):\n", + " pad_y = win_y // 2\n", + " pad_x = win_x // 2\n", + " arr_p = np.pad(arr, ((pad_y, pad_y), (pad_x, pad_x)), mode='reflect')\n", + " windows = sliding_window_view(arr_p, (win_y, win_x))\n", + " return windows.mean(axis=(-2, -1))\n", + "\n", + "def get_kernel(size_y, size_x, num_conv):\n", + " if not isinstance(num_conv, int):\n", + " raise ValueError('num_conv must be an integer')\n", + " k0 = np.ones((size_y, size_x), dtype=np.float32)\n", + " k = k0\n", + " for i in range(num_conv):\n", + " if i > 3:\n", + " k = convolve(k, k0, mode='same')\n", + " else:\n", + " k = convolve(k, k0)\n", + " k = k / np.sum(k)\n", + " return k.astype(np.float32)\n", + "\n", + "\n", + "def _weighted_mean(arr, k):\n", + " # supports complex or real arrays\n", + " return convolve(arr, k, mode='same')\n", + "\n", + "\n", + "\n", + "def lee_filter(img, win_y=5, win_x=5):\n", + " mean = _box_mean(img, win_y, win_x)\n", + " mean_sq = _box_mean(img**2, win_y, win_x)\n", + " var = mean_sq - mean**2\n", + " noise_var = np.nanmedian(var)\n", + " w = var / (var + noise_var + 1e-8)\n", + " return mean + w * (img - mean)\n", + "\n", + "\n", + "def goldstein(\n", + " phase: NDArray[np.complex64] | NDArray[np.float64], alpha: float, psize: int = 32\n", + ") -> np.ndarray:\n", + " \"\"\"Apply the Goldstein adaptive filter to the given data.\"\"\"\n", + "\n", + " def apply_pspec(data: NDArray[np.complex64]) -> np.ndarray:\n", + " if alpha < 0:\n", + " raise ValueError(f\"alpha must be >= 0, got {alpha = }\")\n", + " weight = np.power(np.abs(data) ** 2, alpha / 2)\n", + " data = weight * data\n", + " return data\n", + "\n", + " def make_weight(nxp: int, nyp: int) -> np.ndarray:\n", + " wx = 1.0 - np.abs(np.arange(nxp // 2) - (nxp / 2.0 - 1.0)) / (nxp / 2.0 - 1.0)\n", + " wy = 1.0 - np.abs(np.arange(nyp // 2) - (nyp / 2.0 - 1.0)) / (nyp / 2.0 - 1.0)\n", + " quadrant = np.outer(wy, wx)\n", + " weight = np.block(\n", + " [\n", + " [quadrant, np.flip(quadrant, axis=1)],\n", + " [np.flip(quadrant, axis=0), np.flip(np.flip(quadrant, axis=0), axis=1)],\n", + " ]\n", + " )\n", + " return weight\n", + "\n", + " def patch_goldstein_filter(\n", + " data: NDArray[np.complex64], weight: NDArray[np.float64], psize: int\n", + " ) -> np.ndarray:\n", + " data = np.fft.fft2(data, s=(psize, psize))\n", + " data = apply_pspec(data)\n", + " data = np.fft.ifft2(data, s=(psize, psize))\n", + " return weight * data\n", + "\n", + " def apply_goldstein_filter(data: NDArray[np.complex64]) -> np.ndarray:\n", + " empty_mask = np.isnan(data) | (data == 0)\n", + " if np.all(empty_mask):\n", + " return data\n", + "\n", + " nrows, ncols = data.shape\n", + " step = psize // 2\n", + "\n", + " pad_top = step\n", + " pad_left = step\n", + " pad_bottom = step + (step - (nrows % step)) % step\n", + " pad_right = step + (step - (ncols % step)) % step\n", + " data_padded = np.pad(\n", + " data, ((pad_top, pad_bottom), (pad_left, pad_right)), mode=\"reflect\"\n", + " )\n", + "\n", + " out = np.zeros(data_padded.shape, dtype=np.complex64)\n", + " weight_sum = np.zeros(data_padded.shape, dtype=np.float64)\n", + " weight_matrix = make_weight(psize, psize)\n", + "\n", + " padded_rows, padded_cols = data_padded.shape\n", + " for i in range(0, padded_rows - psize + 1, step):\n", + " for j in range(0, padded_cols - psize + 1, step):\n", + " data_window = data_padded[i : i + psize, j : j + psize]\n", + " filtered_window = patch_goldstein_filter(\n", + " data_window, weight_matrix, psize\n", + " )\n", + " out[i : i + psize, j : j + psize] += filtered_window\n", + " weight_sum[i : i + psize, j : j + psize] += weight_matrix\n", + "\n", + " valid = weight_sum > 0\n", + " out[valid] /= weight_sum[valid]\n", + "\n", + " out = out[pad_top : pad_top + nrows, pad_left : pad_left + ncols]\n", + " out[empty_mask] = 0\n", + " return out\n", + "\n", + " if np.iscomplexobj(phase):\n", + " return apply_goldstein_filter(phase)\n", + " else:\n", + " return apply_goldstein_filter(np.exp(1j * phase))\n", + "\n", + "\n", + "def calc_ifg_coh_filtered(reference, secondary, goldstein_alpha=0.5, coh_win_y=5, coh_win_x=5, looks_y=1, looks_x=1, coh_method=\"standard\", ifg_apply_filter=True, coh_apply_lee=False, coh_kernel=\"boxcar\", coh_kernel_num_conv=5):\n", + " reference = _multilook(reference, looks_y, looks_x)\n", + " secondary = _multilook(secondary, looks_y, looks_x)\n", + " phase = reference * np.conjugate(secondary)\n", + " amp = np.sqrt((reference * np.conjugate(reference)) * (secondary * np.conjugate(secondary)))\n", + " nan_mask = np.isnan(phase)\n", + " ifg_cpx = np.exp(1j * np.nan_to_num(np.angle(phase/amp)))\n", + " if ifg_apply_filter:\n", + " ifg_cpx_f = goldstein(ifg_cpx, alpha=goldstein_alpha, psize=32)\n", + " else:\n", + " ifg_cpx_f = ifg_cpx\n", + " ifg = np.angle(ifg_cpx_f)\n", + " ifg[nan_mask] = np.nan\n", + "\n", + " ifg_cpx_used = ifg_cpx # coherence never uses filtered IFG\n", + "\n", + " if coh_kernel == 'weighted':\n", + " k = get_kernel(coh_win_y, coh_win_x, coh_kernel_num_conv)\n", + " elif coh_kernel == 'boxcar':\n", + " k = None\n", + " else:\n", + " raise ValueError(\"coh_kernel must be 'boxcar' or 'weighted'\")\n", + " if coh_method == \"phase_only\":\n", + " if coh_kernel == 'weighted':\n", + " coh = np.abs(_weighted_mean(ifg_cpx_used, k))\n", + " else:\n", + " coh = np.abs(_box_mean(ifg_cpx_used, coh_win_y, coh_win_x))\n", + " elif coh_method == \"standard\":\n", + " if coh_kernel == 'weighted':\n", + " num = np.abs(_weighted_mean(phase, k))\n", + " den = np.sqrt(_weighted_mean(np.abs(reference)**2, k) * _weighted_mean(np.abs(secondary)**2, k))\n", + " else:\n", + " num = np.abs(_box_mean(phase, coh_win_y, coh_win_x))\n", + " den = np.sqrt(_box_mean(np.abs(reference)**2, coh_win_y, coh_win_x) * _box_mean(np.abs(secondary)**2, coh_win_y, coh_win_x))\n", + " coh = np.where(den > 0, num / den, 0)\n", + " else:\n", + " raise ValueError(\"coh_method must be 'standard' or 'phase_only'\")\n", + " coh = np.clip(coh, 0, 1)\n", + " if coh_apply_lee:\n", + " coh = lee_filter(coh, win_y=coh_win_y, win_x=coh_win_x)\n", + " coh = np.clip(coh, 0, 1)\n", + " zero_mask = phase == 0\n", + " coh[nan_mask] = np.nan\n", + " coh[zero_mask] = 0\n", + " return ifg, coh, amp\n", + "\n", + "\n", + "def calc_ifg_coh(reference, secondary, goldstein_alpha=0.5, coh_win_y=5, coh_win_x=5, looks_y=1, looks_x=1, coh_method=\"standard\", ifg_apply_filter=True, coh_apply_lee=False, coh_kernel=\"boxcar\", coh_kernel_num_conv=5):\n", + " return calc_ifg_coh_filtered(reference, secondary, goldstein_alpha=goldstein_alpha, coh_win_y=coh_win_y, coh_win_x=coh_win_x, looks_y=looks_y, looks_x=looks_x, coh_method=coh_method, ifg_apply_filter=ifg_apply_filter, coh_apply_lee=coh_apply_lee, coh_kernel=coh_kernel, coh_kernel_num_conv=coh_kernel_num_conv)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2775556c", + "metadata": {}, + "outputs": [], + "source": [ + "## For each date-pair, calculate the ifg, coh. Save the results as GeoTiffs.\n", + "for ref_idx, sec_idx in pair_indices:\n", + " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", + " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", + " print(f\"Reference: {ref_date} Secondary: {sec_date}\")\n", + "\n", + " # Calculate ifg, coh, amp\n", + " if \"calc_ifg_coh_filtered\" not in globals():\n", + " raise RuntimeError(\"calc_ifg_coh_filtered is not defined. Run the filter definition cell first.\")\n", + " looks_y, looks_x = MULTILOOK\n", + "\n", + " # Save each interferogram as GeoTiff (no per-burst plotting)\n", + " transform = from_origin(xcoor_stack[ref_idx][0], ycoor_stack[ref_idx][0], dx, np.abs(dy))" + ] + }, + { + "cell_type": "markdown", + "id": "d57d1a71", + "metadata": {}, + "source": [ + "
\n", + "8. Merge the burst-wise interferograms and coherence and save as GeoTiff.\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e98baf1f", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_merge(old_data, new_data, old_nodata, new_nodata, **kwargs):\n", + " # Feather overlaps to reduce burst seams\n", + " if MERGE_BLEND_OVERLAP:\n", + " overlap = np.logical_and(~old_nodata, ~new_nodata)\n", + " if np.any(overlap):\n", + " # distance to nodata inside each valid mask\n", + " dist_old = distance_transform_edt(~old_nodata)\n", + " dist_new = distance_transform_edt(~new_nodata)\n", + " w_new = dist_new / (dist_new + dist_old + MERGE_BLEND_EPS)\n", + " w_new = np.clip(w_new, 0, 1)\n", + " blended = old_data * (1 - w_new) + new_data * w_new\n", + " old_data[overlap] = blended[overlap]\n", + " # fill empty pixels\n", + " mask = np.logical_and(old_nodata, ~new_nodata)\n", + " old_data[mask] = new_data[mask]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "897e44dd", + "metadata": {}, + "outputs": [], + "source": [ + "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", + "# Merge burst-wise interferograms per date-pair\n", + "\n", + "# Group pair indices by date tag\n", + "pairs_by_tag = {}\n", + "for r, s in pair_indices:\n", + " ref_date = cslc_dates.iloc[r].values[0]\n", + " sec_date = cslc_dates.iloc[s].values[0]\n", + " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", + " pairs_by_tag.setdefault(tag, []).append((r, s))\n", + "\n", + "for tag, pairs in pairs_by_tag.items():\n", + " srcs = []\n", + " for r, s in pairs:\n", + " looks_y, looks_x = MULTILOOK\n", + " ifg, coh, amp = calc_ifg_coh(\n", + " cslc_stack[r], cslc_stack[s],\n", + " goldstein_alpha=GOLDSTEIN_ALPHA, coh_win_y=COH_WIN_Y, coh_win_x=COH_WIN_X,\n", + " looks_y=looks_y, looks_x=looks_x,\n", + " coh_method=COH_METHOD,\n", + " ifg_apply_filter=IFG_APPLY_FILTER,\n", + " coh_apply_lee=COH_APPLY_LEE,\n", + " coh_kernel=COH_KERNEL, coh_kernel_num_conv=COH_KERNEL_NUM_CONV,\n", + " )\n", + " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", + " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", + " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", + " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", + " mem = MemoryFile()\n", + " ds = mem.open(\n", + " driver='GTiff', height=ifg.shape[0], width=ifg.shape[1], count=1, dtype=ifg.dtype,\n", + " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " )\n", + " ds.write(ifg, 1)\n", + " srcs.append(ds)\n", + " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", + " dest, output_transform = _trim_nan_border(dest, output_transform)\n", + " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", + " if mask is not None:\n", + " dest[0] = _apply_water_mask(dest[0], mask)\n", + " out_meta = srcs[0].meta.copy()\n", + " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", + " out_path = f\"{savedir}/tifs/merged_ifg_{tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path)):\n", + " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", + " dest1.write(dest)\n", + " if SAVE_WGS84:\n", + " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_ifg_WGS84_{tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_wgs84)):\n", + " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", + " for ds in srcs:\n", + " ds.close()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c41924c", + "metadata": {}, + "outputs": [], + "source": [ + "# Merge burst-wise coherence per date-pair\n", + "\n", + "# Group pair indices by date tag\n", + "pairs_by_tag = {}\n", + "for r, s in pair_indices:\n", + " ref_date = cslc_dates.iloc[r].values[0]\n", + " sec_date = cslc_dates.iloc[s].values[0]\n", + " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", + " pairs_by_tag.setdefault(tag, []).append((r, s))\n", + "\n", + "for tag, pairs in pairs_by_tag.items():\n", + " srcs = []\n", + " for r, s in pairs:\n", + " looks_y, looks_x = MULTILOOK\n", + " ifg, coh, amp = calc_ifg_coh(\n", + " cslc_stack[r], cslc_stack[s],\n", + " goldstein_alpha=GOLDSTEIN_ALPHA, coh_win_y=COH_WIN_Y, coh_win_x=COH_WIN_X,\n", + " looks_y=looks_y, looks_x=looks_x,\n", + " coh_method=COH_METHOD,\n", + " ifg_apply_filter=IFG_APPLY_FILTER,\n", + " coh_apply_lee=COH_APPLY_LEE,\n", + " coh_kernel=COH_KERNEL, coh_kernel_num_conv=COH_KERNEL_NUM_CONV,\n", + " )\n", + " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", + " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", + " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", + " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", + " mem = MemoryFile()\n", + " ds = mem.open(\n", + " driver='GTiff', height=coh.shape[0], width=coh.shape[1], count=1, dtype=coh.dtype,\n", + " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " )\n", + " ds.write(coh, 1)\n", + " srcs.append(ds)\n", + " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", + " dest, output_transform = _trim_nan_border(dest, output_transform)\n", + " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", + " if mask is not None:\n", + " dest[0] = _apply_water_mask(dest[0], mask)\n", + " out_meta = srcs[0].meta.copy()\n", + " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", + " out_path = f\"{savedir}/tifs/merged_coh_{tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path)):\n", + " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", + " dest1.write(dest)\n", + " if SAVE_WGS84:\n", + " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_coh_WGS84_{tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_wgs84)):\n", + " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", + " for ds in srcs:\n", + " ds.close()\n" + ] + }, + { + "cell_type": "markdown", + "id": "858ea831", + "metadata": {}, + "source": [ + "
\n", + "9. Read the merged GeoTiff and Visualize using `matplotlib`\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9cb2a341", + "metadata": {}, + "outputs": [], + "source": [ + "# Read merged IFG/COH files and plot paired grids\n", + "\n", + "\n", + "# Output dir for per-pair PNGs\n", + "pair_png_dir = f\"{savedir}/pairs_png\"\n", + "os.makedirs(pair_png_dir, exist_ok=True)\n", + "\n", + "ifg_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_ifg_*.tif\"))\n", + "coh_norm = mcolors.PowerNorm(gamma=COH_NORM_GAMMA, vmin=0, vmax=1) if COH_USE_GAMMA_NORM else None\n", + "coh_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\"))\n", + "\n", + "ifg_map = {p.split('merged_ifg_')[-1].replace('.tif',''): p for p in ifg_paths}\n", + "coh_map = {p.split('merged_coh_')[-1].replace('.tif',''): p for p in coh_paths}\n", + "\n", + "\n", + "\n", + "def _prep_da(path):\n", + " da = rioxarray.open_rasterio(path)[0]\n", + " data = da.values\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if mask.any():\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " # Trim NaN borders so edges don't show padding\n", + " da = da.isel(y=slice(r0, r1), x=slice(c0, c1))\n", + " return da\n", + "\n", + "pair_tags = sorted(set(ifg_map).intersection(coh_map))\n", + "# Filter pairs by current date range\n", + "date_start_day = dateStart.date()\n", + "date_end_day = dateEnd.date()\n", + "pair_tags = [t for t in pair_tags if (date_start_day <= pd.to_datetime(t.split('-')[0], format='%Y%m%d').date() <= date_end_day and date_start_day <= pd.to_datetime(t.split('-')[1], format='%Y%m%d').date() <= date_end_day)]\n", + "\n", + "if not pair_tags:\n", + " print('No matching IFG/COH pairs found')\n", + "else:\n", + " # Save ALL pairs as PNGs\n", + " for tag in pair_tags:\n", + " fig, axes = plt.subplots(1, 2, figsize=(10, 4), constrained_layout=True)\n", + " ax_ifg, ax_coh = axes\n", + "\n", + " # IFG\n", + " merged_ifg = _prep_da(ifg_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", + " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", + " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", + " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", + " ax_ifg.set_xticks([])\n", + " ax_ifg.set_yticks([])\n", + " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", + "\n", + " # COH\n", + " merged_coh = _prep_da(coh_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", + " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, norm=coh_norm, vmin=None if COH_USE_GAMMA_NORM else 0, vmax=None if COH_USE_GAMMA_NORM else 1.0)\n", + " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", + " ax_coh.set_xticks([])\n", + " ax_coh.set_yticks([])\n", + " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')\n", + "\n", + " out_png = os.path.join(pair_png_dir, f\"pair_{tag}.png\")\n", + " fig.savefig(out_png, dpi=150)\n", + " plt.close(fig)\n", + "\n", + " # Display only last 5 pairs in notebook\n", + " display_tags = pair_tags[-5:]\n", + " n = len(display_tags)\n", + " ncols = 2\n", + " nrows = math.ceil(n / 1) # one pair per row\n", + " fig, axes = plt.subplots(nrows, ncols, figsize=(6*ncols, 3*nrows), constrained_layout=True)\n", + " if nrows == 1:\n", + " axes = [axes]\n", + "\n", + " for i, tag in enumerate(display_tags):\n", + " ax_ifg, ax_coh = axes[i]\n", + "\n", + " # IFG\n", + " merged_ifg = _prep_da(ifg_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", + " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", + " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", + " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", + " ax_ifg.set_xticks([])\n", + " ax_ifg.set_yticks([])\n", + " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", + "\n", + " # COH\n", + " merged_coh = _prep_da(coh_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", + " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, norm=coh_norm, vmin=None if COH_USE_GAMMA_NORM else 0, vmax=None if COH_USE_GAMMA_NORM else 1.0)\n", + " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", + " ax_coh.set_xticks([])\n", + " ax_coh.set_yticks([])\n", + " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')" + ] + }, + { + "cell_type": "markdown", + "id": "ea1e4f24", + "metadata": {}, + "source": [ + "
\n", + "9.5 Merge and plot backscatter (dB) mosaics (per date)\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7f478f4", + "metadata": {}, + "outputs": [], + "source": [ + "def _load_water_mask_match(mask_path, shape, transform, crs):\n", + " with rasterio.open(mask_path) as src:\n", + " src_mask = src.read(1)\n", + " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", + " return src_mask\n", + " dst = np.zeros(shape, dtype=src_mask.dtype)\n", + " reproject(\n", + " source=src_mask,\n", + " destination=dst,\n", + " src_transform=src.transform,\n", + " src_crs=src.crs,\n", + " dst_transform=transform,\n", + " dst_crs=crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " return dst\n", + "\n", + "def _apply_water_mask(arr, mask):\n", + " # mask: 1 = keep land, 0 = water\n", + " return np.where(mask == 0, np.nan, arr)\n", + "\n", + "\n", + "def _trim_nan_border(arr, transform):\n", + " data = arr[0] if arr.ndim == 3 else arr\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if not mask.any():\n", + " return arr, transform\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " data = data[r0:r1, c0:c1]\n", + " if arr.ndim == 3:\n", + " arr = data[None, ...]\n", + " else:\n", + " arr = data\n", + " new_transform = transform * Affine.translation(c0, r0)\n", + " return arr, new_transform\n", + "\n", + "# Build per-date backscatter (dB) mosaics directly from subset H5 (no per-burst GeoTIFFs)\n", + "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", + "\n", + "date_tags = sorted(cslc_df.startTime.astype(str).str.replace('-', '').unique())\n", + "\n", + "def _save_mosaic_utm(out_path, mosaic, transform, epsg):\n", + " with rasterio.open(\n", + " out_path, \"w\", driver=\"GTiff\", height=mosaic.shape[0], width=mosaic.shape[1],\n", + " count=1, dtype=mosaic.dtype, crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " ) as dst_ds:\n", + " dst_ds.write(mosaic, 1)\n", + "\n", + "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", + " dst_crs = \"EPSG:4326\"\n", + " src_crs = CRS.from_epsg(epsg)\n", + " height, width = mosaic.shape\n", + " dst_transform, dst_width, dst_height = calculate_default_transform(\n", + " src_crs, dst_crs, width, height, *rasterio.transform.array_bounds(height, width, transform)\n", + " )\n", + " dst = np.empty((dst_height, dst_width), dtype=mosaic.dtype)\n", + " reproject(\n", + " source=mosaic,\n", + " destination=dst,\n", + " src_transform=transform,\n", + " src_crs=src_crs,\n", + " dst_transform=dst_transform,\n", + " dst_crs=dst_crs,\n", + " resampling=Resampling.bilinear,\n", + " )\n", + " with rasterio.open(\n", + " out_path, \"w\", driver=\"GTiff\", height=dst_height, width=dst_width, count=1,\n", + " dtype=dst.dtype, crs=dst_crs, transform=dst_transform, nodata=np.nan\n", + " ) as dst_ds:\n", + " dst_ds.write(dst, 1)\n", + "\n", + "# Mosaicking helper (in memory)\n", + "def _mosaic_arrays(arrays, transforms, epsg):\n", + " # Convert arrays to in-memory rasterio datasets via MemoryFile\n", + " srcs = []\n", + " for arr, transform in zip(arrays, transforms):\n", + " mem = MemoryFile()\n", + " ds = mem.open(\n", + " driver='GTiff', height=arr.shape[0], width=arr.shape[1], count=1, dtype=arr.dtype,\n", + " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " )\n", + " ds.write(arr, 1)\n", + " srcs.append(ds)\n", + " dest, out_transform = merge.merge(srcs, method=custom_merge)\n", + " for ds in srcs:\n", + " ds.close()\n", + " return dest[0], out_transform\n", + "\n", + "\n", + "def _reproject_calibration_to_data_grid(sigma, cal_x, cal_y, cal_dx, cal_dy, xcoor, ycoor, dx, dy, epsg):\n", + " cal_dy_signed = (cal_y[1] - cal_y[0]) if len(cal_y) > 1 else -cal_dy\n", + " data_dy_signed = (ycoor[1] - ycoor[0]) if len(ycoor) > 1 else -dy\n", + " if sigma.shape == (len(ycoor), len(xcoor)) and cal_dx == dx and abs(cal_dy_signed) == abs(data_dy_signed):\n", + " return sigma\n", + " src_transform = from_origin(cal_x[0], cal_y[0], cal_dx, abs(cal_dy_signed))\n", + " dst_transform = from_origin(xcoor[0], ycoor[0], dx, abs(data_dy_signed))\n", + " dst = np.empty((len(ycoor), len(xcoor)), dtype=sigma.dtype)\n", + " reproject(\n", + " source=sigma,\n", + " destination=dst,\n", + " src_transform=src_transform,\n", + " src_crs=CRS.from_epsg(epsg),\n", + " dst_transform=dst_transform,\n", + " dst_crs=CRS.from_epsg(epsg),\n", + " resampling=Resampling.bilinear,\n", + " )\n", + " return dst\n", + "\n", + "looks_y, looks_x = MULTILOOK\n", + "# Backscatter calibration mode tag for filenames\n", + "bsc_mode_tag = CALIBRATION_MODE\n", + "beta_naught = None\n", + "for date_tag in date_tags:\n", + " # collect subset H5 files for this date\n", + " rows = cslc_df[cslc_df.startTime.astype(str).str.replace('-', '') == date_tag]\n", + " arrays = []\n", + " transforms = []\n", + " epsg = None\n", + " for fileID in rows.fileID:\n", + " subset_path = f\"{savedir}/subset_cslc/{fileID}.h5\"\n", + " with h5py.File(subset_path, 'r') as h5:\n", + " cslc = h5['/data/VV'][:]\n", + " xcoor = h5['/data/x_coordinates'][:]\n", + " ycoor = h5['/data/y_coordinates'][:]\n", + " dx = int(h5['/data/x_spacing'][()])\n", + " dy = int(h5['/data/y_spacing'][()])\n", + " epsg = int(h5['/data/projection'][()])\n", + " sigma = h5['/metadata/calibration_information/sigma_naught'][:]\n", + " cal_x = h5['/metadata/calibration_information/x_coordinates'][:]\n", + " cal_y = h5['/metadata/calibration_information/y_coordinates'][:]\n", + " cal_dx = int(h5['/metadata/calibration_information/x_spacing'][()])\n", + " cal_dy = int(h5['/metadata/calibration_information/y_spacing'][()])\n", + " beta_naught = h5['/metadata/calibration_information/beta_naught'][()]\n", + " power_ml = _multilook(np.abs(cslc)**2, looks_y, looks_x)\n", + " if CALIBRATION_MODE == 'sigma0':\n", + " sigma_on_data = _reproject_calibration_to_data_grid(\n", + " sigma, cal_x, cal_y, cal_dx, cal_dy, xcoor, ycoor, dx, dy, epsg\n", + " )\n", + " sigma_ml = _multilook(sigma_on_data, looks_y, looks_x)\n", + " sigma_ml = np.where(sigma_ml <= 0, np.nan, sigma_ml)\n", + " sigma_factor = sigma_ml**2 if CALIBRATION_FACTOR_IS_AMPLITUDE else sigma_ml\n", + " bsc = 10*np.log10(power_ml / sigma_factor)\n", + " elif CALIBRATION_MODE == 'beta0':\n", + " beta = beta_naught\n", + " if beta is None or beta <= 0:\n", + " bsc = np.full(power_ml.shape, np.nan, dtype=power_ml.dtype)\n", + " else:\n", + " beta_factor = beta**2 if CALIBRATION_FACTOR_IS_AMPLITUDE else beta\n", + " bsc = 10*np.log10(power_ml / beta_factor)\n", + " else:\n", + " raise ValueError(\"CALIBRATION_MODE must be 'sigma0' or 'beta0'\")\n", + " dy_signed = (ycoor[1] - ycoor[0]) if len(ycoor) > 1 else -dy\n", + " x0 = xcoor[0] + (looks_x - 1) * dx / 2\n", + " y0 = ycoor[0] + (looks_y - 1) * dy_signed / 2\n", + " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", + " arrays.append(bsc)\n", + " transforms.append(transform)\n", + "\n", + " if not arrays:\n", + " continue\n", + " mosaic, out_transform = _mosaic_arrays(arrays, transforms, epsg)\n", + " out_path_utm = f\"{savedir}/tifs/merged_bsc_{bsc_mode_tag}_{date_tag}.tif\"\n", + " mosaic, out_transform = _trim_nan_border(mosaic, out_transform)\n", + " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " mask = _load_water_mask_match(WATER_MASK_PATH, mosaic.shape, out_transform, CRS.from_epsg(epsg))\n", + " mosaic = _apply_water_mask(mosaic, mask)\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_utm)):\n", + " _save_mosaic_utm(out_path_utm, mosaic, out_transform, epsg)\n", + " if SAVE_WGS84:\n", + " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_bsc_{bsc_mode_tag}_WGS84_{date_tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_wgs84)):\n", + " _save_mosaic_utm_to_wgs84(out_path_wgs84, mosaic, out_transform, epsg)\n", + "\n", + "# Plot merged backscatter (dB) mosaics in a grid (native CRS from saved GeoTIFFs)\n", + "\n", + "# Output dir for backscatter PNGs\n", + "bsc_png_dir = f\"{savedir}/bsc_png\"\n", + "os.makedirs(bsc_png_dir, exist_ok=True)\n", + "\n", + "paths = sorted(glob.glob(f\"{savedir}/tifs/merged_bsc_{bsc_mode_tag}_*.tif\"))\n", + "paths = [p for p in paths if 'WGS84' not in p]\n", + "all_vals = []\n", + "for p in paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " all_vals.append(da.values.ravel())\n", + "if all_vals:\n", + " all_vals = np.concatenate(all_vals)\n", + " gmin = np.nanpercentile(all_vals, 2)\n", + " gmax = np.nanpercentile(all_vals, 90)\n", + "else:\n", + " gmin, gmax = None, None\n", + "n = len(paths)\n", + "if n == 0:\n", + " print('No merged backscatter files found')\n", + "else:\n", + " # Save ALL backscatter PNGs\n", + " for path in paths:\n", + " src = rioxarray.open_rasterio(path)\n", + " bsc = src[0]\n", + " minlon, minlat, maxlon, maxlat = bsc.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " fig, ax = plt.subplots(figsize=(5,4))\n", + " im = ax.imshow(bsc.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", + " tag = path.split(f'merged_bsc_{bsc_mode_tag}_')[-1].replace('.tif','')\n", + " ax.set_title(f\"{tag}\", fontsize=10)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " fig.colorbar(im, ax=ax, orientation='vertical', fraction=0.046, pad=0.02, label=f\"{bsc_mode_tag} (dB)\")\n", + " out_png = os.path.join(bsc_png_dir, f\"bsc_{tag}.png\")\n", + " fig.savefig(out_png, dpi=150)\n", + " plt.close(fig)\n", + "\n", + " # Show only last 5 in notebook\n", + " display_paths = paths[-5:]\n", + " n = len(display_paths)\n", + " ncols = 3\n", + " nrows = math.ceil(n / ncols)\n", + " fig, axes = plt.subplots(nrows, ncols, figsize=(4*ncols, 3*nrows), constrained_layout=True)\n", + " axes = axes.ravel()\n", + " for ax, path in zip(axes, display_paths):\n", + " src = rioxarray.open_rasterio(path)\n", + " bsc = src[0]\n", + " minlon, minlat, maxlon, maxlat = bsc.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " im = ax.imshow(bsc.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", + " tag = path.split(f'merged_bsc_{bsc_mode_tag}_')[-1].replace('.tif','')\n", + " ax.set_title(f\"{tag}\", fontsize=10)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " for ax in axes[n:]:\n", + " ax.axis('off')\n", + " fig.colorbar(im, ax=axes.tolist(), orientation='vertical', fraction=0.02, pad=0.02, label=f\"{bsc_mode_tag} (dB)\")" + ] + }, + { + "cell_type": "markdown", + "id": "df66a59f", + "metadata": {}, + "source": [ + "
\n", + "10. Monthly mean coherence calendar (per year)\n", + "\n", + "This calendar bins each pair into a month using the midpoint date between the reference and secondary scenes.\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c31a947a", + "metadata": {}, + "outputs": [], + "source": [ + "# # CALENDAR_CMAP = ROCKET_CMAP # previous\n", + "CALENDAR_CMAP = cmc.bilbao\n", + "# CALENDAR_CMAP = cmc.bilbao\n", + "\n", + "# Build an index of merged coherence files by midpoint year-month\n", + "records = []\n", + "for path in sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\")):\n", + " tag = path.split('merged_coh_')[-1].replace('.tif','')\n", + " try:\n", + " ref_str, sec_str = tag.split('-')\n", + " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", + " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", + " mid_date = ref_date + (sec_date - ref_date) / 2\n", + " except Exception:\n", + " continue\n", + " records.append({\"path\": path, \"mid_date\": mid_date})\n", + "\n", + "df_paths = pd.DataFrame(records)\n", + "if df_paths.empty:\n", + " print('No merged coherence files found for calendar')\n", + " raise SystemExit\n", + "\n", + "# Apply current date range using midpoint date\n", + "date_start_day = dateStart.date()\n", + "date_end_day = dateEnd.date()\n", + "df_paths = df_paths[(df_paths['mid_date'].dt.date >= date_start_day) & (df_paths['mid_date'].dt.date <= date_end_day)]\n", + "\n", + "# Calendar year labeling\n", + "if USE_WATER_YEAR:\n", + " # Water year starts Oct (10) and ends Sep (9)\n", + " df_paths['year'] = df_paths['mid_date'].dt.year + (df_paths['mid_date'].dt.month >= 10).astype(int)\n", + " month_order = [10,11,12,1,2,3,4,5,6,7,8,9]\n", + " month_labels = ['Oct','Nov','Dec','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep']\n", + "else:\n", + " df_paths['year'] = df_paths['mid_date'].dt.year\n", + " month_order = list(range(1,13))\n", + " month_labels = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']\n", + "\n", + "years = sorted(df_paths['year'].unique())\n", + "\n", + "# Contrast stretch for low coherence (red = low)\n", + "norm = mcolors.PowerNorm(gamma=0.3, vmin=0, vmax=1)\n", + "\n", + "# One row per year, 12 columns\n", + "fig, axes = plt.subplots(len(years), 12, figsize=(24, 2.5*len(years)), constrained_layout=True)\n", + "if len(years) == 1:\n", + " axes = np.array([axes])\n", + "\n", + "for row_idx, y in enumerate(years):\n", + " # pick a template for consistent grid within the year (first available file)\n", + " year_paths = df_paths[df_paths['year'] == y]['path'].tolist()\n", + " if not year_paths:\n", + " continue\n", + " template = rioxarray.open_rasterio(year_paths[0])\n", + "\n", + " for col_idx, m in enumerate(month_order):\n", + " ax = axes[row_idx, col_idx]\n", + " month_paths = df_paths[(df_paths['year'] == y) & (df_paths['mid_date'].dt.month == m)]['path'].tolist()\n", + " if USE_WATER_YEAR:\n", + " year_for_month = y - 1 if m in (10, 11, 12) else y\n", + " else:\n", + " year_for_month = y\n", + " title = f\"{month_labels[col_idx]} {year_for_month}\"\n", + " if not month_paths:\n", + " ax.set_title(title, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " # keep a visible box for empty months\n", + " for spine in ax.spines.values():\n", + " spine.set_visible(True)\n", + " spine.set_linewidth(0.8)\n", + " spine.set_color('0.5')\n", + " continue\n", + " stacks = []\n", + " for p in month_paths:\n", + " da = rioxarray.open_rasterio(p)\n", + " da = da.rio.reproject_match(template)\n", + " stacks.append(da)\n", + " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", + " minlon, minlat, maxlon, maxlat = da_month.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " im = ax.imshow(da_month.values.squeeze(), cmap=CALENDAR_CMAP, norm=norm, origin='upper', extent=bbox, interpolation='none')\n", + " ax.set_title(title, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " for spine in ax.spines.values():\n", + " spine.set_visible(True)\n", + " spine.set_linewidth(0.8)\n", + " spine.set_color('0.5')\n", + " # left-side year label\n", + " if USE_WATER_YEAR:\n", + " label = f\"WY {y}\"\n", + " else:\n", + " label = str(y)\n", + " axes[row_idx, 0].set_ylabel(label, rotation=90, labelpad=6, fontsize=9)\n", + " axes[row_idx, 0].yaxis.set_label_coords(-0.06, 0.5)\n", + "\n", + "fig.colorbar(im, ax=axes, orientation='vertical', fraction=0.02, pad=0.02, label='Mean coherence')" + ] + }, + { + "cell_type": "markdown", + "id": "169216f0", + "metadata": {}, + "source": [ + "
\n", + "11. Create GIF animations (Backscatter (dB) + Coherence)\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89109813", + "metadata": {}, + "outputs": [], + "source": [ + "# GIF helper functions\n", + "\n", + "def _global_bounds(paths):\n", + " bounds = []\n", + " for p in paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " minx, miny, maxx, maxy = da.rio.bounds()\n", + " bounds.append((minx, miny, maxx, maxy))\n", + " minx = min(b[0] for b in bounds)\n", + " miny = min(b[1] for b in bounds)\n", + " maxx = max(b[2] for b in bounds)\n", + " maxy = max(b[3] for b in bounds)\n", + " return [minx, maxx, miny, maxy]\n", + "\n", + "def _extract_date_from_tag(tag):\n", + " m = re.search(r\"(\\d{8})\", tag)\n", + " if not m:\n", + " return None\n", + " try:\n", + " return pd.to_datetime(m.group(1), format=\"%Y%m%d\")\n", + " except Exception:\n", + " return None\n", + "\n", + "def _format_title(tag, date=None):\n", + " if date is not None:\n", + " return date.strftime('%Y-%m-%d')\n", + " m = re.findall(r\"(\\d{8})\", tag)\n", + " if len(m) >= 2:\n", + " try:\n", + " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\")\n", + " d1 = pd.to_datetime(m[1], format=\"%Y%m%d\")\n", + " return f\"{d0.strftime('%Y-%m-%d')} to {d1.strftime('%Y-%m-%d')}\"\n", + " except Exception:\n", + " pass\n", + " if len(m) == 1:\n", + " try:\n", + " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\")\n", + " return d0.strftime('%Y-%m-%d')\n", + " except Exception:\n", + " pass\n", + " return tag\n", + "\n", + "def _filter_paths_by_date(paths, date_start, date_end):\n", + " if not paths:\n", + " return paths\n", + " ds = pd.to_datetime(date_start).date()\n", + " de = pd.to_datetime(date_end).date()\n", + " kept = []\n", + " for p in paths:\n", + " tag = os.path.basename(p).replace('.tif','')\n", + " m = re.findall(r\"(\\d{8})\", tag)\n", + " if len(m) >= 2:\n", + " try:\n", + " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\").date()\n", + " d1 = pd.to_datetime(m[1], format=\"%Y%m%d\").date()\n", + " mid = d0 + (d1 - d0) / 2\n", + " mid_date = mid if hasattr(mid, 'year') else mid.date()\n", + " if ds <= mid_date <= de:\n", + " kept.append(p)\n", + " except Exception:\n", + " kept.append(p)\n", + " elif len(m) == 1:\n", + " try:\n", + " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\").date()\n", + " if ds <= d0 <= de:\n", + " kept.append(p)\n", + " except Exception:\n", + " kept.append(p)\n", + " else:\n", + " kept.append(p)\n", + " return kept\n", + "\n", + "def _render_frames(tif_paths, out_dir, cmap, vmin=None, vmax=None, title_prefix=None, extent=None, cbar_label=None, cbar_ticks=None, template=None, dates=None, title_dates=None, show_time_bar=False, norm=None):\n", + " os.makedirs(out_dir, exist_ok=True)\n", + " frames = []\n", + " if template is None and tif_paths:\n", + " template = rioxarray.open_rasterio(tif_paths[0])[0]\n", + " if extent is None and template is not None:\n", + " minlon, minlat, maxlon, maxlat = template.rio.bounds()\n", + " extent = [minlon, maxlon, minlat, maxlat]\n", + "\n", + " date_min = date_max = None\n", + " if show_time_bar and dates:\n", + " valid_dates = [d for d in dates if d is not None]\n", + " if valid_dates:\n", + " date_min = min(valid_dates)\n", + " date_max = max(valid_dates)\n", + "\n", + " for i, p in enumerate(tif_paths):\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " if template is not None:\n", + " da = da.rio.reproject_match(template)\n", + " fig, ax = plt.subplots(figsize=(6,4))\n", + " im = ax.imshow(da.values, cmap=cmap, origin='upper', extent=extent, norm=norm, vmin=None if norm is not None else vmin, vmax=None if norm is not None else vmax)\n", + " tag = os.path.basename(p).replace('.tif','')\n", + " if title_prefix is None:\n", + " title_date = title_dates[i] if title_dates and i < len(title_dates) else None\n", + " title = _format_title(tag, title_date)\n", + " else:\n", + " title = f\"{title_prefix}{tag}\"\n", + " ax.set_title(title, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", + " if cbar_label:\n", + " cb.set_label(cbar_label)\n", + " if cbar_ticks is not None:\n", + " cb.set_ticks(cbar_ticks)\n", + "\n", + " if show_time_bar and date_min is not None and date_max is not None:\n", + " ax_time = fig.add_axes([0.12, 0.04, 0.76, 0.05])\n", + " ax_time.plot([0, 1], [0.5, 0.5], color=\"0.6\", linewidth=3, solid_capstyle='round')\n", + " cur_date = dates[i] if dates and i < len(dates) else None\n", + " if cur_date is not None and date_max != date_min:\n", + " pos = (cur_date - date_min) / (date_max - date_min)\n", + " pos = max(0, min(1, float(pos)))\n", + " else:\n", + " pos = 0.0 if date_max == date_min else 0.5\n", + " ax_time.plot([pos, pos], [0.2, 0.8], color=\"crimson\", linewidth=2, zorder=5)\n", + " ax_time.scatter([pos], [0.5], s=90, color=\"crimson\", zorder=6)\n", + "\n", + " if date_max == date_min:\n", + " tick_dates = [date_min]\n", + " else:\n", + " tick_dates = pd.date_range(date_min, date_max, periods=5)\n", + " for d in tick_dates:\n", + " t = (d - date_min) / (date_max - date_min)\n", + " t = max(0, min(1, float(t)))\n", + " ax_time.plot([t, t], [0.35, 0.65], color=\"0.4\", linewidth=1)\n", + " ax_time.text(t, -0.05, d.strftime('%Y-%m-%d'), ha='center', va='top', fontsize=6)\n", + "\n", + " ax_time.set_xlim(0, 1)\n", + " ax_time.set_ylim(0, 1)\n", + " ax_time.axis('off')\n", + "\n", + " frame_path = os.path.join(out_dir, f\"{tag}.png\")\n", + " fig.savefig(frame_path, dpi=150)\n", + " plt.close(fig)\n", + " frames.append(frame_path)\n", + " return frames\n", + "\n", + "def _pad_frames(frame_paths):\n", + " imgs = [imageio.imread(f) for f in frame_paths]\n", + " max_h = max(im.shape[0] for im in imgs)\n", + " max_w = max(im.shape[1] for im in imgs)\n", + " padded = []\n", + " for im in imgs:\n", + " pad_h = max_h - im.shape[0]\n", + " pad_w = max_w - im.shape[1]\n", + " top = pad_h // 2\n", + " bottom = pad_h - top\n", + " left = pad_w // 2\n", + " right = pad_w - left\n", + " padded.append(np.pad(im, ((top, bottom), (left, right), (0, 0)), mode='edge'))\n", + " return padded\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8397a42", + "metadata": {}, + "outputs": [], + "source": [ + "# Prepare coherence extent/template for GIFs\n", + "coh_template = None\n", + "coh_extent = None\n", + "if coh_paths:\n", + " coh_template = rioxarray.open_rasterio(coh_paths[0])[0]\n", + " minlon, minlat, maxlon, maxlat = coh_template.rio.bounds()\n", + " coh_extent = [minlon, maxlon, minlat, maxlat]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15d63323", + "metadata": {}, + "outputs": [], + "source": [ + "# GIF prep (paths, dates, extent)\n", + "gif_dir = f\"{savedir}/gifs\"\n", + "os.makedirs(gif_dir, exist_ok=True)\n", + "\n", + "coh_paths = _filter_paths_by_date(coh_paths, dateStart, dateEnd)\n", + "coh_dates = []\n", + "for p in coh_paths:\n", + " tag = Path(p).name.replace('merged_coh_', '').replace('.tif', '')\n", + " try:\n", + " ref_str, sec_str = tag.split('-')\n", + " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", + " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", + " coh_dates.append(ref_date + (sec_date - ref_date) / 2)\n", + " except Exception:\n", + " coh_dates.append(None)\n", + "\n", + "coh_template = None\n", + "coh_extent = None\n", + "if coh_paths:\n", + " coh_template = rioxarray.open_rasterio(coh_paths[0])[0]\n", + " minlon, minlat, maxlon, maxlat = coh_template.rio.bounds()\n", + " coh_extent = [minlon, maxlon, minlat, maxlat]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60b7c196", + "metadata": {}, + "outputs": [], + "source": [ + "# Coherence GIF\n", + "# Adjust GIF speed based on number of frames\n", + "\n", + "def _gif_duration(n_frames, min_sec=0.3, max_sec=1.2, target_total_sec=12.0):\n", + " if n_frames <= 0:\n", + " return max_sec\n", + " return max(min_sec, min(max_sec, target_total_sec / n_frames))\n", + "\n", + "coh_norm = mcolors.PowerNorm(gamma=COH_NORM_GAMMA, vmin=0, vmax=1) if COH_USE_GAMMA_NORM else None\n", + "if coh_paths:\n", + " coh_frames = _render_frames(\n", + " coh_paths, f\"{gif_dir}/coh_frames\", cmap=cmc.bilbao, vmin=0, vmax=1,\n", + " title_prefix=None, extent=coh_extent, cbar_label='Coherence', cbar_ticks=[0,0.5,1],\n", + " template=coh_template, dates=coh_dates, title_dates=None, show_time_bar=True, norm=coh_norm\n", + " )\n", + " coh_gif = f\"{gif_dir}/coherence.gif\"\n", + " coh_imgs = _pad_frames(coh_frames)\n", + " imageio.mimsave(coh_gif, coh_imgs, duration=_gif_duration(len(coh_imgs)))\n", + " print(f\"Wrote {coh_gif}\")\n", + "else:\n", + " print('No merged coherence files found for GIF')\n", + "\n", + "\n", + "# Monthly mean coherence GIF\n", + "if coh_paths:\n", + " monthly_dir = f\"{gif_dir}/coh_monthly_frames\"\n", + " os.makedirs(monthly_dir, exist_ok=True)\n", + " # Build month index using midpoint dates\n", + " monthly = {}\n", + " for p, d in zip(coh_paths, coh_dates):\n", + " if d is None:\n", + " continue\n", + " key = d.strftime('%Y-%m')\n", + " monthly.setdefault(key, []).append(p)\n", + "\n", + " monthly_frames = []\n", + " for key in sorted(monthly.keys()):\n", + " stacks = []\n", + " for p in monthly[key]:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " if coh_template is not None:\n", + " da = da.rio.reproject_match(coh_template)\n", + " stacks.append(da)\n", + " if not stacks:\n", + " continue\n", + " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", + " minlon, minlat, maxlon, maxlat = da_month.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " fig, ax = plt.subplots(figsize=(6,4))\n", + " im = ax.imshow(da_month.values, cmap=cmc.bilbao, origin='upper', extent=bbox, interpolation='none', norm=coh_norm, vmin=None if COH_USE_GAMMA_NORM else 0, vmax=None if COH_USE_GAMMA_NORM else 1)\n", + " ax.set_title(key, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", + " cb.set_label('Mean coherence')\n", + " frame_path = os.path.join(monthly_dir, f\"{key}.png\")\n", + " fig.savefig(frame_path, dpi=150)\n", + " plt.close(fig)\n", + " monthly_frames.append(frame_path)\n", + "\n", + " if monthly_frames:\n", + " monthly_gif = f\"{gif_dir}/coherence_monthly.gif\"\n", + " monthly_imgs = _pad_frames(monthly_frames)\n", + " imageio.mimsave(monthly_gif, monthly_imgs, duration=_gif_duration(len(monthly_imgs)))\n", + " print(f\"Wrote {monthly_gif}\")\n", + " else:\n", + " print('No monthly coherence frames found for GIF')\n", + "\n", + "# Backscatter GIF (sigma0/beta0)\n", + "bsc_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_bsc_{CALIBRATION_MODE}_*.tif\"))\n", + "bsc_paths = _filter_paths_by_date(bsc_paths, dateStart, dateEnd)\n", + "bsc_dates = []\n", + "\n", + "# Fixed color range across the series\n", + "bsc_all = []\n", + "for p in bsc_paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " bsc_all.append(da.values.ravel())\n", + "if bsc_all:\n", + " bsc_all = np.concatenate(bsc_all)\n", + " bsc_vmin = np.nanpercentile(bsc_all, 2)\n", + " bsc_vmax = np.nanpercentile(bsc_all, 90)\n", + "else:\n", + " bsc_vmin, bsc_vmax = None, None\n", + "for p in bsc_paths:\n", + " tag = Path(p).name.replace('.tif','')\n", + " bsc_dates.append(_extract_date_from_tag(tag))\n", + "\n", + "if bsc_paths:\n", + " bsc_template = rioxarray.open_rasterio(bsc_paths[0])[0]\n", + " minlon, minlat, maxlon, maxlat = bsc_template.rio.bounds()\n", + " bsc_extent = [minlon, maxlon, minlat, maxlat]\n", + " bsc_frames = _render_frames(\n", + " bsc_paths, f\"{gif_dir}/bsc_frames_{CALIBRATION_MODE}\", cmap=cmc.bilbao,\n", + " vmin=bsc_vmin, vmax=bsc_vmax, title_prefix=None, extent=bsc_extent,\n", + " cbar_label=f\"Backscatter ({CALIBRATION_MODE}) dB\", cbar_ticks=None,\n", + " template=bsc_template, dates=bsc_dates, title_dates=None, show_time_bar=True\n", + " )\n", + " bsc_gif = f\"{gif_dir}/backscatter_{CALIBRATION_MODE}.gif\"\n", + " bsc_imgs = _pad_frames(bsc_frames)\n", + " imageio.mimsave(bsc_gif, bsc_imgs, duration=_gif_duration(len(bsc_imgs)))\n", + " print(f\"Wrote {bsc_gif}\")\n", + "else:\n", + " print('No merged backscatter files found for GIF')\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "opera_cslc_slides", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/CSLC/Landslides/environment.yml b/CSLC/Landslides/environment.yml new file mode 100644 index 0000000..4f31f00 --- /dev/null +++ b/CSLC/Landslides/environment.yml @@ -0,0 +1,32 @@ +name: opera_cslc_slides +channels: + - conda-forge +dependencies: + - python>=3.10 + - asf_search + - affine + - cartopy + - cmcrameri + - folium + - gdal + - geopandas + - h5py + - imageio + - ipykernel + - jupyterlab + - matplotlib + - numpy + - opera-utils + - pandas + - pyproj + - rasterio + - requests + - rioxarray + - rich + - shapely + - tqdm + - xarray + - pip + - pip: + - seaborn + - watermark From 5861b1fa993aa9de6a89c950abccc822140e822d Mon Sep 17 00:00:00 2001 From: Al Handwerger Date: Tue, 3 Feb 2026 09:48:04 -0800 Subject: [PATCH 6/7] Delete CSLC/Landslides/CSLC-S1_for_landslides.ipynb --- CSLC/Landslides/CSLC-S1_for_landslides.ipynb | 2368 ------------------ 1 file changed, 2368 deletions(-) delete mode 100644 CSLC/Landslides/CSLC-S1_for_landslides.ipynb diff --git a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb deleted file mode 100644 index 93ac184..0000000 --- a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb +++ /dev/null @@ -1,2368 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "36315bdf", - "metadata": {}, - "source": [ - "# Generate wrapped interferograms, coherence, and backscatter (dB) maps and animations using OPERA CSLC-S1\n", - "\n", - "--- \n", - "\n", - "This notebook:\n", - "- Searches OPERA CSLC-S1 products for your AOI + date range\n", - "- Subsets CSLCs **before download** using opera-utils\n", - "- Builds interferograms/coherence and merges bursts\n", - "- Exports mosaics and visualizations\n", - "\n", - "**Quick start**\n", - "1) Set parameters in the next cell (AOI, date range, pairing)\n", - "2) Run cells top-to-bottom\n", - "3) Outputs land in `savedir/`\n", - "\n", - "**Key toggles**\n", - "- `SAVE_WGS84`: save WGS84 GeoTIFF mosaics when True\n", - "- `DOWNLOAD_WITH_releaseOGRESS`: show progress bar for downloads\n", - "- `USE_WATER_YEAR`: Oct–Sep calendar layout when True\n", - "- `pair_mode` / `t_span`: control IFG pairing (all vs fixed separation)\n", - "\n", - "**Outputs**\n", - "- Subset CSLC H5: `savedir/subset_cslc/*.h5`\n", - "- Mosaics (IFG/COH, native CRS): `savedir/tifs/merged_ifg_*`, `merged_coh_*`\n", - "- WGS84 mosaics: `savedir/tifs/WGS84/merged_ifg_WGS84_*`, `merged_coh_WGS84_*`, `merged_bsc__WGS84_*` (backscatter, dB)\n", - "- Calibrated backscatter (dB) mosaics (native CRS; = sigma0 or beta0): `savedir/tifs/merged_bsc__*.tif`\n", - "- GIFs: `savedir/gifs/*.gif`\n", - "\n", - "### Data Used in the Example: \n", - "\n", - "- **10 meter (Northing) x 5 meter (Easting) North America OPERA Coregistered Single Look Complex from Sentinel-1 products**\n", - " - This dataset contains Level-2 OPERA coregistered single-look-complex (CSLC) data from Sentinel-1 (S1). The data in this example are geocoded CSLC-S1 data covering Palos Verdes landslides, California, USA. \n", - " \n", - " - The OPERA project is generating geocoded burst-wise CSLC-S1 products over North America which includes USA and US Territories within 200 km from the US border, Canada, and all mainland countries from the southern US border down to and including Panama. Each pixel within a burst SLC is represented by a complex number and contains both the amplitude and phase information. The CSLC-S1 products are distributed over projected map coordinates using the Universal Transverse Mercator (UTM) projection with spacing in the X- and Y-directions of 5 m and 10 m, respectively. Each OPERA CSLC-S1 product is distributed as a HDF5 file following the CF-1.8 convention with separate groups containing the data raster layers, the low-resolution correction layers, and relevant product metadata.\n", - "\n", - " - For more information about the OPERA project and other products please visit our website at https://www.jpl.nasa.gov/go/opera .\n", - "\n", - "Please refer to the [OPERA Product Specification Document](https://d2pn8kiwq2w21t.cloudfront.net/documents/OPERA_CSLC-S1_ProductSpec_v1.0.0_D-108278_Initial_2023-09-11_URS321269.pdf) for details about the CSLC-S1 product.\n", - "\n", - "*Prepared by Al Handwerger and M. Grace Bato*\n", - "\n", - "---\n", - "\n", - "## 0. Setup your conda environment\n", - "\n", - "Assuming you have conda installed. Open your terminal and run the following:\n", - "```\n", - "\n", - "# Create the OPERA CSLC environment\n", - "conda (or mamba) env create -f environment.yml\n", - "conda (or mamba) activate opera_cslc_slides\n", - "python -m ipykernel install --user --name opera_cslc_slides\n", - "\n", - "```\n", - "\n", - "---\n", - "\n", - "## 1. Load Python modules" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "84a1468d-0aaf-4f06-9875-6b753d94ad42", - "metadata": {}, - "outputs": [], - "source": [ - "## Load necessary modules\n", - "%load_ext watermark\n", - "\n", - "import asf_search as asf\n", - "import cartopy.crs as ccrs\n", - "import cmcrameri.cm as cmc\n", - "import datetime as dt\n", - "import folium\n", - "import geopandas as gpd\n", - "import glob\n", - "import h5py\n", - "import imageio.v2 as imageio\n", - "import math\n", - "import matplotlib.colors as mcolors\n", - "import matplotlib.patches as patches\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from numpy.lib.stride_tricks import sliding_window_view\n", - "from numpy.typing import NDArray\n", - "import os\n", - "import pandas as pd\n", - "from pathlib import Path\n", - "from pyproj import Transformer\n", - "import rasterio\n", - "from rasterio import merge\n", - "from rasterio.crs import CRS\n", - "from rasterio.io import MemoryFile\n", - "from rasterio.transform import from_origin\n", - "from rasterio.warp import calculate_default_transform, reproject, Resampling\n", - "import requests\n", - "import re\n", - "import rioxarray\n", - "from shapely.geometry import box, Point\n", - "from shapely.ops import transform as shp_transform\n", - "import shapely\n", - "import shapely.wkt as wkt\n", - "from subprocess import Popen\n", - "from platform import system\n", - "import sys\n", - "import tempfile\n", - "import time\n", - "from tqdm.auto import tqdm\n", - "import warnings\n", - "\n", - "from affine import Affine\n", - "from concurrent.futures import ThreadPoolExecutor, as_completed\n", - "from getpass import getpass\n", - "from netrc import netrc\n", - "from opera_utils.credentials import get_earthdata_username_password\n", - "from opera_utils.disp._remote import open_file\n", - "from opera_utils.disp._utils import _get_netcdf_encoding\n", - "from osgeo import gdal\n", - "\n", - "proj_dir = os.path.join(sys.prefix, \"share\", \"proj\")\n", - "os.environ[\"PROJ_LIB\"] = proj_dir # for older PROJ\n", - "os.environ[\"PROJ_DATA\"] = proj_dir # for newer PROJ\n", - "\n", - "%watermark --iversions\n", - "\n", - "\n", - "import seaborn as sns\n", - "# ROCKET_CMAP = sns.color_palette('rocket', as_cmap=True)\n", - "from scipy.signal import convolve\n", - "from scipy.ndimage import distance_transform_edt\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9952139c", - "metadata": {}, - "outputs": [], - "source": [ - "# Environment check\n", - "import sys\n", - "import importlib\n", - "\n", - "REQUIRED_PKGS = [\n", - " 'asf_search','cartopy','folium','geopandas','h5py','imageio','matplotlib','numpy','pandas',\n", - " 'pyproj','rasterio','rioxarray','shapely','xarray','opera_utils','tqdm','rich','cmcrameri','seaborn'\n", - "]\n", - "missing = []\n", - "for pkg in REQUIRED_PKGS:\n", - " try:\n", - " importlib.import_module(pkg)\n", - " except Exception:\n", - " missing.append(pkg)\n", - "\n", - "if missing:\n", - " raise ImportError(\"Missing packages: \" + ', '.join(missing) + \". Activate opera_cslc env or install from environment.yml\")\n", - "\n", - "print(f\"Python: {sys.executable}\")\n", - "# Colormap sanity check\n", - "import cmcrameri.cm as cmc\n", - "# _ = ROCKET_CMAP\n", - "\n", - "print('Environment check OK')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1faa1ef0-3d5e-424e-b602-3392b720af6d", - "metadata": {}, - "outputs": [], - "source": [ - "## Notebook display setup\n", - "%matplotlib inline\n", - "%config InlineBackend.figure_format='retina'\n", - "\n", - "# Pandas display\n", - "# pd.set_option('display.max_rows', None)\n", - "pd.set_option('display.max_columns', None)\n", - "\n", - "# Optional reprojection helper for display\n", - "\n", - "# Avoid lots of warnings printing to notebook from asf_search\n", - "warnings.filterwarnings('ignore')" - ] - }, - { - "cell_type": "markdown", - "id": "32ef4ad4", - "metadata": {}, - "source": [ - "## 2. Set up your NASA Earthdata Login Credentials" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0b6ad5ac-9089-4377-be63-ddf1731263f2", - "metadata": {}, - "outputs": [], - "source": [ - "urs = 'urs.earthdata.nasa.gov'\n", - "prompts = ['Enter NASA Earthdata Login Username: ',\n", - " 'Enter NASA Earthdata Login Password: ']\n", - "\n", - "netrc_name = \"_netrc\" if system() == \"Windows\" else \".netrc\"\n", - "netrc_path = os.path.expanduser(f\"~/{netrc_name}\")\n", - "\n", - "def write_netrc():\n", - " username = getpass(prompt=prompts[0])\n", - " password = getpass(prompt=prompts[1])\n", - " with open(netrc_path, 'a') as f:\n", - " f.write(f\"\\nmachine {urs}\\n\")\n", - " f.write(f\"login {username}\\n\")\n", - " f.write(f\"password {password}\\n\")\n", - " os.chmod(netrc_path, 0o600)\n", - "\n", - "def has_urs_credentials():\n", - " try:\n", - " creds = netrc(netrc_path).authenticators(urs)\n", - " return creds is not None\n", - " except (FileNotFoundError, NetrcParseError):\n", - " return False\n", - "\n", - "if not has_urs_credentials():\n", - " if not os.path.exists(netrc_path):\n", - " open(netrc_path, 'w').close()\n", - " write_netrc()\n", - "\n", - "os.environ[\"GDAL_HTTP_NETRC\"] = \"YES\"\n", - "os.environ[\"GDAL_HTTP_NETRC_FILE\"] = netrc_path" - ] - }, - { - "cell_type": "markdown", - "id": "4f948345", - "metadata": {}, - "source": [ - "## 3. Enter user-defined parameters\n", - "\n", - "**Parameter guide (impact on downstream steps):**\n", - "- **AOI / orbit / path / burst filters**: control which CSLCs are found and subset; changing these changes *all* downstream data.\n", - "- **dateStart / dateEnd**: controls query window and calendar/GIF coverage.\n", - "- **pair_mode / pair_t_span_days**: controls which interferometric pairs are formed; impacts IFG/COH density.\n", - "- **MULTILOOK / TARGET_PIXEL_M**: sets spatial averaging; affects resolution and noise level in IFG/COH/BSC.\n", - "- **CALIBRATION_MODE**: choose backscatter calibration: `sigma0` or `beta0`.\n", - "- **COH_METHOD**: `standard` (default) or `phase_only` (previous behavior).\n", - "- **COH_APPLY_LEE**: apply Lee filter to coherence (optional smoothing).\n", - "- **IFG_APPLY_FILTER**: apply Goldstein filter to interferogram phase.\n", - "- **Note**: `IFG_APPLY_FILTER` only affects the **interferogram phase** (for display/plots). Coherence is always computed from the unfiltered data; use `COH_METHOD` to choose standard vs phase-only, and `COH_APPLY_LEE` for optional smoothing.\n", - "- **MERGE_BLEND_OVERLAP**: if `True`, blend overlap regions between bursts using a distance‑based feather to reduce seams.\n", - "- **MERGE_BLEND_EPS**: small constant to avoid divide‑by‑zero in blend weights. Increase slightly if you see artifacts.\n", - "- **APPLY_WATER_MASK / WATER_MASK_PATH**: masks out water before saving outputs.\n", - "- **SAVE_WGS84**: optional WGS84 GeoTIFFs for easy display.\n", - "- **SKIP_EXISTING_TIFS**: reuses existing GeoTIFFs to speed re-runs (delete old files to force regeneration).\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d5eeba6e", - "metadata": {}, - "outputs": [], - "source": [ - "# User parameters (edit these)\n", - "# AOI is a WKT polygon in EPSG:4326\n", - "# check ASF Vertex to look for correct pass/path for your AOI https://search.asf.alaska.edu/#/?maxResults=250&dataset=OPERA-S1&polygon=POLYGON((-118.3955%2033.7342,-118.3464%2033.7342,-118.3464%2033.7616,-118.3955%2033.7616,-118.3955%2033.7342))&productTypes=CSLC&resultsLoaded=true&granule=OPERA_L2_CSLC-S1_T071-151230-IW3_20260128T135247Z_20260129T074245Z_S1A_VV_v1.1&zoom=10.065¢er=-118.699,33.446\n", - "## Enter user-defined parameters\n", - "SITE_NAME = \"Palos_Verdes_Landslides\" # used for output folder naming\n", - "aoi = \"POLYGON((-118.3955 33.7342,-118.3464 33.7342,-118.3464 33.7616,-118.3955 33.7616,-118.3955 33.7342))\"\n", - "orbitPass = \"DESCENDING\" # ASCENDING or DESCENDING\n", - "pathNumber = 71 #71 DESC and 64 ASC\n", - "# Optional burst selection before download\n", - "# Use subswath (e.g., 'IW2', 'IW3') or specific OPERA burst ID (e.g., 'T071_151230_IW3')\n", - "BURST_SUBSWATH = 'IW3' # e.g., 'IW2' or ['IW2', 'IW3'] or None \n", - "# BURST_SUBSWATH = 'IW3' # e.g., 'IW2' or ['IW2', 'IW3'] or None \n", - "\n", - "BURST_ID = None # e.g., 'T071_151230_IW3' or list of burst IDs\n", - "\n", - "dateStart = dt.datetime.fromisoformat('2024-10-01 00:00:00') #'YY/YY-MM-DD HH:MM:SS'\n", - "dateEnd = dt.datetime.fromisoformat('2025-09-30 23:59:59') #'YYYY-MM-DD HH:MM:SS'\n", - "\n", - "# Pairing options\n", - "pair_mode = 't_span' # 'all' or 't_span'\n", - "pair_t_span_days = [12] # int or list of ints (e.g., [6, 12, 24])\n", - "\n", - "# Seasonal filters (exclude pairs if either endpoint falls in these windows)\n", - "EXCLUDE_DATE_RANGES = [] # list of (start, end) like [(\"2023-12-15\",\"2024-03-31\")] or []\n", - "EXCLUDE_MONTHDAY_RANGES = [] # list of (\"MM-DD\",\"MM-DD\") e.g., [(\"12-15\",\"02-15\")], or []\n", - "\n", - "#backscatter\n", - "CALIBRATION_MODE = 'sigma0' # 'sigma0' or 'beta0'\n", - "CALIBRATION_FACTOR_IS_AMPLITUDE = True # False: LUT applies to power (default); True: LUT applies to amplitude, so use squared factor\n", - "\n", - "# Multilooking (spatial averaging)\n", - "# Set either MULTILOOK (looks_y, looks_x) OR TARGET_PIXEL_M (meters). TARGET overrides MULTILOOK.\n", - "MULTILOOK = (1, 1) # e.g., (3, 6) for 30m from (dy=10m, dx=5m)\n", - "TARGET_PIXEL_M = None # e.g., 30.0 meter or 90.0 meter or None\n", - "GOLDSTEIN_ALPHA = 0.5 # 0 (none) to 1 (strong)\n", - "COH_METHOD = 'standard' # 'standard' or 'phase_only'\n", - "COH_USE_GAMMA_NORM = True # True to apply gamma normalization in plots (coherence only)\n", - "COH_NORM_GAMMA = 0.5\n", - "COH_KERNEL = 'boxcar' # 'boxcar' or 'weighted'\n", - "COH_KERNEL_NUM_CONV = 3 # only used for weighted kernel\n", - "COH_APPLY_LEE = False # apply Lee filter to coherence\n", - "IFG_APPLY_FILTER = True # apply Goldstein filter to interferogram phase\n", - "\n", - "#for burst overlaps\n", - "MERGE_BLEND_OVERLAP = True # feather overlaps to reduce burst overlaps\n", - "MERGE_BLEND_EPS = 1e-6\n", - "\n", - "SAVE_WGS84 = False # set True to save WGS84 GeoTIFF mosaics\n", - "SKIP_EXISTING_TIFS = False # skip writing GeoTIFFs if they already exist\n", - "\n", - "DOWNLOAD_WITH_PROGRESS = True # set True for per-file progress bar\n", - "\n", - "\n", - "min_t_span_days = 0 # minimum separation (days)\n", - "max_t_span_days = 12 # maximum separation (days) or None\n", - "max_pairs_per_burst = None # int or None\n", - "max_pairs_total = None # int or None\n", - "\n", - "# Calendar settings\n", - "USE_WATER_YEAR = True # True: Oct–Sep, False: Jan–Dec\n", - "\n", - "DOWNLOAD_PROCESSES = min(8, max(2, (os.cpu_count() or 4) // 2))\n", - "# DOWNLOAD_BATCH_SIZE = 5\n", - "\n", - "\n", - "# Normalize name for filesystem (letters/numbers/_/- only)\n", - "site_slug = re.sub(r\"[^A-Za-z0-9_-]+\", \"\", SITE_NAME)\n", - "orbit_code = orbitPass[0].upper() # 'A' or 'D'\n", - "savedir = f'./{site_slug}_{orbit_code}{pathNumber:03d}/'\n", - "\n", - "# Water mask options\n", - "# in params cell\n", - "WATER_MASK_PATH = f\"{savedir}/water_mask/water_mask_esa_wc2021.tif\"\n", - "APPLY_WATER_MASK = True" - ] - }, - { - "cell_type": "markdown", - "id": "80842b39", - "metadata": {}, - "source": [ - "**AOI size guidance**\n", - "\n", - "This notebook is tuned for *small* AOIs (e.g., individual landslides). The main opera-utils release includes fixes for small AOI subsetting. Large AOIs can dramatically increase download time, disk usage, and RAM needs. If you use a large polygon, consider:\n", - "- Shorter date ranges or fewer bursts\n", - "- Coarser `TARGET_PIXEL_M` (more multilooking)\n", - "- Tiling the AOI into smaller polygons and mosaicking later\n", - "\n", - "If you see memory errors or very long runtimes, reduce the AOI size or date range.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "89fa0047", - "metadata": {}, - "outputs": [], - "source": [ - "# Utilities (used by multiple sections)\n", - "\n", - "def _load_water_mask_match(mask_path, shape, transform, crs):\n", - " if mask_path is None or not Path(mask_path).exists():\n", - " return None\n", - " with rasterio.open(mask_path) as src:\n", - " src_mask = src.read(1)\n", - " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", - " return src_mask\n", - " dst = np.zeros(shape, dtype=src_mask.dtype)\n", - " reproject(\n", - " source=src_mask,\n", - " destination=dst,\n", - " src_transform=src.transform,\n", - " src_crs=src.crs,\n", - " dst_transform=transform,\n", - " dst_crs=crs,\n", - " resampling=Resampling.nearest,\n", - " )\n", - " return dst\n", - "\n", - "\n", - "def _apply_water_mask(arr, mask):\n", - " # mask: 1 = keep land, 0 = water\n", - " return np.where(mask == 0, np.nan, arr)\n", - "\n", - "\n", - "def _trim_nan_border(arr, transform):\n", - " data = arr[0] if arr.ndim == 3 else arr\n", - " mask = np.isfinite(data) & (data != 0)\n", - " if not mask.any():\n", - " return arr, transform\n", - " rows = np.where(mask.any(axis=1))[0]\n", - " cols = np.where(mask.any(axis=0))[0]\n", - " r0, r1 = rows[0], rows[-1] + 1\n", - " c0, c1 = cols[0], cols[-1] + 1\n", - " data = data[r0:r1, c0:c1]\n", - " if arr.ndim == 3:\n", - " arr = data[None, ...]\n", - " else:\n", - " arr = data\n", - " new_transform = transform * Affine.translation(c0, r0)\n", - " return arr, new_transform\n", - "\n", - "\n", - "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", - " dst_crs = 'EPSG:4326'\n", - " dst_transform, width, height = calculate_default_transform(\n", - " f'EPSG:{epsg}', dst_crs, mosaic.shape[2], mosaic.shape[1],\n", - " *rasterio.transform.array_bounds(mosaic.shape[1], mosaic.shape[2], transform)\n", - " )\n", - " dest = np.zeros((1, height, width), dtype=mosaic.dtype)\n", - " reproject(\n", - " source=mosaic,\n", - " destination=dest,\n", - " src_transform=transform,\n", - " src_crs=f'EPSG:{epsg}',\n", - " dst_transform=dst_transform,\n", - " dst_crs=dst_crs,\n", - " resampling=Resampling.nearest,\n", - " )\n", - " out_meta = {\n", - " 'driver': 'GTiff',\n", - " 'height': height,\n", - " 'width': width,\n", - " 'count': 1,\n", - " 'dtype': mosaic.dtype,\n", - " 'crs': dst_crs,\n", - " 'transform': dst_transform,\n", - " }\n", - " with rasterio.open(out_path, 'w', **out_meta) as dst:\n", - " dst.write(dest)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6f072529", - "metadata": {}, - "outputs": [], - "source": [ - "# ESA WorldCover 2021 water mask (GDAL-only)\n", - "\n", - "ESA_WC_GRID_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/esa_worldcover_grid.fgb\"\n", - "ESA_WC_BASE_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/v200/2021/map\"\n", - "# ESA WorldCover class codes: 80 = Permanent water bodies\n", - "ESA_WC_WATER_CLASSES = {80}\n", - "\n", - "\n", - "def build_worldcover_water_mask(aoi_wkt, out_path, target_res_deg=None):\n", - " # Create a binary land mask from ESA WorldCover (1=land, 0=water).\n", - " out_path = Path(out_path)\n", - " out_path.parent.mkdir(parents=True, exist_ok=True)\n", - " if out_path.exists():\n", - " return out_path\n", - "\n", - "\n", - " aoi_geom = shapely.wkt.loads(aoi_wkt)\n", - " # Load tile grid and select intersecting tiles\n", - " grid = gpd.read_file(ESA_WC_GRID_URL)\n", - " grid = grid.to_crs(\"EPSG:4326\")\n", - " # Find tile id column (varies by grid version)\n", - " tile_col = next((c for c in grid.columns if 'tile' in c.lower()), None)\n", - " if tile_col is None:\n", - " raise RuntimeError(f\"No tile column found in grid columns: {list(grid.columns)}\")\n", - " tiles = grid[grid.intersects(aoi_geom)][tile_col].tolist()\n", - " if not tiles:\n", - " raise RuntimeError(\"No WorldCover tiles intersect AOI\")\n", - "\n", - " print(f\"Selected tiles: {tiles}\")\n", - "\n", - " tile_urls = [\n", - " f\"{ESA_WC_BASE_URL}/ESA_WorldCover_10m_2021_v200_{t}_Map.tif\"\n", - " for t in tiles\n", - " ]\n", - "\n", - " # Quick URL check for first tile\n", - " first_url = tile_urls[0]\n", - " try:\n", - " _ = gdal.Open(first_url)\n", - " except Exception as e:\n", - " raise RuntimeError(f\"GDAL cannot open first tile URL: {first_url}\\n{e}\")\n", - "\n", - " vrt_path = out_path.with_suffix(\".vrt\")\n", - " gdal.BuildVRT(str(vrt_path), tile_urls)\n", - "\n", - " minx, miny, maxx, maxy = aoi_geom.bounds\n", - " warp_kwargs = dict(\n", - " format=\"GTiff\",\n", - " outputBounds=[minx, miny, maxx, maxy],\n", - " multithread=True,\n", - " )\n", - " if target_res_deg is not None:\n", - " warp_kwargs.update(dict(xRes=target_res_deg, yRes=target_res_deg, targetAlignedPixels=True))\n", - "\n", - " tmp_map = out_path.with_name(out_path.stem + \"_map.tif\")\n", - " warp_ds = gdal.Warp(str(tmp_map), str(vrt_path), **warp_kwargs)\n", - " if warp_ds is None:\n", - " raise RuntimeError(\"GDAL Warp returned None. Check network access/URL.\")\n", - " warp_ds = None\n", - "\n", - " with rasterio.open(tmp_map) as src:\n", - " data = src.read(1)\n", - " profile = src.profile\n", - "\n", - " mask = (~np.isin(data, list(ESA_WC_WATER_CLASSES))).astype(\"uint8\")\n", - " profile.update(dtype=\"uint8\", count=1, nodata=0)\n", - "\n", - " with rasterio.open(out_path, \"w\", **profile) as dst:\n", - " dst.write(mask, 1)\n", - "\n", - " return out_path\n", - "\n", - "\n", - "if APPLY_WATER_MASK and WATER_MASK_PATH:\n", - " WATER_MASK_PATH = build_worldcover_water_mask(aoi, WATER_MASK_PATH)" - ] - }, - { - "cell_type": "markdown", - "id": "98916bef", - "metadata": {}, - "source": [ - "## 4. Query OPERA CSLCs using `asf_search`" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5e33d72e-2e75-4099-93d6-5cd058643e35", - "metadata": {}, - "outputs": [], - "source": [ - "## Search for OPERA CSLC data in ASF DAAC\n", - "try:\n", - " search_params = dict(\n", - " intersectsWith= aoi,\n", - " dataset='OPERA-S1',\n", - " processingLevel='CSLC',\n", - " flightDirection = orbitPass,\n", - " start=dateStart,\n", - " end=dateEnd)\n", - "\n", - " ## Return results\n", - " results = asf.search(**search_params)\n", - " print(f\"Length of Results: {len(results)}\")\n", - "\n", - "except TypeError:\n", - " search_params = dict(\n", - " intersectsWith= aoi.wkt,\n", - " dataset='OPERA-S1',\n", - " processingLevel='CSLC',\n", - " flightDirection = orbitPass,\n", - " start=dateStart,\n", - " end=dateEnd)\n", - "\n", - " ## Return results\n", - " results = asf.search(**search_params)\n", - " print(f\"Length of Results: {len(results)}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b2f5b5df", - "metadata": {}, - "outputs": [], - "source": [ - "## Save the results in a geopandas dataframe\n", - "gf = gpd.GeoDataFrame.from_features(results.geojson(), crs='EPSG:4326')\n", - "\n", - "## Filter data based on specified track number\n", - "gf = gf[gf.pathNumber==pathNumber]\n", - "# gf = gf[gf.pgeVersion==\"2.1.1\"] " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fe3a0963-1d13-4b99-955a-b2c0fa45f1e3", - "metadata": {}, - "outputs": [], - "source": [ - "# Get only relevant metadata\n", - "cslc_df = gf[['operaBurstID', 'fileID', 'startTime', 'stopTime', 'url', 'geometry', 'pgeVersion']]\n", - "cslc_df['startTime'] = pd.to_datetime(cslc_df.startTime).dt.date\n", - "cslc_df['stopTime'] = pd.to_datetime(cslc_df.stopTime).dt.date\n", - "\n", - "# Extract production time from fileID (2nd date token)\n", - "def _prod_time_from_fileid(file_id):\n", - " # Example: OPERA_L2_CSLC-S1_..._20221122T161650Z_20240504T081640Z_...\n", - " parts = str(file_id).split('_')\n", - " return parts[5] if len(parts) > 5 else None\n", - "\n", - "cslc_df['productionTime'] = pd.to_datetime(cslc_df['fileID'].apply(_prod_time_from_fileid), format='%Y%m%dT%H%M%SZ', errors='coerce')\n", - "\n", - "# Keep newest duplicate by productionTime (fallback to pgeVersion, stopTime)\n", - "cslc_df = cslc_df.sort_values(by=['operaBurstID', 'startTime', 'productionTime', 'pgeVersion', 'stopTime'])\n", - "cslc_df = cslc_df.drop_duplicates(subset=['operaBurstID', 'startTime'], keep='last', ignore_index=True)\n", - "\n", - "\n", - "def _subswath_from_fileid(file_id):\n", - " # Example: ...-IW2_... -> IW2\n", - " m = re.search(r\"-IW[1-3]_\", str(file_id))\n", - " return m.group(0)[1:4] if m else None\n", - "\n", - "cslc_df['burstSubswath'] = cslc_df['fileID'].apply(_subswath_from_fileid)\n", - "\n", - "# Optional filtering by subswath or specific burst IDs\n", - "if BURST_SUBSWATH:\n", - " if isinstance(BURST_SUBSWATH, (list, tuple, set)):\n", - " subswaths = {str(s).upper() for s in BURST_SUBSWATH}\n", - " else:\n", - " subswaths = {str(BURST_SUBSWATH).upper()}\n", - " cslc_df = cslc_df[cslc_df['burstSubswath'].str.upper().isin(subswaths)]\n", - "\n", - "if BURST_ID:\n", - " if isinstance(BURST_ID, (list, tuple, set)):\n", - " burst_ids = {str(b) for b in BURST_ID}\n", - " else:\n", - " burst_ids = {str(BURST_ID)}\n", - " cslc_df = cslc_df[cslc_df['operaBurstID'].isin(burst_ids)]\n", - "cslc_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "76cb1007", - "metadata": {}, - "outputs": [], - "source": [ - "# Build AOI geometry\n", - "aoi_geom = wkt.loads(aoi)\n", - "aoi_gdf = gpd.GeoDataFrame(geometry=[aoi_geom], crs=\"EPSG:4326\")\n", - "\n", - "# Map center\n", - "centroid = aoi_gdf.geometry[0].centroid\n", - "m = folium.Map(location=[centroid.y, centroid.x], zoom_start=9, tiles=\"Esri.WorldImagery\")\n", - "\n", - "# Add CSLC footprints\n", - "folium.GeoJson(\n", - " data=cslc_df[['operaBurstID','geometry']].set_geometry('geometry').to_crs(\"EPSG:4326\").__geo_interface__,\n", - " name=\"CSLC footprints\",\n", - " style_function=lambda x: {\n", - " \"fillColor\": \"blue\",\n", - " \"color\": \"blue\",\n", - " \"weight\": 2,\n", - " \"fillOpacity\": 0.1,\n", - " },\n", - ").add_to(m)\n", - "\n", - "# Add AOI\n", - "folium.GeoJson(\n", - " data=aoi_gdf.__geo_interface__,\n", - " name=\"AOI\",\n", - " style_function=lambda x: {\n", - " \"fillColor\": \"red\",\n", - " \"color\": \"red\",\n", - " \"weight\": 2,\n", - " \"fillOpacity\": 0.1,\n", - " },\n", - ").add_to(m)\n", - "\n", - "folium.LayerControl().add_to(m)\n", - "\n", - "m" - ] - }, - { - "cell_type": "markdown", - "id": "9ca8a611", - "metadata": {}, - "source": [ - "## 5. Download the CSLC-S1 locally" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aea6a5b7", - "metadata": {}, - "outputs": [], - "source": [ - "## Download step skipped: CSLC subsets are streamed via opera-utils\n", - "print('Skipping full CSLC downloads; using opera-utils HTTP subsetting.')\n", - "\n", - "# Sort the CSLC-S1 by burstID and date\n", - "cslc_df = cslc_df.sort_values(by=[\"operaBurstID\", \"startTime\"], ignore_index=True)\n", - "cslc_df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa4801fc", - "metadata": {}, - "outputs": [], - "source": [ - "# Enforce date range on dataframe (useful when re-running with narrower dates)\n", - "date_start_day = dateStart.date()\n", - "date_end_day = dateEnd.date()\n", - "cslc_df = cslc_df[(cslc_df['startTime'] >= date_start_day) & (cslc_df['startTime'] <= date_end_day)]\n", - "cslc_df = cslc_df.reset_index(drop=True)\n", - "cslc_df" - ] - }, - { - "cell_type": "markdown", - "id": "48add9c3", - "metadata": {}, - "source": [ - "## 6. Read each CSLC-S1 and stack them together\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "01891a96", - "metadata": {}, - "outputs": [], - "source": [ - "cslc_stack = []; cslc_dates = []; bbox_stack = []; xcoor_stack = []; ycoor_stack = []\n", - "\n", - "\n", - "subset_dir = f\"{savedir}/subset_cslc\"\n", - "os.makedirs(subset_dir, exist_ok=True)\n", - "\n", - "def _extract_subset(input_obj, outpath, rows, cols, chunks=(1,256,256)):\n", - " X0, X1 = (cols.start, cols.stop) if cols is not None else (None, None)\n", - " Y0, Y1 = (rows.start, rows.stop) if rows is not None else (None, None)\n", - " ds = xr.open_dataset(input_obj, engine=\"h5netcdf\", group=\"data\")\n", - " if 'VV' not in ds.data_vars:\n", - " raise ValueError('Source missing VV data')\n", - " subset = ds.isel(y_coordinates=slice(Y0, Y1), x_coordinates=slice(X0, X1))\n", - " subset.to_netcdf(\n", - " outpath,\n", - " engine=\"h5netcdf\",\n", - " group=\"data\",\n", - " encoding=_get_netcdf_encoding(subset, chunks=chunks),\n", - " )\n", - " for group in (\"metadata\", \"identification\"):\n", - " with h5py.File(input_obj) as hf, h5py.File(outpath, \"a\") as dest_hf:\n", - " hf.copy(group, dest_hf, name=group)\n", - " with h5py.File(outpath, \"a\") as hf:\n", - " ctype = h5py.h5t.py_create(np.complex64)\n", - " ctype.commit(hf[\"/\"].id, np.bytes_(\"complex64\"))\n", - "\n", - "def _subset_h5_to_disk(url, aoi_wkt, out_dir):\n", - " outpath = Path(out_dir) / Path(url).name\n", - " if outpath.exists():\n", - " if outpath.stat().st_size < 100 * 1024:\n", - " outpath.unlink()\n", - " else:\n", - " return outpath\n", - "\n", - " # determine row/col slices by reading coords\n", - " with open_file(url) as in_f:\n", - " ds = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", - " xcoor = ds[\"x_coordinates\"].values\n", - " ycoor = ds[\"y_coordinates\"].values\n", - " epsg = int(ds[\"projection\"].values)\n", - "\n", - " aoi_geom = wkt.loads(aoi_wkt)\n", - " if epsg != 4326:\n", - " transformer = Transformer.from_crs('EPSG:4326', f'EPSG:{epsg}', always_xy=True)\n", - " aoi_geom = shp_transform(transformer.transform, aoi_geom)\n", - " minx, miny, maxx, maxy = aoi_geom.bounds\n", - " x_mask = (xcoor >= minx) & (xcoor <= maxx)\n", - " y_mask = (ycoor >= miny) & (ycoor <= maxy)\n", - " if not x_mask.any() or not y_mask.any():\n", - " raise ValueError('AOI does not intersect this CSLC extent')\n", - " ix = np.where(x_mask)[0]\n", - " iy = np.where(y_mask)[0]\n", - " rows = slice(iy.min(), iy.max()+1)\n", - " cols = slice(ix.min(), ix.max()+1)\n", - "\n", - " if url.startswith('s3://'):\n", - " with open_file(url) as in_f:\n", - " _extract_subset(in_f, outpath, rows, cols)\n", - " else:\n", - " # HTTPS: download to temp then subset\n", - " with tempfile.NamedTemporaryFile(suffix='.h5') as tf:\n", - " if url.startswith('http'):\n", - " session = requests.Session()\n", - " username, password = get_earthdata_username_password()\n", - " session.auth = (username, password)\n", - " resp = session.get(url, stream=True)\n", - " resp.raise_for_status()\n", - " content_type = resp.headers.get('Content-Type', '').lower()\n", - " if 'text/html' in content_type:\n", - " raise ValueError('Got HTML response instead of HDF5; check Earthdata login/auth')\n", - " for chunk in resp.iter_content(chunk_size=1024 * 1024):\n", - " if chunk:\n", - " tf.write(chunk)\n", - " tf.flush()\n", - " if tf.tell() < 100 * 1024:\n", - " raise ValueError('Downloaded file too small; likely auth/redirect issue')\n", - " _extract_subset(tf.name, outpath, rows, cols)\n", - " # validate output has VV; remove tiny/invalid outputs\n", - " try:\n", - " with h5py.File(outpath, 'r') as h5:\n", - " if '/data/VV' not in h5:\n", - " raise ValueError('Subset missing /data/VV')\n", - " if outpath.stat().st_size < 100 * 1024:\n", - " raise ValueError('Subset file too small')\n", - " except Exception:\n", - " if Path(outpath).exists():\n", - " Path(outpath).unlink()\n", - " raise\n", - " return outpath\n", - "\n", - "def _load_subset(file_id, url, start_date):\n", - " try:\n", - " outpath = _subset_h5_to_disk(url, aoi, subset_dir)\n", - " except FileNotFoundError:\n", - " return None # skip missing products\n", - " # now read subset locally with h5py (fast)\n", - " with h5py.File(outpath, 'r') as h5:\n", - " cslc = h5['/data/VV'][:]\n", - " xcoor = h5['/data/x_coordinates'][:]\n", - " ycoor = h5['/data/y_coordinates'][:]\n", - " dx = int(h5['/data/x_spacing'][()])\n", - " dy = int(h5['/data/y_spacing'][()])\n", - " epsg = int(h5['/data/projection'][()])\n", - " sensing_start = h5['/metadata/processing_information/input_burst_metadata/sensing_start'][()].astype(str)\n", - " sensing_stop = h5['/metadata/processing_information/input_burst_metadata/sensing_stop'][()].astype(str)\n", - " dims = h5['/metadata/processing_information/input_burst_metadata/shape'][:]\n", - " bounding_polygon = h5['/identification/bounding_polygon'][()].astype(str)\n", - " orbit_direction = h5['/identification/orbit_pass_direction'][()].astype(str)\n", - " center_lon, center_lat = h5['/metadata/processing_information/input_burst_metadata/center']\n", - " wavelength = h5['/metadata/processing_information/input_burst_metadata/wavelength'][()].astype(str)\n", - " subset_bbox = [float(xcoor.min()), float(xcoor.max()), float(ycoor.min()), float(ycoor.max())]\n", - " return cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox\n", - "\n", - "# Subset with progress (parallel)\n", - "\n", - "items = list(zip(cslc_df.fileID, cslc_df.url, cslc_df.startTime))\n", - "import xarray as xr\n", - "# Diagnostic: check pixel spacing before multilooking\n", - "with open_file(items[0][1]) as in_f:\n", - " ds0 = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", - " dx0 = float(ds0[\"x_spacing\"].values)\n", - " dy0 = float(ds0[\"y_spacing\"].values)\n", - "print(f\"Pixel spacing (dx, dy) = ({dx0}, {dy0})\")\n", - "\n", - "# Derive anisotropic coherence window from TARGET_PIXEL_M or MULTILOOK\n", - "def _odd_at_least_one(n):\n", - " n = max(1, int(round(n)))\n", - " return n if n % 2 == 1 else n + 1\n", - "\n", - "# Aim for ~60 m window; if multilook pixel size exceeds 60 m, use that instead\n", - "if TARGET_PIXEL_M is not None:\n", - " base_win_m = TARGET_PIXEL_M\n", - "else:\n", - " base_win_m = max(abs(dx0) * MULTILOOK[1], abs(dy0) * MULTILOOK[0])\n", - "\n", - "COH_WIN_M = max(60.0, base_win_m)\n", - "COH_WIN_X = _odd_at_least_one(COH_WIN_M / abs(dx0))\n", - "COH_WIN_Y = _odd_at_least_one(COH_WIN_M / abs(dy0))\n", - "print(f\"Using anisotropic COH_WIN (Y,X)=({COH_WIN_Y},{COH_WIN_X}) from COH_WIN_M={COH_WIN_M} m\")\n", - "\n", - "# Derive MULTILOOK from TARGET_PIXEL_M if provided\n", - "if TARGET_PIXEL_M is not None:\n", - " looks_x = max(1, int(round(TARGET_PIXEL_M / abs(dx0))))\n", - " looks_y = max(1, int(round(TARGET_PIXEL_M / abs(dy0))))\n", - " MULTILOOK = (looks_y, looks_x)\n", - " print(f\"Using MULTILOOK={MULTILOOK} for TARGET_PIXEL_M={TARGET_PIXEL_M} (dx={dx0}, dy={dy0})\")\n", - "\n", - "\n", - "results = [None] * len(items)\n", - "_t0 = time.perf_counter()\n", - "with ThreadPoolExecutor(max_workers=DOWNLOAD_PROCESSES) as ex:\n", - " futures = {ex.submit(_load_subset, fileID, url, start_date): i for i, (fileID, url, start_date) in enumerate(items)}\n", - " for fut in tqdm(as_completed(futures), total=len(futures), desc='Subsetting CSLC'):\n", - " i = futures[fut]\n", - " results[i] = fut.result()\n", - "_t1 = time.perf_counter()\n", - "print(f\"Subset/download time: {_t1 - _t0:.1f} s\")\n", - "\n", - "valid_idx = [i for i, res in enumerate(results) if res is not None]\n", - "if len(valid_idx) != len(results):\n", - " print(f\"Skipping {len(results) - len(valid_idx)} failed subsets\")\n", - " cslc_df = cslc_df.iloc[valid_idx].reset_index(drop=True)\n", - " results = [results[i] for i in valid_idx]\n", - "\n", - "for (fileID, start_date), res in zip(zip(cslc_df.fileID, cslc_df.startTime), results):\n", - " cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox = res\n", - " cslc_stack.append(cslc)\n", - " cslc_dates.append(pd.to_datetime(sensing_start).date())\n", - " if subset_bbox is not None:\n", - " bbox = subset_bbox\n", - " else:\n", - " cslc_poly = wkt.loads(bounding_polygon)\n", - " bbox = [cslc_poly.bounds[0], cslc_poly.bounds[2], cslc_poly.bounds[1], cslc_poly.bounds[3]]\n", - " bbox_stack.append(bbox)\n", - " xcoor_stack.append(xcoor)\n", - " ycoor_stack.append(ycoor)" - ] - }, - { - "cell_type": "markdown", - "id": "f5e97e45", - "metadata": {}, - "source": [ - "
\n", - "7. Generate the interferograms, compute for the coherence, save the files as GeoTiffs\n", - "\n", - "
" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d87c1f9", - "metadata": {}, - "outputs": [], - "source": [ - "import h5py, os, glob\n", - "f = sorted(glob.glob(f\"{savedir}/subset_cslc/*.h5\"))[0]\n", - "print(f, os.path.getsize(f))\n", - "with h5py.File(f, \"r\") as h5:\n", - " print(list(h5[\"/data\"].keys()))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "57c74ac4", - "metadata": {}, - "outputs": [], - "source": [ - "def colorize(array=[], cmap='RdBu', cmin=[], cmax=[]):\n", - " normed_data = (array - cmin) / (cmax - cmin) \n", - " cm = plt.cm.get_cmap(cmap)\n", - " return cm(normed_data) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25c56034", - "metadata": {}, - "outputs": [], - "source": [ - "def goldstein_filter(ifg_cpx, alpha=0.5, pad=32, edge_trim=16):\n", - " # Goldstein filter with padding + taper + mask to reduce edge effects\n", - " mask = np.isfinite(ifg_cpx)\n", - " data = np.nan_to_num(ifg_cpx, nan=0.0)\n", - " if pad and pad > 0:\n", - " data = np.pad(data, ((pad, pad), (pad, pad)), mode=\"reflect\")\n", - " mask = np.pad(mask, ((pad, pad), (pad, pad)), mode=\"constant\", constant_values=False)\n", - " # Apply 2D Hann window (taper)\n", - " wy = np.hanning(data.shape[0])\n", - " wx = np.hanning(data.shape[1])\n", - " window = wy[:, None] * wx[None, :]\n", - " f = np.fft.fft2(data * window)\n", - " s = np.abs(f)\n", - " s = s / (s.max() + 1e-8)\n", - " f_filt = f * (s ** alpha)\n", - " out = np.fft.ifft2(f_filt)\n", - " if pad and pad > 0:\n", - " out = out[pad:-pad, pad:-pad]\n", - " mask = mask[pad:-pad, pad:-pad]\n", - " # restore NaNs outside valid mask\n", - " out[~mask] = np.nan\n", - " if edge_trim and edge_trim > 0:\n", - " out[:edge_trim, :] = np.nan\n", - " out[-edge_trim:, :] = np.nan\n", - " out[:, :edge_trim] = np.nan\n", - " out[:, -edge_trim:] = np.nan\n", - " return out" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "82bda5ae", - "metadata": {}, - "outputs": [], - "source": [ - "def rasterWrite(outtif,arr,transform,epsg,dtype='float32'):\n", - " #writing geotiff using rasterio\n", - " \n", - " new_dataset = rasterio.open(outtif, 'w', driver='GTiff',\n", - " height = arr.shape[0], width = arr.shape[1],\n", - " count=1, dtype=dtype,\n", - " crs=CRS.from_epsg(epsg),\n", - " transform=transform,nodata=np.nan)\n", - " new_dataset.write(arr, 1)\n", - " new_dataset.close() " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b47f82a5", - "metadata": {}, - "outputs": [], - "source": [ - "## Build date pairs per burstID\n", - "cslc_dates = cslc_df[[\"startTime\"]]\n", - "burstID = cslc_df.operaBurstID.drop_duplicates(ignore_index=True)\n", - "n_unique_burstID = len(burstID)\n", - "\n", - "def _lag_list(lag):\n", - " if lag is None:\n", - " return []\n", - " if isinstance(lag, (list, tuple, set)):\n", - " return sorted({int(x) for x in lag})\n", - " return [int(lag)]\n", - "\n", - "pair_lags = _lag_list(pair_t_span_days)\n", - "pair_indices = [] # list of (ref_idx, sec_idx) in cslc_df order\n", - "\n", - "for bid, group in cslc_df.groupby('operaBurstID'):\n", - " group = group.sort_values('startTime')\n", - " idx = group.index.to_list()\n", - " dates = group['startTime'].to_list()\n", - "\n", - " burst_pairs = []\n", - " if pair_mode == 'all':\n", - " for i in range(len(idx)):\n", - " for j in range(i+1, len(idx)):\n", - " delta = (dates[j] - dates[i]).days\n", - " if delta < min_t_span_days:\n", - " continue\n", - " if max_t_span_days is not None and delta > max_t_span_days:\n", - " continue\n", - " burst_pairs.append((idx[i], idx[j]))\n", - " elif pair_mode == 't_span':\n", - " for i in range(len(idx)):\n", - " for j in range(i+1, len(idx)):\n", - " delta = (dates[j] - dates[i]).days\n", - " if delta in pair_lags and delta >= min_t_span_days and (max_t_span_days is None or delta <= max_t_span_days):\n", - " burst_pairs.append((idx[i], idx[j]))\n", - " else:\n", - " raise ValueError(\"pair_mode must be 'all' or 't_span'\")\n", - "\n", - " if max_pairs_per_burst is not None:\n", - " burst_pairs = burst_pairs[:int(max_pairs_per_burst)]\n", - "\n", - " pair_indices.extend(burst_pairs)\n", - "\n", - "if max_pairs_total is not None:\n", - " pair_indices = pair_indices[:int(max_pairs_total)]\n", - "\n", - "# Sort pairs by date, then burstID (so same dates group together)\n", - "def _pair_sort_key(pair):\n", - " ref_idx, sec_idx = pair\n", - " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", - " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", - " burst = cslc_df.operaBurstID.iloc[ref_idx]\n", - " return (ref_date, sec_date, burst)\n", - "pair_indices = sorted(pair_indices, key=_pair_sort_key)\n", - "print(f'Pair count: {len(pair_indices)}')\n", - "\n", - "# Seasonal filtering based on pair endpoints\n", - "from datetime import date\n", - "\n", - "# Normalize exclude ranges to date objects\n", - "_excl_ranges = []\n", - "for s, e in EXCLUDE_DATE_RANGES:\n", - " try:\n", - " s_d = pd.to_datetime(s).date()\n", - " e_d = pd.to_datetime(e).date()\n", - " except Exception:\n", - " continue\n", - " _excl_ranges.append((s_d, e_d))\n", - "\n", - "# Normalize month-day ranges\n", - "if EXCLUDE_MONTHDAY_RANGES:\n", - " if len(EXCLUDE_MONTHDAY_RANGES) == 2 and all(isinstance(x, str) for x in EXCLUDE_MONTHDAY_RANGES):\n", - " EXCLUDE_MONTHDAY_RANGES = [(EXCLUDE_MONTHDAY_RANGES[0], EXCLUDE_MONTHDAY_RANGES[1])]\n", - "\n", - "# Filter pairs: exclude if either endpoint falls in excluded months or ranges\n", - "filtered_pairs = []\n", - "for r, s in pair_indices:\n", - " d1 = pd.to_datetime(cslc_dates.iloc[r].values[0]).date()\n", - " d2 = pd.to_datetime(cslc_dates.iloc[s].values[0]).date()\n", - "\n", - " # Date-range exclusion\n", - " excluded = False\n", - " for rs, re in _excl_ranges:\n", - " if rs <= d1 <= re or rs <= d2 <= re:\n", - " excluded = True\n", - " break\n", - " if excluded:\n", - " continue\n", - "\n", - " # Month-day exclusion (recurring each year)\n", - " if EXCLUDE_MONTHDAY_RANGES:\n", - " md1 = (d1.month, d1.day)\n", - " md2 = (d2.month, d2.day)\n", - " for rng in EXCLUDE_MONTHDAY_RANGES:\n", - " if not (isinstance(rng, (list, tuple)) and len(rng) == 2):\n", - " continue\n", - " s_md, e_md = rng\n", - " try:\n", - " s_m, s_d = map(int, s_md.split('-'))\n", - " e_m, e_d = map(int, e_md.split('-'))\n", - " except Exception:\n", - " continue\n", - " start = (s_m, s_d)\n", - " end = (e_m, e_d)\n", - " # handle ranges that wrap year end (e.g., 12-15 to 02-15)\n", - " def _in_range(md):\n", - " if start <= end:\n", - " return start <= md <= end\n", - " return md >= start or md <= end\n", - " if _in_range(md1) or _in_range(md2):\n", - " excluded = True\n", - " break\n", - " if excluded:\n", - " continue\n", - "\n", - " filtered_pairs.append((r, s))\n", - "\n", - "pair_indices = filtered_pairs\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6852f778", - "metadata": {}, - "outputs": [], - "source": [ - "def take_looks(arr, row_looks, col_looks, func_type=\"nanmean\", edge_strategy=\"cutoff\"):\n", - " if row_looks == 1 and col_looks == 1:\n", - " return arr\n", - " if arr.ndim != 2:\n", - " raise ValueError(\"take_looks expects 2D array\")\n", - " rows, cols = arr.shape\n", - " if edge_strategy == \"cutoff\":\n", - " rows = (rows // row_looks) * row_looks\n", - " cols = (cols // col_looks) * col_looks\n", - " arr = arr[:rows, :cols]\n", - " elif edge_strategy == \"pad\":\n", - " pad_r = (-rows) % row_looks\n", - " pad_c = (-cols) % col_looks\n", - " if pad_r or pad_c:\n", - " arr = np.pad(arr, ((0, pad_r), (0, pad_c)), mode=\"constant\", constant_values=np.nan)\n", - " rows, cols = arr.shape\n", - " else:\n", - " raise ValueError(\"edge_strategy must be 'cutoff' or 'pad'\")\n", - "\n", - " new_rows = rows // row_looks\n", - " new_cols = cols // col_looks\n", - " func = getattr(np, func_type)\n", - " with warnings.catch_warnings():\n", - " warnings.simplefilter(\"ignore\", category=RuntimeWarning)\n", - " return func(arr.reshape(new_rows, row_looks, new_cols, col_looks), axis=(1, 3))\n", - "\n", - "\n", - "def _multilook(arr, looks_y=1, looks_x=1):\n", - " return take_looks(arr, looks_y, looks_x, func_type=\"nanmean\", edge_strategy=\"cutoff\")\n", - "\n", - "\n", - "def _box_mean(arr, win_y, win_x):\n", - " pad_y = win_y // 2\n", - " pad_x = win_x // 2\n", - " arr_p = np.pad(arr, ((pad_y, pad_y), (pad_x, pad_x)), mode='reflect')\n", - " windows = sliding_window_view(arr_p, (win_y, win_x))\n", - " return windows.mean(axis=(-2, -1))\n", - "\n", - "def get_kernel(size_y, size_x, num_conv):\n", - " if not isinstance(num_conv, int):\n", - " raise ValueError('num_conv must be an integer')\n", - " k0 = np.ones((size_y, size_x), dtype=np.float32)\n", - " k = k0\n", - " for i in range(num_conv):\n", - " if i > 3:\n", - " k = convolve(k, k0, mode='same')\n", - " else:\n", - " k = convolve(k, k0)\n", - " k = k / np.sum(k)\n", - " return k.astype(np.float32)\n", - "\n", - "\n", - "def _weighted_mean(arr, k):\n", - " # supports complex or real arrays\n", - " return convolve(arr, k, mode='same')\n", - "\n", - "\n", - "\n", - "def lee_filter(img, win_y=5, win_x=5):\n", - " mean = _box_mean(img, win_y, win_x)\n", - " mean_sq = _box_mean(img**2, win_y, win_x)\n", - " var = mean_sq - mean**2\n", - " noise_var = np.nanmedian(var)\n", - " w = var / (var + noise_var + 1e-8)\n", - " return mean + w * (img - mean)\n", - "\n", - "\n", - "def goldstein(\n", - " phase: NDArray[np.complex64] | NDArray[np.float64], alpha: float, psize: int = 32\n", - ") -> np.ndarray:\n", - " \"\"\"Apply the Goldstein adaptive filter to the given data.\"\"\"\n", - "\n", - " def apply_pspec(data: NDArray[np.complex64]) -> np.ndarray:\n", - " if alpha < 0:\n", - " raise ValueError(f\"alpha must be >= 0, got {alpha = }\")\n", - " weight = np.power(np.abs(data) ** 2, alpha / 2)\n", - " data = weight * data\n", - " return data\n", - "\n", - " def make_weight(nxp: int, nyp: int) -> np.ndarray:\n", - " wx = 1.0 - np.abs(np.arange(nxp // 2) - (nxp / 2.0 - 1.0)) / (nxp / 2.0 - 1.0)\n", - " wy = 1.0 - np.abs(np.arange(nyp // 2) - (nyp / 2.0 - 1.0)) / (nyp / 2.0 - 1.0)\n", - " quadrant = np.outer(wy, wx)\n", - " weight = np.block(\n", - " [\n", - " [quadrant, np.flip(quadrant, axis=1)],\n", - " [np.flip(quadrant, axis=0), np.flip(np.flip(quadrant, axis=0), axis=1)],\n", - " ]\n", - " )\n", - " return weight\n", - "\n", - " def patch_goldstein_filter(\n", - " data: NDArray[np.complex64], weight: NDArray[np.float64], psize: int\n", - " ) -> np.ndarray:\n", - " data = np.fft.fft2(data, s=(psize, psize))\n", - " data = apply_pspec(data)\n", - " data = np.fft.ifft2(data, s=(psize, psize))\n", - " return weight * data\n", - "\n", - " def apply_goldstein_filter(data: NDArray[np.complex64]) -> np.ndarray:\n", - " empty_mask = np.isnan(data) | (data == 0)\n", - " if np.all(empty_mask):\n", - " return data\n", - "\n", - " nrows, ncols = data.shape\n", - " step = psize // 2\n", - "\n", - " pad_top = step\n", - " pad_left = step\n", - " pad_bottom = step + (step - (nrows % step)) % step\n", - " pad_right = step + (step - (ncols % step)) % step\n", - " data_padded = np.pad(\n", - " data, ((pad_top, pad_bottom), (pad_left, pad_right)), mode=\"reflect\"\n", - " )\n", - "\n", - " out = np.zeros(data_padded.shape, dtype=np.complex64)\n", - " weight_sum = np.zeros(data_padded.shape, dtype=np.float64)\n", - " weight_matrix = make_weight(psize, psize)\n", - "\n", - " padded_rows, padded_cols = data_padded.shape\n", - " for i in range(0, padded_rows - psize + 1, step):\n", - " for j in range(0, padded_cols - psize + 1, step):\n", - " data_window = data_padded[i : i + psize, j : j + psize]\n", - " filtered_window = patch_goldstein_filter(\n", - " data_window, weight_matrix, psize\n", - " )\n", - " out[i : i + psize, j : j + psize] += filtered_window\n", - " weight_sum[i : i + psize, j : j + psize] += weight_matrix\n", - "\n", - " valid = weight_sum > 0\n", - " out[valid] /= weight_sum[valid]\n", - "\n", - " out = out[pad_top : pad_top + nrows, pad_left : pad_left + ncols]\n", - " out[empty_mask] = 0\n", - " return out\n", - "\n", - " if np.iscomplexobj(phase):\n", - " return apply_goldstein_filter(phase)\n", - " else:\n", - " return apply_goldstein_filter(np.exp(1j * phase))\n", - "\n", - "\n", - "def calc_ifg_coh_filtered(reference, secondary, goldstein_alpha=0.5, coh_win_y=5, coh_win_x=5, looks_y=1, looks_x=1, coh_method=\"standard\", ifg_apply_filter=True, coh_apply_lee=False, coh_kernel=\"boxcar\", coh_kernel_num_conv=5):\n", - " reference = _multilook(reference, looks_y, looks_x)\n", - " secondary = _multilook(secondary, looks_y, looks_x)\n", - " phase = reference * np.conjugate(secondary)\n", - " amp = np.sqrt((reference * np.conjugate(reference)) * (secondary * np.conjugate(secondary)))\n", - " nan_mask = np.isnan(phase)\n", - " ifg_cpx = np.exp(1j * np.nan_to_num(np.angle(phase/amp)))\n", - " if ifg_apply_filter:\n", - " ifg_cpx_f = goldstein(ifg_cpx, alpha=goldstein_alpha, psize=32)\n", - " else:\n", - " ifg_cpx_f = ifg_cpx\n", - " ifg = np.angle(ifg_cpx_f)\n", - " ifg[nan_mask] = np.nan\n", - "\n", - " ifg_cpx_used = ifg_cpx # coherence never uses filtered IFG\n", - "\n", - " if coh_kernel == 'weighted':\n", - " k = get_kernel(coh_win_y, coh_win_x, coh_kernel_num_conv)\n", - " elif coh_kernel == 'boxcar':\n", - " k = None\n", - " else:\n", - " raise ValueError(\"coh_kernel must be 'boxcar' or 'weighted'\")\n", - " if coh_method == \"phase_only\":\n", - " if coh_kernel == 'weighted':\n", - " coh = np.abs(_weighted_mean(ifg_cpx_used, k))\n", - " else:\n", - " coh = np.abs(_box_mean(ifg_cpx_used, coh_win_y, coh_win_x))\n", - " elif coh_method == \"standard\":\n", - " if coh_kernel == 'weighted':\n", - " num = np.abs(_weighted_mean(phase, k))\n", - " den = np.sqrt(_weighted_mean(np.abs(reference)**2, k) * _weighted_mean(np.abs(secondary)**2, k))\n", - " else:\n", - " num = np.abs(_box_mean(phase, coh_win_y, coh_win_x))\n", - " den = np.sqrt(_box_mean(np.abs(reference)**2, coh_win_y, coh_win_x) * _box_mean(np.abs(secondary)**2, coh_win_y, coh_win_x))\n", - " coh = np.where(den > 0, num / den, 0)\n", - " else:\n", - " raise ValueError(\"coh_method must be 'standard' or 'phase_only'\")\n", - " coh = np.clip(coh, 0, 1)\n", - " if coh_apply_lee:\n", - " coh = lee_filter(coh, win_y=coh_win_y, win_x=coh_win_x)\n", - " coh = np.clip(coh, 0, 1)\n", - " zero_mask = phase == 0\n", - " coh[nan_mask] = np.nan\n", - " coh[zero_mask] = 0\n", - " return ifg, coh, amp\n", - "\n", - "\n", - "def calc_ifg_coh(reference, secondary, goldstein_alpha=0.5, coh_win_y=5, coh_win_x=5, looks_y=1, looks_x=1, coh_method=\"standard\", ifg_apply_filter=True, coh_apply_lee=False, coh_kernel=\"boxcar\", coh_kernel_num_conv=5):\n", - " return calc_ifg_coh_filtered(reference, secondary, goldstein_alpha=goldstein_alpha, coh_win_y=coh_win_y, coh_win_x=coh_win_x, looks_y=looks_y, looks_x=looks_x, coh_method=coh_method, ifg_apply_filter=ifg_apply_filter, coh_apply_lee=coh_apply_lee, coh_kernel=coh_kernel, coh_kernel_num_conv=coh_kernel_num_conv)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2775556c", - "metadata": {}, - "outputs": [], - "source": [ - "## For each date-pair, calculate the ifg, coh. Save the results as GeoTiffs.\n", - "for ref_idx, sec_idx in pair_indices:\n", - " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", - " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", - " print(f\"Reference: {ref_date} Secondary: {sec_date}\")\n", - "\n", - " # Calculate ifg, coh, amp\n", - " if \"calc_ifg_coh_filtered\" not in globals():\n", - " raise RuntimeError(\"calc_ifg_coh_filtered is not defined. Run the filter definition cell first.\")\n", - " looks_y, looks_x = MULTILOOK\n", - "\n", - " # Save each interferogram as GeoTiff (no per-burst plotting)\n", - " transform = from_origin(xcoor_stack[ref_idx][0], ycoor_stack[ref_idx][0], dx, np.abs(dy))" - ] - }, - { - "cell_type": "markdown", - "id": "d57d1a71", - "metadata": {}, - "source": [ - "
\n", - "8. Merge the burst-wise interferograms and coherence and save as GeoTiff.\n", - "\n", - "
" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e98baf1f", - "metadata": {}, - "outputs": [], - "source": [ - "def custom_merge(old_data, new_data, old_nodata, new_nodata, **kwargs):\n", - " # Feather overlaps to reduce burst seams\n", - " if MERGE_BLEND_OVERLAP:\n", - " overlap = np.logical_and(~old_nodata, ~new_nodata)\n", - " if np.any(overlap):\n", - " # distance to nodata inside each valid mask\n", - " dist_old = distance_transform_edt(~old_nodata)\n", - " dist_new = distance_transform_edt(~new_nodata)\n", - " w_new = dist_new / (dist_new + dist_old + MERGE_BLEND_EPS)\n", - " w_new = np.clip(w_new, 0, 1)\n", - " blended = old_data * (1 - w_new) + new_data * w_new\n", - " old_data[overlap] = blended[overlap]\n", - " # fill empty pixels\n", - " mask = np.logical_and(old_nodata, ~new_nodata)\n", - " old_data[mask] = new_data[mask]\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "897e44dd", - "metadata": {}, - "outputs": [], - "source": [ - "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", - "# Merge burst-wise interferograms per date-pair\n", - "\n", - "# Group pair indices by date tag\n", - "pairs_by_tag = {}\n", - "for r, s in pair_indices:\n", - " ref_date = cslc_dates.iloc[r].values[0]\n", - " sec_date = cslc_dates.iloc[s].values[0]\n", - " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", - " pairs_by_tag.setdefault(tag, []).append((r, s))\n", - "\n", - "for tag, pairs in pairs_by_tag.items():\n", - " srcs = []\n", - " for r, s in pairs:\n", - " looks_y, looks_x = MULTILOOK\n", - " ifg, coh, amp = calc_ifg_coh(\n", - " cslc_stack[r], cslc_stack[s],\n", - " goldstein_alpha=GOLDSTEIN_ALPHA, coh_win_y=COH_WIN_Y, coh_win_x=COH_WIN_X,\n", - " looks_y=looks_y, looks_x=looks_x,\n", - " coh_method=COH_METHOD,\n", - " ifg_apply_filter=IFG_APPLY_FILTER,\n", - " coh_apply_lee=COH_APPLY_LEE,\n", - " coh_kernel=COH_KERNEL, coh_kernel_num_conv=COH_KERNEL_NUM_CONV,\n", - " )\n", - " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", - " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", - " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", - " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", - " mem = MemoryFile()\n", - " ds = mem.open(\n", - " driver='GTiff', height=ifg.shape[0], width=ifg.shape[1], count=1, dtype=ifg.dtype,\n", - " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", - " )\n", - " ds.write(ifg, 1)\n", - " srcs.append(ds)\n", - " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", - " dest, output_transform = _trim_nan_border(dest, output_transform)\n", - " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", - " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", - " if mask is not None:\n", - " dest[0] = _apply_water_mask(dest[0], mask)\n", - " out_meta = srcs[0].meta.copy()\n", - " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", - " out_path = f\"{savedir}/tifs/merged_ifg_{tag}.tif\"\n", - " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path)):\n", - " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", - " dest1.write(dest)\n", - " if SAVE_WGS84:\n", - " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_ifg_WGS84_{tag}.tif\"\n", - " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_wgs84)):\n", - " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", - " for ds in srcs:\n", - " ds.close()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7c41924c", - "metadata": {}, - "outputs": [], - "source": [ - "# Merge burst-wise coherence per date-pair\n", - "\n", - "# Group pair indices by date tag\n", - "pairs_by_tag = {}\n", - "for r, s in pair_indices:\n", - " ref_date = cslc_dates.iloc[r].values[0]\n", - " sec_date = cslc_dates.iloc[s].values[0]\n", - " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", - " pairs_by_tag.setdefault(tag, []).append((r, s))\n", - "\n", - "for tag, pairs in pairs_by_tag.items():\n", - " srcs = []\n", - " for r, s in pairs:\n", - " looks_y, looks_x = MULTILOOK\n", - " ifg, coh, amp = calc_ifg_coh(\n", - " cslc_stack[r], cslc_stack[s],\n", - " goldstein_alpha=GOLDSTEIN_ALPHA, coh_win_y=COH_WIN_Y, coh_win_x=COH_WIN_X,\n", - " looks_y=looks_y, looks_x=looks_x,\n", - " coh_method=COH_METHOD,\n", - " ifg_apply_filter=IFG_APPLY_FILTER,\n", - " coh_apply_lee=COH_APPLY_LEE,\n", - " coh_kernel=COH_KERNEL, coh_kernel_num_conv=COH_KERNEL_NUM_CONV,\n", - " )\n", - " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", - " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", - " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", - " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", - " mem = MemoryFile()\n", - " ds = mem.open(\n", - " driver='GTiff', height=coh.shape[0], width=coh.shape[1], count=1, dtype=coh.dtype,\n", - " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", - " )\n", - " ds.write(coh, 1)\n", - " srcs.append(ds)\n", - " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", - " dest, output_transform = _trim_nan_border(dest, output_transform)\n", - " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", - " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", - " if mask is not None:\n", - " dest[0] = _apply_water_mask(dest[0], mask)\n", - " out_meta = srcs[0].meta.copy()\n", - " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", - " out_path = f\"{savedir}/tifs/merged_coh_{tag}.tif\"\n", - " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path)):\n", - " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", - " dest1.write(dest)\n", - " if SAVE_WGS84:\n", - " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_coh_WGS84_{tag}.tif\"\n", - " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_wgs84)):\n", - " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", - " for ds in srcs:\n", - " ds.close()\n" - ] - }, - { - "cell_type": "markdown", - "id": "858ea831", - "metadata": {}, - "source": [ - "
\n", - "9. Read the merged GeoTiff and Visualize using `matplotlib`\n", - "\n", - "
" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9cb2a341", - "metadata": {}, - "outputs": [], - "source": [ - "# Read merged IFG/COH files and plot paired grids\n", - "\n", - "\n", - "# Output dir for per-pair PNGs\n", - "pair_png_dir = f\"{savedir}/pairs_png\"\n", - "os.makedirs(pair_png_dir, exist_ok=True)\n", - "\n", - "ifg_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_ifg_*.tif\"))\n", - "coh_norm = mcolors.PowerNorm(gamma=COH_NORM_GAMMA, vmin=0, vmax=1) if COH_USE_GAMMA_NORM else None\n", - "coh_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\"))\n", - "\n", - "ifg_map = {p.split('merged_ifg_')[-1].replace('.tif',''): p for p in ifg_paths}\n", - "coh_map = {p.split('merged_coh_')[-1].replace('.tif',''): p for p in coh_paths}\n", - "\n", - "\n", - "\n", - "def _prep_da(path):\n", - " da = rioxarray.open_rasterio(path)[0]\n", - " data = da.values\n", - " mask = np.isfinite(data) & (data != 0)\n", - " if mask.any():\n", - " rows = np.where(mask.any(axis=1))[0]\n", - " cols = np.where(mask.any(axis=0))[0]\n", - " r0, r1 = rows[0], rows[-1] + 1\n", - " c0, c1 = cols[0], cols[-1] + 1\n", - " # Trim NaN borders so edges don't show padding\n", - " da = da.isel(y=slice(r0, r1), x=slice(c0, c1))\n", - " return da\n", - "\n", - "pair_tags = sorted(set(ifg_map).intersection(coh_map))\n", - "# Filter pairs by current date range\n", - "date_start_day = dateStart.date()\n", - "date_end_day = dateEnd.date()\n", - "pair_tags = [t for t in pair_tags if (date_start_day <= pd.to_datetime(t.split('-')[0], format='%Y%m%d').date() <= date_end_day and date_start_day <= pd.to_datetime(t.split('-')[1], format='%Y%m%d').date() <= date_end_day)]\n", - "\n", - "if not pair_tags:\n", - " print('No matching IFG/COH pairs found')\n", - "else:\n", - " # Save ALL pairs as PNGs\n", - " for tag in pair_tags:\n", - " fig, axes = plt.subplots(1, 2, figsize=(10, 4), constrained_layout=True)\n", - " ax_ifg, ax_coh = axes\n", - "\n", - " # IFG\n", - " merged_ifg = _prep_da(ifg_map[tag])\n", - " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", - " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", - " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", - " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", - " ax_ifg.set_xticks([])\n", - " ax_ifg.set_yticks([])\n", - " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", - "\n", - " # COH\n", - " merged_coh = _prep_da(coh_map[tag])\n", - " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", - " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, norm=coh_norm, vmin=None if COH_USE_GAMMA_NORM else 0, vmax=None if COH_USE_GAMMA_NORM else 1.0)\n", - " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", - " ax_coh.set_xticks([])\n", - " ax_coh.set_yticks([])\n", - " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')\n", - "\n", - " out_png = os.path.join(pair_png_dir, f\"pair_{tag}.png\")\n", - " fig.savefig(out_png, dpi=150)\n", - " plt.close(fig)\n", - "\n", - " # Display only last 5 pairs in notebook\n", - " display_tags = pair_tags[-5:]\n", - " n = len(display_tags)\n", - " ncols = 2\n", - " nrows = math.ceil(n / 1) # one pair per row\n", - " fig, axes = plt.subplots(nrows, ncols, figsize=(6*ncols, 3*nrows), constrained_layout=True)\n", - " if nrows == 1:\n", - " axes = [axes]\n", - "\n", - " for i, tag in enumerate(display_tags):\n", - " ax_ifg, ax_coh = axes[i]\n", - "\n", - " # IFG\n", - " merged_ifg = _prep_da(ifg_map[tag])\n", - " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", - " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", - " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", - " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", - " ax_ifg.set_xticks([])\n", - " ax_ifg.set_yticks([])\n", - " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", - "\n", - " # COH\n", - " merged_coh = _prep_da(coh_map[tag])\n", - " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", - " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, norm=coh_norm, vmin=None if COH_USE_GAMMA_NORM else 0, vmax=None if COH_USE_GAMMA_NORM else 1.0)\n", - " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", - " ax_coh.set_xticks([])\n", - " ax_coh.set_yticks([])\n", - " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')" - ] - }, - { - "cell_type": "markdown", - "id": "ea1e4f24", - "metadata": {}, - "source": [ - "
\n", - "9.5 Merge and plot backscatter (dB) mosaics (per date)\n", - "\n", - "
" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e7f478f4", - "metadata": {}, - "outputs": [], - "source": [ - "def _load_water_mask_match(mask_path, shape, transform, crs):\n", - " with rasterio.open(mask_path) as src:\n", - " src_mask = src.read(1)\n", - " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", - " return src_mask\n", - " dst = np.zeros(shape, dtype=src_mask.dtype)\n", - " reproject(\n", - " source=src_mask,\n", - " destination=dst,\n", - " src_transform=src.transform,\n", - " src_crs=src.crs,\n", - " dst_transform=transform,\n", - " dst_crs=crs,\n", - " resampling=Resampling.nearest,\n", - " )\n", - " return dst\n", - "\n", - "def _apply_water_mask(arr, mask):\n", - " # mask: 1 = keep land, 0 = water\n", - " return np.where(mask == 0, np.nan, arr)\n", - "\n", - "\n", - "def _trim_nan_border(arr, transform):\n", - " data = arr[0] if arr.ndim == 3 else arr\n", - " mask = np.isfinite(data) & (data != 0)\n", - " if not mask.any():\n", - " return arr, transform\n", - " rows = np.where(mask.any(axis=1))[0]\n", - " cols = np.where(mask.any(axis=0))[0]\n", - " r0, r1 = rows[0], rows[-1] + 1\n", - " c0, c1 = cols[0], cols[-1] + 1\n", - " data = data[r0:r1, c0:c1]\n", - " if arr.ndim == 3:\n", - " arr = data[None, ...]\n", - " else:\n", - " arr = data\n", - " new_transform = transform * Affine.translation(c0, r0)\n", - " return arr, new_transform\n", - "\n", - "# Build per-date backscatter (dB) mosaics directly from subset H5 (no per-burst GeoTIFFs)\n", - "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", - "\n", - "date_tags = sorted(cslc_df.startTime.astype(str).str.replace('-', '').unique())\n", - "\n", - "def _save_mosaic_utm(out_path, mosaic, transform, epsg):\n", - " with rasterio.open(\n", - " out_path, \"w\", driver=\"GTiff\", height=mosaic.shape[0], width=mosaic.shape[1],\n", - " count=1, dtype=mosaic.dtype, crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", - " ) as dst_ds:\n", - " dst_ds.write(mosaic, 1)\n", - "\n", - "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", - " dst_crs = \"EPSG:4326\"\n", - " src_crs = CRS.from_epsg(epsg)\n", - " height, width = mosaic.shape\n", - " dst_transform, dst_width, dst_height = calculate_default_transform(\n", - " src_crs, dst_crs, width, height, *rasterio.transform.array_bounds(height, width, transform)\n", - " )\n", - " dst = np.empty((dst_height, dst_width), dtype=mosaic.dtype)\n", - " reproject(\n", - " source=mosaic,\n", - " destination=dst,\n", - " src_transform=transform,\n", - " src_crs=src_crs,\n", - " dst_transform=dst_transform,\n", - " dst_crs=dst_crs,\n", - " resampling=Resampling.bilinear,\n", - " )\n", - " with rasterio.open(\n", - " out_path, \"w\", driver=\"GTiff\", height=dst_height, width=dst_width, count=1,\n", - " dtype=dst.dtype, crs=dst_crs, transform=dst_transform, nodata=np.nan\n", - " ) as dst_ds:\n", - " dst_ds.write(dst, 1)\n", - "\n", - "# Mosaicking helper (in memory)\n", - "def _mosaic_arrays(arrays, transforms, epsg):\n", - " # Convert arrays to in-memory rasterio datasets via MemoryFile\n", - " srcs = []\n", - " for arr, transform in zip(arrays, transforms):\n", - " mem = MemoryFile()\n", - " ds = mem.open(\n", - " driver='GTiff', height=arr.shape[0], width=arr.shape[1], count=1, dtype=arr.dtype,\n", - " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", - " )\n", - " ds.write(arr, 1)\n", - " srcs.append(ds)\n", - " dest, out_transform = merge.merge(srcs, method=custom_merge)\n", - " for ds in srcs:\n", - " ds.close()\n", - " return dest[0], out_transform\n", - "\n", - "\n", - "def _reproject_calibration_to_data_grid(sigma, cal_x, cal_y, cal_dx, cal_dy, xcoor, ycoor, dx, dy, epsg):\n", - " cal_dy_signed = (cal_y[1] - cal_y[0]) if len(cal_y) > 1 else -cal_dy\n", - " data_dy_signed = (ycoor[1] - ycoor[0]) if len(ycoor) > 1 else -dy\n", - " if sigma.shape == (len(ycoor), len(xcoor)) and cal_dx == dx and abs(cal_dy_signed) == abs(data_dy_signed):\n", - " return sigma\n", - " src_transform = from_origin(cal_x[0], cal_y[0], cal_dx, abs(cal_dy_signed))\n", - " dst_transform = from_origin(xcoor[0], ycoor[0], dx, abs(data_dy_signed))\n", - " dst = np.empty((len(ycoor), len(xcoor)), dtype=sigma.dtype)\n", - " reproject(\n", - " source=sigma,\n", - " destination=dst,\n", - " src_transform=src_transform,\n", - " src_crs=CRS.from_epsg(epsg),\n", - " dst_transform=dst_transform,\n", - " dst_crs=CRS.from_epsg(epsg),\n", - " resampling=Resampling.bilinear,\n", - " )\n", - " return dst\n", - "\n", - "looks_y, looks_x = MULTILOOK\n", - "# Backscatter calibration mode tag for filenames\n", - "bsc_mode_tag = CALIBRATION_MODE\n", - "beta_naught = None\n", - "for date_tag in date_tags:\n", - " # collect subset H5 files for this date\n", - " rows = cslc_df[cslc_df.startTime.astype(str).str.replace('-', '') == date_tag]\n", - " arrays = []\n", - " transforms = []\n", - " epsg = None\n", - " for fileID in rows.fileID:\n", - " subset_path = f\"{savedir}/subset_cslc/{fileID}.h5\"\n", - " with h5py.File(subset_path, 'r') as h5:\n", - " cslc = h5['/data/VV'][:]\n", - " xcoor = h5['/data/x_coordinates'][:]\n", - " ycoor = h5['/data/y_coordinates'][:]\n", - " dx = int(h5['/data/x_spacing'][()])\n", - " dy = int(h5['/data/y_spacing'][()])\n", - " epsg = int(h5['/data/projection'][()])\n", - " sigma = h5['/metadata/calibration_information/sigma_naught'][:]\n", - " cal_x = h5['/metadata/calibration_information/x_coordinates'][:]\n", - " cal_y = h5['/metadata/calibration_information/y_coordinates'][:]\n", - " cal_dx = int(h5['/metadata/calibration_information/x_spacing'][()])\n", - " cal_dy = int(h5['/metadata/calibration_information/y_spacing'][()])\n", - " beta_naught = h5['/metadata/calibration_information/beta_naught'][()]\n", - " power_ml = _multilook(np.abs(cslc)**2, looks_y, looks_x)\n", - " if CALIBRATION_MODE == 'sigma0':\n", - " sigma_on_data = _reproject_calibration_to_data_grid(\n", - " sigma, cal_x, cal_y, cal_dx, cal_dy, xcoor, ycoor, dx, dy, epsg\n", - " )\n", - " sigma_ml = _multilook(sigma_on_data, looks_y, looks_x)\n", - " sigma_ml = np.where(sigma_ml <= 0, np.nan, sigma_ml)\n", - " sigma_factor = sigma_ml**2 if CALIBRATION_FACTOR_IS_AMPLITUDE else sigma_ml\n", - " bsc = 10*np.log10(power_ml / sigma_factor)\n", - " elif CALIBRATION_MODE == 'beta0':\n", - " beta = beta_naught\n", - " if beta is None or beta <= 0:\n", - " bsc = np.full(power_ml.shape, np.nan, dtype=power_ml.dtype)\n", - " else:\n", - " beta_factor = beta**2 if CALIBRATION_FACTOR_IS_AMPLITUDE else beta\n", - " bsc = 10*np.log10(power_ml / beta_factor)\n", - " else:\n", - " raise ValueError(\"CALIBRATION_MODE must be 'sigma0' or 'beta0'\")\n", - " dy_signed = (ycoor[1] - ycoor[0]) if len(ycoor) > 1 else -dy\n", - " x0 = xcoor[0] + (looks_x - 1) * dx / 2\n", - " y0 = ycoor[0] + (looks_y - 1) * dy_signed / 2\n", - " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", - " arrays.append(bsc)\n", - " transforms.append(transform)\n", - "\n", - " if not arrays:\n", - " continue\n", - " mosaic, out_transform = _mosaic_arrays(arrays, transforms, epsg)\n", - " out_path_utm = f\"{savedir}/tifs/merged_bsc_{bsc_mode_tag}_{date_tag}.tif\"\n", - " mosaic, out_transform = _trim_nan_border(mosaic, out_transform)\n", - " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", - " mask = _load_water_mask_match(WATER_MASK_PATH, mosaic.shape, out_transform, CRS.from_epsg(epsg))\n", - " mosaic = _apply_water_mask(mosaic, mask)\n", - " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_utm)):\n", - " _save_mosaic_utm(out_path_utm, mosaic, out_transform, epsg)\n", - " if SAVE_WGS84:\n", - " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_bsc_{bsc_mode_tag}_WGS84_{date_tag}.tif\"\n", - " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_wgs84)):\n", - " _save_mosaic_utm_to_wgs84(out_path_wgs84, mosaic, out_transform, epsg)\n", - "\n", - "# Plot merged backscatter (dB) mosaics in a grid (native CRS from saved GeoTIFFs)\n", - "\n", - "# Output dir for backscatter PNGs\n", - "bsc_png_dir = f\"{savedir}/bsc_png\"\n", - "os.makedirs(bsc_png_dir, exist_ok=True)\n", - "\n", - "paths = sorted(glob.glob(f\"{savedir}/tifs/merged_bsc_{bsc_mode_tag}_*.tif\"))\n", - "paths = [p for p in paths if 'WGS84' not in p]\n", - "all_vals = []\n", - "for p in paths:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " all_vals.append(da.values.ravel())\n", - "if all_vals:\n", - " all_vals = np.concatenate(all_vals)\n", - " gmin = np.nanpercentile(all_vals, 2)\n", - " gmax = np.nanpercentile(all_vals, 90)\n", - "else:\n", - " gmin, gmax = None, None\n", - "n = len(paths)\n", - "if n == 0:\n", - " print('No merged backscatter files found')\n", - "else:\n", - " # Save ALL backscatter PNGs\n", - " for path in paths:\n", - " src = rioxarray.open_rasterio(path)\n", - " bsc = src[0]\n", - " minlon, minlat, maxlon, maxlat = bsc.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " fig, ax = plt.subplots(figsize=(5,4))\n", - " im = ax.imshow(bsc.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", - " tag = path.split(f'merged_bsc_{bsc_mode_tag}_')[-1].replace('.tif','')\n", - " ax.set_title(f\"{tag}\", fontsize=10)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " fig.colorbar(im, ax=ax, orientation='vertical', fraction=0.046, pad=0.02, label=f\"{bsc_mode_tag} (dB)\")\n", - " out_png = os.path.join(bsc_png_dir, f\"bsc_{tag}.png\")\n", - " fig.savefig(out_png, dpi=150)\n", - " plt.close(fig)\n", - "\n", - " # Show only last 5 in notebook\n", - " display_paths = paths[-5:]\n", - " n = len(display_paths)\n", - " ncols = 3\n", - " nrows = math.ceil(n / ncols)\n", - " fig, axes = plt.subplots(nrows, ncols, figsize=(4*ncols, 3*nrows), constrained_layout=True)\n", - " axes = axes.ravel()\n", - " for ax, path in zip(axes, display_paths):\n", - " src = rioxarray.open_rasterio(path)\n", - " bsc = src[0]\n", - " minlon, minlat, maxlon, maxlat = bsc.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " im = ax.imshow(bsc.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", - " tag = path.split(f'merged_bsc_{bsc_mode_tag}_')[-1].replace('.tif','')\n", - " ax.set_title(f\"{tag}\", fontsize=10)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " for ax in axes[n:]:\n", - " ax.axis('off')\n", - " fig.colorbar(im, ax=axes.tolist(), orientation='vertical', fraction=0.02, pad=0.02, label=f\"{bsc_mode_tag} (dB)\")" - ] - }, - { - "cell_type": "markdown", - "id": "df66a59f", - "metadata": {}, - "source": [ - "
\n", - "10. Monthly mean coherence calendar (per year)\n", - "\n", - "This calendar bins each pair into a month using the midpoint date between the reference and secondary scenes.\n", - "
" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "c31a947a", - "metadata": {}, - "outputs": [], - "source": [ - "# # CALENDAR_CMAP = ROCKET_CMAP # previous\n", - "CALENDAR_CMAP = cmc.bilbao\n", - "# CALENDAR_CMAP = cmc.bilbao\n", - "\n", - "# Build an index of merged coherence files by midpoint year-month\n", - "records = []\n", - "for path in sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\")):\n", - " tag = path.split('merged_coh_')[-1].replace('.tif','')\n", - " try:\n", - " ref_str, sec_str = tag.split('-')\n", - " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", - " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", - " mid_date = ref_date + (sec_date - ref_date) / 2\n", - " except Exception:\n", - " continue\n", - " records.append({\"path\": path, \"mid_date\": mid_date})\n", - "\n", - "df_paths = pd.DataFrame(records)\n", - "if df_paths.empty:\n", - " print('No merged coherence files found for calendar')\n", - " raise SystemExit\n", - "\n", - "# Apply current date range using midpoint date\n", - "date_start_day = dateStart.date()\n", - "date_end_day = dateEnd.date()\n", - "df_paths = df_paths[(df_paths['mid_date'].dt.date >= date_start_day) & (df_paths['mid_date'].dt.date <= date_end_day)]\n", - "\n", - "# Calendar year labeling\n", - "if USE_WATER_YEAR:\n", - " # Water year starts Oct (10) and ends Sep (9)\n", - " df_paths['year'] = df_paths['mid_date'].dt.year + (df_paths['mid_date'].dt.month >= 10).astype(int)\n", - " month_order = [10,11,12,1,2,3,4,5,6,7,8,9]\n", - " month_labels = ['Oct','Nov','Dec','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep']\n", - "else:\n", - " df_paths['year'] = df_paths['mid_date'].dt.year\n", - " month_order = list(range(1,13))\n", - " month_labels = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']\n", - "\n", - "years = sorted(df_paths['year'].unique())\n", - "\n", - "# Contrast stretch for low coherence (red = low)\n", - "norm = mcolors.PowerNorm(gamma=0.3, vmin=0, vmax=1)\n", - "\n", - "# One row per year, 12 columns\n", - "fig, axes = plt.subplots(len(years), 12, figsize=(24, 2.5*len(years)), constrained_layout=True)\n", - "if len(years) == 1:\n", - " axes = np.array([axes])\n", - "\n", - "for row_idx, y in enumerate(years):\n", - " # pick a template for consistent grid within the year (first available file)\n", - " year_paths = df_paths[df_paths['year'] == y]['path'].tolist()\n", - " if not year_paths:\n", - " continue\n", - " template = rioxarray.open_rasterio(year_paths[0])\n", - "\n", - " for col_idx, m in enumerate(month_order):\n", - " ax = axes[row_idx, col_idx]\n", - " month_paths = df_paths[(df_paths['year'] == y) & (df_paths['mid_date'].dt.month == m)]['path'].tolist()\n", - " if USE_WATER_YEAR:\n", - " year_for_month = y - 1 if m in (10, 11, 12) else y\n", - " else:\n", - " year_for_month = y\n", - " title = f\"{month_labels[col_idx]} {year_for_month}\"\n", - " if not month_paths:\n", - " ax.set_title(title, fontsize=9)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " # keep a visible box for empty months\n", - " for spine in ax.spines.values():\n", - " spine.set_visible(True)\n", - " spine.set_linewidth(0.8)\n", - " spine.set_color('0.5')\n", - " continue\n", - " stacks = []\n", - " for p in month_paths:\n", - " da = rioxarray.open_rasterio(p)\n", - " da = da.rio.reproject_match(template)\n", - " stacks.append(da)\n", - " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", - " minlon, minlat, maxlon, maxlat = da_month.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " im = ax.imshow(da_month.values.squeeze(), cmap=CALENDAR_CMAP, norm=norm, origin='upper', extent=bbox, interpolation='none')\n", - " ax.set_title(title, fontsize=9)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " for spine in ax.spines.values():\n", - " spine.set_visible(True)\n", - " spine.set_linewidth(0.8)\n", - " spine.set_color('0.5')\n", - " # left-side year label\n", - " if USE_WATER_YEAR:\n", - " label = f\"WY {y}\"\n", - " else:\n", - " label = str(y)\n", - " axes[row_idx, 0].set_ylabel(label, rotation=90, labelpad=6, fontsize=9)\n", - " axes[row_idx, 0].yaxis.set_label_coords(-0.06, 0.5)\n", - "\n", - "fig.colorbar(im, ax=axes, orientation='vertical', fraction=0.02, pad=0.02, label='Mean coherence')" - ] - }, - { - "cell_type": "markdown", - "id": "169216f0", - "metadata": {}, - "source": [ - "
\n", - "11. Create GIF animations (Backscatter (dB) + Coherence)\n", - "\n", - "
" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "89109813", - "metadata": {}, - "outputs": [], - "source": [ - "# GIF helper functions\n", - "\n", - "def _global_bounds(paths):\n", - " bounds = []\n", - " for p in paths:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " minx, miny, maxx, maxy = da.rio.bounds()\n", - " bounds.append((minx, miny, maxx, maxy))\n", - " minx = min(b[0] for b in bounds)\n", - " miny = min(b[1] for b in bounds)\n", - " maxx = max(b[2] for b in bounds)\n", - " maxy = max(b[3] for b in bounds)\n", - " return [minx, maxx, miny, maxy]\n", - "\n", - "def _extract_date_from_tag(tag):\n", - " m = re.search(r\"(\\d{8})\", tag)\n", - " if not m:\n", - " return None\n", - " try:\n", - " return pd.to_datetime(m.group(1), format=\"%Y%m%d\")\n", - " except Exception:\n", - " return None\n", - "\n", - "def _format_title(tag, date=None):\n", - " if date is not None:\n", - " return date.strftime('%Y-%m-%d')\n", - " m = re.findall(r\"(\\d{8})\", tag)\n", - " if len(m) >= 2:\n", - " try:\n", - " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\")\n", - " d1 = pd.to_datetime(m[1], format=\"%Y%m%d\")\n", - " return f\"{d0.strftime('%Y-%m-%d')} to {d1.strftime('%Y-%m-%d')}\"\n", - " except Exception:\n", - " pass\n", - " if len(m) == 1:\n", - " try:\n", - " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\")\n", - " return d0.strftime('%Y-%m-%d')\n", - " except Exception:\n", - " pass\n", - " return tag\n", - "\n", - "def _filter_paths_by_date(paths, date_start, date_end):\n", - " if not paths:\n", - " return paths\n", - " ds = pd.to_datetime(date_start).date()\n", - " de = pd.to_datetime(date_end).date()\n", - " kept = []\n", - " for p in paths:\n", - " tag = os.path.basename(p).replace('.tif','')\n", - " m = re.findall(r\"(\\d{8})\", tag)\n", - " if len(m) >= 2:\n", - " try:\n", - " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\").date()\n", - " d1 = pd.to_datetime(m[1], format=\"%Y%m%d\").date()\n", - " mid = d0 + (d1 - d0) / 2\n", - " mid_date = mid if hasattr(mid, 'year') else mid.date()\n", - " if ds <= mid_date <= de:\n", - " kept.append(p)\n", - " except Exception:\n", - " kept.append(p)\n", - " elif len(m) == 1:\n", - " try:\n", - " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\").date()\n", - " if ds <= d0 <= de:\n", - " kept.append(p)\n", - " except Exception:\n", - " kept.append(p)\n", - " else:\n", - " kept.append(p)\n", - " return kept\n", - "\n", - "def _render_frames(tif_paths, out_dir, cmap, vmin=None, vmax=None, title_prefix=None, extent=None, cbar_label=None, cbar_ticks=None, template=None, dates=None, title_dates=None, show_time_bar=False, norm=None):\n", - " os.makedirs(out_dir, exist_ok=True)\n", - " frames = []\n", - " if template is None and tif_paths:\n", - " template = rioxarray.open_rasterio(tif_paths[0])[0]\n", - " if extent is None and template is not None:\n", - " minlon, minlat, maxlon, maxlat = template.rio.bounds()\n", - " extent = [minlon, maxlon, minlat, maxlat]\n", - "\n", - " date_min = date_max = None\n", - " if show_time_bar and dates:\n", - " valid_dates = [d for d in dates if d is not None]\n", - " if valid_dates:\n", - " date_min = min(valid_dates)\n", - " date_max = max(valid_dates)\n", - "\n", - " for i, p in enumerate(tif_paths):\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " if template is not None:\n", - " da = da.rio.reproject_match(template)\n", - " fig, ax = plt.subplots(figsize=(6,4))\n", - " im = ax.imshow(da.values, cmap=cmap, origin='upper', extent=extent, norm=norm, vmin=None if norm is not None else vmin, vmax=None if norm is not None else vmax)\n", - " tag = os.path.basename(p).replace('.tif','')\n", - " if title_prefix is None:\n", - " title_date = title_dates[i] if title_dates and i < len(title_dates) else None\n", - " title = _format_title(tag, title_date)\n", - " else:\n", - " title = f\"{title_prefix}{tag}\"\n", - " ax.set_title(title, fontsize=9)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", - " if cbar_label:\n", - " cb.set_label(cbar_label)\n", - " if cbar_ticks is not None:\n", - " cb.set_ticks(cbar_ticks)\n", - "\n", - " if show_time_bar and date_min is not None and date_max is not None:\n", - " ax_time = fig.add_axes([0.12, 0.04, 0.76, 0.05])\n", - " ax_time.plot([0, 1], [0.5, 0.5], color=\"0.6\", linewidth=3, solid_capstyle='round')\n", - " cur_date = dates[i] if dates and i < len(dates) else None\n", - " if cur_date is not None and date_max != date_min:\n", - " pos = (cur_date - date_min) / (date_max - date_min)\n", - " pos = max(0, min(1, float(pos)))\n", - " else:\n", - " pos = 0.0 if date_max == date_min else 0.5\n", - " ax_time.plot([pos, pos], [0.2, 0.8], color=\"crimson\", linewidth=2, zorder=5)\n", - " ax_time.scatter([pos], [0.5], s=90, color=\"crimson\", zorder=6)\n", - "\n", - " if date_max == date_min:\n", - " tick_dates = [date_min]\n", - " else:\n", - " tick_dates = pd.date_range(date_min, date_max, periods=5)\n", - " for d in tick_dates:\n", - " t = (d - date_min) / (date_max - date_min)\n", - " t = max(0, min(1, float(t)))\n", - " ax_time.plot([t, t], [0.35, 0.65], color=\"0.4\", linewidth=1)\n", - " ax_time.text(t, -0.05, d.strftime('%Y-%m-%d'), ha='center', va='top', fontsize=6)\n", - "\n", - " ax_time.set_xlim(0, 1)\n", - " ax_time.set_ylim(0, 1)\n", - " ax_time.axis('off')\n", - "\n", - " frame_path = os.path.join(out_dir, f\"{tag}.png\")\n", - " fig.savefig(frame_path, dpi=150)\n", - " plt.close(fig)\n", - " frames.append(frame_path)\n", - " return frames\n", - "\n", - "def _pad_frames(frame_paths):\n", - " imgs = [imageio.imread(f) for f in frame_paths]\n", - " max_h = max(im.shape[0] for im in imgs)\n", - " max_w = max(im.shape[1] for im in imgs)\n", - " padded = []\n", - " for im in imgs:\n", - " pad_h = max_h - im.shape[0]\n", - " pad_w = max_w - im.shape[1]\n", - " top = pad_h // 2\n", - " bottom = pad_h - top\n", - " left = pad_w // 2\n", - " right = pad_w - left\n", - " padded.append(np.pad(im, ((top, bottom), (left, right), (0, 0)), mode='edge'))\n", - " return padded\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d8397a42", - "metadata": {}, - "outputs": [], - "source": [ - "# Prepare coherence extent/template for GIFs\n", - "coh_template = None\n", - "coh_extent = None\n", - "if coh_paths:\n", - " coh_template = rioxarray.open_rasterio(coh_paths[0])[0]\n", - " minlon, minlat, maxlon, maxlat = coh_template.rio.bounds()\n", - " coh_extent = [minlon, maxlon, minlat, maxlat]\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15d63323", - "metadata": {}, - "outputs": [], - "source": [ - "# GIF prep (paths, dates, extent)\n", - "gif_dir = f\"{savedir}/gifs\"\n", - "os.makedirs(gif_dir, exist_ok=True)\n", - "\n", - "coh_paths = _filter_paths_by_date(coh_paths, dateStart, dateEnd)\n", - "coh_dates = []\n", - "for p in coh_paths:\n", - " tag = Path(p).name.replace('merged_coh_', '').replace('.tif', '')\n", - " try:\n", - " ref_str, sec_str = tag.split('-')\n", - " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", - " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", - " coh_dates.append(ref_date + (sec_date - ref_date) / 2)\n", - " except Exception:\n", - " coh_dates.append(None)\n", - "\n", - "coh_template = None\n", - "coh_extent = None\n", - "if coh_paths:\n", - " coh_template = rioxarray.open_rasterio(coh_paths[0])[0]\n", - " minlon, minlat, maxlon, maxlat = coh_template.rio.bounds()\n", - " coh_extent = [minlon, maxlon, minlat, maxlat]\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "60b7c196", - "metadata": {}, - "outputs": [], - "source": [ - "# Coherence GIF\n", - "# Adjust GIF speed based on number of frames\n", - "\n", - "def _gif_duration(n_frames, min_sec=0.3, max_sec=1.2, target_total_sec=12.0):\n", - " if n_frames <= 0:\n", - " return max_sec\n", - " return max(min_sec, min(max_sec, target_total_sec / n_frames))\n", - "\n", - "coh_norm = mcolors.PowerNorm(gamma=COH_NORM_GAMMA, vmin=0, vmax=1) if COH_USE_GAMMA_NORM else None\n", - "if coh_paths:\n", - " coh_frames = _render_frames(\n", - " coh_paths, f\"{gif_dir}/coh_frames\", cmap=cmc.bilbao, vmin=0, vmax=1,\n", - " title_prefix=None, extent=coh_extent, cbar_label='Coherence', cbar_ticks=[0,0.5,1],\n", - " template=coh_template, dates=coh_dates, title_dates=None, show_time_bar=True, norm=coh_norm\n", - " )\n", - " coh_gif = f\"{gif_dir}/coherence.gif\"\n", - " coh_imgs = _pad_frames(coh_frames)\n", - " imageio.mimsave(coh_gif, coh_imgs, duration=_gif_duration(len(coh_imgs)))\n", - " print(f\"Wrote {coh_gif}\")\n", - "else:\n", - " print('No merged coherence files found for GIF')\n", - "\n", - "\n", - "# Monthly mean coherence GIF\n", - "if coh_paths:\n", - " monthly_dir = f\"{gif_dir}/coh_monthly_frames\"\n", - " os.makedirs(monthly_dir, exist_ok=True)\n", - " # Build month index using midpoint dates\n", - " monthly = {}\n", - " for p, d in zip(coh_paths, coh_dates):\n", - " if d is None:\n", - " continue\n", - " key = d.strftime('%Y-%m')\n", - " monthly.setdefault(key, []).append(p)\n", - "\n", - " monthly_frames = []\n", - " for key in sorted(monthly.keys()):\n", - " stacks = []\n", - " for p in monthly[key]:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " if coh_template is not None:\n", - " da = da.rio.reproject_match(coh_template)\n", - " stacks.append(da)\n", - " if not stacks:\n", - " continue\n", - " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", - " minlon, minlat, maxlon, maxlat = da_month.rio.bounds()\n", - " bbox = [minlon, maxlon, minlat, maxlat]\n", - " fig, ax = plt.subplots(figsize=(6,4))\n", - " im = ax.imshow(da_month.values, cmap=cmc.bilbao, origin='upper', extent=bbox, interpolation='none', norm=coh_norm, vmin=None if COH_USE_GAMMA_NORM else 0, vmax=None if COH_USE_GAMMA_NORM else 1)\n", - " ax.set_title(key, fontsize=9)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", - " cb.set_label('Mean coherence')\n", - " frame_path = os.path.join(monthly_dir, f\"{key}.png\")\n", - " fig.savefig(frame_path, dpi=150)\n", - " plt.close(fig)\n", - " monthly_frames.append(frame_path)\n", - "\n", - " if monthly_frames:\n", - " monthly_gif = f\"{gif_dir}/coherence_monthly.gif\"\n", - " monthly_imgs = _pad_frames(monthly_frames)\n", - " imageio.mimsave(monthly_gif, monthly_imgs, duration=_gif_duration(len(monthly_imgs)))\n", - " print(f\"Wrote {monthly_gif}\")\n", - " else:\n", - " print('No monthly coherence frames found for GIF')\n", - "\n", - "# Backscatter GIF (sigma0/beta0)\n", - "bsc_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_bsc_{CALIBRATION_MODE}_*.tif\"))\n", - "bsc_paths = _filter_paths_by_date(bsc_paths, dateStart, dateEnd)\n", - "bsc_dates = []\n", - "\n", - "# Fixed color range across the series\n", - "bsc_all = []\n", - "for p in bsc_paths:\n", - " da = rioxarray.open_rasterio(p)[0]\n", - " bsc_all.append(da.values.ravel())\n", - "if bsc_all:\n", - " bsc_all = np.concatenate(bsc_all)\n", - " bsc_vmin = np.nanpercentile(bsc_all, 2)\n", - " bsc_vmax = np.nanpercentile(bsc_all, 90)\n", - "else:\n", - " bsc_vmin, bsc_vmax = None, None\n", - "for p in bsc_paths:\n", - " tag = Path(p).name.replace('.tif','')\n", - " bsc_dates.append(_extract_date_from_tag(tag))\n", - "\n", - "if bsc_paths:\n", - " bsc_template = rioxarray.open_rasterio(bsc_paths[0])[0]\n", - " minlon, minlat, maxlon, maxlat = bsc_template.rio.bounds()\n", - " bsc_extent = [minlon, maxlon, minlat, maxlat]\n", - " bsc_frames = _render_frames(\n", - " bsc_paths, f\"{gif_dir}/bsc_frames_{CALIBRATION_MODE}\", cmap=cmc.bilbao,\n", - " vmin=bsc_vmin, vmax=bsc_vmax, title_prefix=None, extent=bsc_extent,\n", - " cbar_label=f\"Backscatter ({CALIBRATION_MODE}) dB\", cbar_ticks=None,\n", - " template=bsc_template, dates=bsc_dates, title_dates=None, show_time_bar=True\n", - " )\n", - " bsc_gif = f\"{gif_dir}/backscatter_{CALIBRATION_MODE}.gif\"\n", - " bsc_imgs = _pad_frames(bsc_frames)\n", - " imageio.mimsave(bsc_gif, bsc_imgs, duration=_gif_duration(len(bsc_imgs)))\n", - " print(f\"Wrote {bsc_gif}\")\n", - "else:\n", - " print('No merged backscatter files found for GIF')\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "opera_cslc_slides", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.14.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 4abb380771c90a50816781ad376849d9a97b3e58 Mon Sep 17 00:00:00 2001 From: Al Handwerger Date: Tue, 3 Feb 2026 09:48:43 -0800 Subject: [PATCH 7/7] Add files via upload fixed chunk issue --- CSLC/Landslides/CSLC-S1_for_landslides.ipynb | 2377 ++++++++++++++++++ 1 file changed, 2377 insertions(+) create mode 100644 CSLC/Landslides/CSLC-S1_for_landslides.ipynb diff --git a/CSLC/Landslides/CSLC-S1_for_landslides.ipynb b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb new file mode 100644 index 0000000..66e222c --- /dev/null +++ b/CSLC/Landslides/CSLC-S1_for_landslides.ipynb @@ -0,0 +1,2377 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "36315bdf", + "metadata": {}, + "source": [ + "# Generate wrapped interferograms, coherence, and backscatter (dB) maps and animations using OPERA CSLC-S1\n", + "\n", + "--- \n", + "\n", + "This notebook:\n", + "- Searches OPERA CSLC-S1 products for your AOI + date range\n", + "- Subsets CSLCs **before download** using opera-utils\n", + "- Builds interferograms/coherence and merges bursts\n", + "- Exports mosaics and visualizations\n", + "\n", + "**Quick start**\n", + "1) Set parameters in the next cell (AOI, date range, pairing)\n", + "2) Run cells top-to-bottom\n", + "3) Outputs land in `savedir/`\n", + "\n", + "**Key toggles**\n", + "- `SAVE_WGS84`: save WGS84 GeoTIFF mosaics when True\n", + "- `DOWNLOAD_WITH_releaseOGRESS`: show progress bar for downloads\n", + "- `USE_WATER_YEAR`: Oct–Sep calendar layout when True\n", + "- `pair_mode` / `t_span`: control IFG pairing (all vs fixed separation)\n", + "\n", + "**Outputs**\n", + "- Subset CSLC H5: `savedir/subset_cslc/*.h5`\n", + "- Mosaics (IFG/COH, native CRS): `savedir/tifs/merged_ifg_*`, `merged_coh_*`\n", + "- WGS84 mosaics: `savedir/tifs/WGS84/merged_ifg_WGS84_*`, `merged_coh_WGS84_*`, `merged_bsc__WGS84_*` (backscatter, dB)\n", + "- Calibrated backscatter (dB) mosaics (native CRS; = sigma0 or beta0): `savedir/tifs/merged_bsc__*.tif`\n", + "- GIFs: `savedir/gifs/*.gif`\n", + "\n", + "### Data Used in the Example: \n", + "\n", + "- **10 meter (Northing) x 5 meter (Easting) North America OPERA Coregistered Single Look Complex from Sentinel-1 products**\n", + " - This dataset contains Level-2 OPERA coregistered single-look-complex (CSLC) data from Sentinel-1 (S1). The data in this example are geocoded CSLC-S1 data covering Palos Verdes landslides, California, USA. \n", + " \n", + " - The OPERA project is generating geocoded burst-wise CSLC-S1 products over North America which includes USA and US Territories within 200 km from the US border, Canada, and all mainland countries from the southern US border down to and including Panama. Each pixel within a burst SLC is represented by a complex number and contains both the amplitude and phase information. The CSLC-S1 products are distributed over projected map coordinates using the Universal Transverse Mercator (UTM) projection with spacing in the X- and Y-directions of 5 m and 10 m, respectively. Each OPERA CSLC-S1 product is distributed as a HDF5 file following the CF-1.8 convention with separate groups containing the data raster layers, the low-resolution correction layers, and relevant product metadata.\n", + "\n", + " - For more information about the OPERA project and other products please visit our website at https://www.jpl.nasa.gov/go/opera .\n", + "\n", + "Please refer to the [OPERA Product Specification Document](https://d2pn8kiwq2w21t.cloudfront.net/documents/OPERA_CSLC-S1_ProductSpec_v1.0.0_D-108278_Initial_2023-09-11_URS321269.pdf) for details about the CSLC-S1 product.\n", + "\n", + "*Prepared by Al Handwerger and M. Grace Bato*\n", + "\n", + "---\n", + "\n", + "## 0. Setup your conda environment\n", + "\n", + "Assuming you have conda installed. Open your terminal and run the following:\n", + "```\n", + "\n", + "# Create the OPERA CSLC environment\n", + "conda (or mamba) env create -f environment.yml\n", + "conda (or mamba) activate opera_cslc_slides\n", + "python -m ipykernel install --user --name opera_cslc_slides\n", + "\n", + "```\n", + "\n", + "---\n", + "\n", + "## 1. Load Python modules" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84a1468d-0aaf-4f06-9875-6b753d94ad42", + "metadata": {}, + "outputs": [], + "source": [ + "## Load necessary modules\n", + "%load_ext watermark\n", + "\n", + "import asf_search as asf\n", + "import cartopy.crs as ccrs\n", + "import cmcrameri.cm as cmc\n", + "import datetime as dt\n", + "import folium\n", + "import geopandas as gpd\n", + "import glob\n", + "import h5py\n", + "import imageio.v2 as imageio\n", + "import math\n", + "import matplotlib.colors as mcolors\n", + "import matplotlib.patches as patches\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from numpy.lib.stride_tricks import sliding_window_view\n", + "from numpy.typing import NDArray\n", + "import os\n", + "import pandas as pd\n", + "from pathlib import Path\n", + "from pyproj import Transformer\n", + "import rasterio\n", + "from rasterio import merge\n", + "from rasterio.crs import CRS\n", + "from rasterio.io import MemoryFile\n", + "from rasterio.transform import from_origin\n", + "from rasterio.warp import calculate_default_transform, reproject, Resampling\n", + "import requests\n", + "import re\n", + "import rioxarray\n", + "from shapely.geometry import box, Point\n", + "from shapely.ops import transform as shp_transform\n", + "import shapely\n", + "import shapely.wkt as wkt\n", + "from subprocess import Popen\n", + "from platform import system\n", + "import sys\n", + "import tempfile\n", + "import time\n", + "from tqdm.auto import tqdm\n", + "import warnings\n", + "\n", + "from affine import Affine\n", + "from concurrent.futures import ThreadPoolExecutor, as_completed\n", + "from getpass import getpass\n", + "from netrc import netrc\n", + "from opera_utils.credentials import get_earthdata_username_password\n", + "from opera_utils.disp._remote import open_file\n", + "from opera_utils.disp._utils import _get_netcdf_encoding\n", + "from osgeo import gdal\n", + "\n", + "proj_dir = os.path.join(sys.prefix, \"share\", \"proj\")\n", + "os.environ[\"PROJ_LIB\"] = proj_dir # for older PROJ\n", + "os.environ[\"PROJ_DATA\"] = proj_dir # for newer PROJ\n", + "\n", + "%watermark --iversions\n", + "\n", + "\n", + "import seaborn as sns\n", + "# ROCKET_CMAP = sns.color_palette('rocket', as_cmap=True)\n", + "from scipy.signal import convolve\n", + "from scipy.ndimage import distance_transform_edt\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9952139c", + "metadata": {}, + "outputs": [], + "source": [ + "# Environment check\n", + "import sys\n", + "import importlib\n", + "\n", + "REQUIRED_PKGS = [\n", + " 'asf_search','cartopy','folium','geopandas','h5py','imageio','matplotlib','numpy','pandas',\n", + " 'pyproj','rasterio','rioxarray','shapely','xarray','opera_utils','tqdm','rich','cmcrameri','seaborn'\n", + "]\n", + "missing = []\n", + "for pkg in REQUIRED_PKGS:\n", + " try:\n", + " importlib.import_module(pkg)\n", + " except Exception:\n", + " missing.append(pkg)\n", + "\n", + "if missing:\n", + " raise ImportError(\"Missing packages: \" + ', '.join(missing) + \". Activate opera_cslc env or install from environment.yml\")\n", + "\n", + "print(f\"Python: {sys.executable}\")\n", + "# Colormap sanity check\n", + "import cmcrameri.cm as cmc\n", + "# _ = ROCKET_CMAP\n", + "\n", + "print('Environment check OK')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1faa1ef0-3d5e-424e-b602-3392b720af6d", + "metadata": {}, + "outputs": [], + "source": [ + "## Notebook display setup\n", + "%matplotlib inline\n", + "%config InlineBackend.figure_format='retina'\n", + "\n", + "# Pandas display\n", + "# pd.set_option('display.max_rows', None)\n", + "pd.set_option('display.max_columns', None)\n", + "\n", + "# Optional reprojection helper for display\n", + "\n", + "# Avoid lots of warnings printing to notebook from asf_search\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "id": "32ef4ad4", + "metadata": {}, + "source": [ + "## 2. Set up your NASA Earthdata Login Credentials" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b6ad5ac-9089-4377-be63-ddf1731263f2", + "metadata": {}, + "outputs": [], + "source": [ + "urs = 'urs.earthdata.nasa.gov'\n", + "prompts = ['Enter NASA Earthdata Login Username: ',\n", + " 'Enter NASA Earthdata Login Password: ']\n", + "\n", + "netrc_name = \"_netrc\" if system() == \"Windows\" else \".netrc\"\n", + "netrc_path = os.path.expanduser(f\"~/{netrc_name}\")\n", + "\n", + "def write_netrc():\n", + " username = getpass(prompt=prompts[0])\n", + " password = getpass(prompt=prompts[1])\n", + " with open(netrc_path, 'a') as f:\n", + " f.write(f\"\\nmachine {urs}\\n\")\n", + " f.write(f\"login {username}\\n\")\n", + " f.write(f\"password {password}\\n\")\n", + " os.chmod(netrc_path, 0o600)\n", + "\n", + "def has_urs_credentials():\n", + " try:\n", + " creds = netrc(netrc_path).authenticators(urs)\n", + " return creds is not None\n", + " except (FileNotFoundError, NetrcParseError):\n", + " return False\n", + "\n", + "if not has_urs_credentials():\n", + " if not os.path.exists(netrc_path):\n", + " open(netrc_path, 'w').close()\n", + " write_netrc()\n", + "\n", + "os.environ[\"GDAL_HTTP_NETRC\"] = \"YES\"\n", + "os.environ[\"GDAL_HTTP_NETRC_FILE\"] = netrc_path" + ] + }, + { + "cell_type": "markdown", + "id": "4f948345", + "metadata": {}, + "source": [ + "## 3. Enter user-defined parameters\n", + "\n", + "**Parameter guide (impact on downstream steps):**\n", + "- **AOI / orbit / path / burst filters**: control which CSLCs are found and subset; changing these changes *all* downstream data.\n", + "- **dateStart / dateEnd**: controls query window and calendar/GIF coverage.\n", + "- **pair_mode / pair_t_span_days**: controls which interferometric pairs are formed; impacts IFG/COH density.\n", + "- **MULTILOOK / TARGET_PIXEL_M**: sets spatial averaging; affects resolution and noise level in IFG/COH/BSC.\n", + "- **CALIBRATION_MODE**: choose backscatter calibration: `sigma0` or `beta0`.\n", + "- **COH_METHOD**: `standard` (default) or `phase_only` (previous behavior).\n", + "- **COH_APPLY_LEE**: apply Lee filter to coherence (optional smoothing).\n", + "- **IFG_APPLY_FILTER**: apply Goldstein filter to interferogram phase.\n", + "- **Note**: `IFG_APPLY_FILTER` only affects the **interferogram phase** (for display/plots). Coherence is always computed from the unfiltered data; use `COH_METHOD` to choose standard vs phase-only, and `COH_APPLY_LEE` for optional smoothing.\n", + "- **MERGE_BLEND_OVERLAP**: if `True`, blend overlap regions between bursts using a distance‑based feather to reduce seams.\n", + "- **MERGE_BLEND_EPS**: small constant to avoid divide‑by‑zero in blend weights. Increase slightly if you see artifacts.\n", + "- **APPLY_WATER_MASK / WATER_MASK_PATH**: masks out water before saving outputs.\n", + "- **SAVE_WGS84**: optional WGS84 GeoTIFFs for easy display.\n", + "- **SKIP_EXISTING_TIFS**: reuses existing GeoTIFFs to speed re-runs (delete old files to force regeneration).\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d5eeba6e", + "metadata": {}, + "outputs": [], + "source": [ + "# User parameters (edit these)\n", + "# AOI is a WKT polygon in EPSG:4326\n", + "# check ASF Vertex to look for correct pass/path for your AOI https://search.asf.alaska.edu/#/?maxResults=250&dataset=OPERA-S1&polygon=POLYGON((-118.3955%2033.7342,-118.3464%2033.7342,-118.3464%2033.7616,-118.3955%2033.7616,-118.3955%2033.7342))&productTypes=CSLC&resultsLoaded=true&granule=OPERA_L2_CSLC-S1_T071-151230-IW3_20260128T135247Z_20260129T074245Z_S1A_VV_v1.1&zoom=10.065¢er=-118.699,33.446\n", + "## Enter user-defined parameters\n", + "SITE_NAME = \"Palos_Verdes_Landslides\" # used for output folder naming\n", + "aoi = \"POLYGON((-118.3955 33.7342,-118.3464 33.7342,-118.3464 33.7616,-118.3955 33.7616,-118.3955 33.7342))\"\n", + "orbitPass = \"DESCENDING\" # ASCENDING or DESCENDING\n", + "pathNumber = 71 #71 DESC and 64 ASC\n", + "# Optional burst selection before download\n", + "# Use subswath (e.g., 'IW2', 'IW3') or specific OPERA burst ID (e.g., 'T071_151230_IW3')\n", + "BURST_SUBSWATH = 'IW3' # e.g., 'IW2' or ['IW2', 'IW3'] or None \n", + "# BURST_SUBSWATH = 'IW3' # e.g., 'IW2' or ['IW2', 'IW3'] or None \n", + "\n", + "BURST_ID = None # e.g., 'T071_151230_IW3' or list of burst IDs\n", + "\n", + "dateStart = dt.datetime.fromisoformat('2025-05-01 00:00:00') #'YYYY-MM-DD HH:MM:SS'\n", + "dateEnd = dt.datetime.fromisoformat('2025-06-01 23:59:59') #'YYYY-MM-DD HH:MM:SS'\n", + "\n", + "# Pairing options\n", + "pair_mode = 't_span' # 'all' or 't_span'\n", + "pair_t_span_days = [12] # int or list of ints (e.g., [6, 12, 24])\n", + "\n", + "# Seasonal filters (exclude pairs if either endpoint falls in these windows)\n", + "EXCLUDE_DATE_RANGES = [] # list of (start, end) like [(\"2023-12-15\",\"2024-03-31\")] or []\n", + "EXCLUDE_MONTHDAY_RANGES = [] # list of (\"MM-DD\",\"MM-DD\") e.g., [(\"12-15\",\"02-15\")], or []\n", + "\n", + "#backscatter\n", + "CALIBRATION_MODE = 'sigma0' # 'sigma0' or 'beta0'\n", + "CALIBRATION_FACTOR_IS_AMPLITUDE = True # False: LUT applies to power (default); True: LUT applies to amplitude, so use squared factor\n", + "\n", + "# Multilooking (spatial averaging)\n", + "# Set either MULTILOOK (looks_y, looks_x) OR TARGET_PIXEL_M (meters). TARGET overrides MULTILOOK.\n", + "MULTILOOK = (1, 1) # e.g., (3, 6) for 30m from (dy=10m, dx=5m)\n", + "TARGET_PIXEL_M = None # e.g., 30.0 meter or 90.0 meter or None\n", + "GOLDSTEIN_ALPHA = 0.5 # 0 (none) to 1 (strong)\n", + "COH_METHOD = 'standard' # 'standard' or 'phase_only'\n", + "COH_USE_GAMMA_NORM = True # True to apply gamma normalization in plots (coherence only)\n", + "COH_NORM_GAMMA = 0.5\n", + "COH_KERNEL = 'boxcar' # 'boxcar' or 'weighted'\n", + "COH_KERNEL_NUM_CONV = 3 # only used for weighted kernel\n", + "COH_APPLY_LEE = False # apply Lee filter to coherence\n", + "IFG_APPLY_FILTER = True # apply Goldstein filter to interferogram phase\n", + "\n", + "#for burst overlaps\n", + "MERGE_BLEND_OVERLAP = True # feather overlaps to reduce burst overlaps\n", + "MERGE_BLEND_EPS = 1e-6\n", + "\n", + "SAVE_WGS84 = False # set True to save WGS84 GeoTIFF mosaics\n", + "SKIP_EXISTING_TIFS = False # skip writing GeoTIFFs if they already exist\n", + "\n", + "DOWNLOAD_WITH_PROGRESS = True # set True for per-file progress bar\n", + "\n", + "\n", + "min_t_span_days = 0 # minimum separation (days)\n", + "max_t_span_days = 12 # maximum separation (days) or None\n", + "max_pairs_per_burst = None # int or None\n", + "max_pairs_total = None # int or None\n", + "\n", + "# Calendar settings\n", + "USE_WATER_YEAR = True # True: Oct–Sep, False: Jan–Dec\n", + "\n", + "DOWNLOAD_PROCESSES = min(8, max(2, (os.cpu_count() or 4) // 2))\n", + "# DOWNLOAD_BATCH_SIZE = 5\n", + "\n", + "\n", + "# Normalize name for filesystem (letters/numbers/_/- only)\n", + "site_slug = re.sub(r\"[^A-Za-z0-9_-]+\", \"\", SITE_NAME)\n", + "orbit_code = orbitPass[0].upper() # 'A' or 'D'\n", + "savedir = f'./{site_slug}_{orbit_code}{pathNumber:03d}/'\n", + "\n", + "# Water mask options\n", + "# in params cell\n", + "WATER_MASK_PATH = f\"{savedir}/water_mask/water_mask_esa_wc2021.tif\"\n", + "APPLY_WATER_MASK = True" + ] + }, + { + "cell_type": "markdown", + "id": "80842b39", + "metadata": {}, + "source": [ + "**AOI size guidance**\n", + "\n", + "This notebook is tuned for *small* AOIs (e.g., individual landslides). The main opera-utils release includes fixes for small AOI subsetting. Large AOIs can dramatically increase download time, disk usage, and RAM needs. If you use a large polygon, consider:\n", + "- Shorter date ranges or fewer bursts\n", + "- Coarser `TARGET_PIXEL_M` (more multilooking)\n", + "- Tiling the AOI into smaller polygons and mosaicking later\n", + "\n", + "If you see memory errors or very long runtimes, reduce the AOI size or date range.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89fa0047", + "metadata": {}, + "outputs": [], + "source": [ + "# Utilities (used by multiple sections)\n", + "\n", + "def _load_water_mask_match(mask_path, shape, transform, crs):\n", + " if mask_path is None or not Path(mask_path).exists():\n", + " return None\n", + " with rasterio.open(mask_path) as src:\n", + " src_mask = src.read(1)\n", + " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", + " return src_mask\n", + " dst = np.zeros(shape, dtype=src_mask.dtype)\n", + " reproject(\n", + " source=src_mask,\n", + " destination=dst,\n", + " src_transform=src.transform,\n", + " src_crs=src.crs,\n", + " dst_transform=transform,\n", + " dst_crs=crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " return dst\n", + "\n", + "\n", + "def _apply_water_mask(arr, mask):\n", + " # mask: 1 = keep land, 0 = water\n", + " return np.where(mask == 0, np.nan, arr)\n", + "\n", + "\n", + "def _trim_nan_border(arr, transform):\n", + " data = arr[0] if arr.ndim == 3 else arr\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if not mask.any():\n", + " return arr, transform\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " data = data[r0:r1, c0:c1]\n", + " if arr.ndim == 3:\n", + " arr = data[None, ...]\n", + " else:\n", + " arr = data\n", + " new_transform = transform * Affine.translation(c0, r0)\n", + " return arr, new_transform\n", + "\n", + "\n", + "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", + " dst_crs = 'EPSG:4326'\n", + " dst_transform, width, height = calculate_default_transform(\n", + " f'EPSG:{epsg}', dst_crs, mosaic.shape[2], mosaic.shape[1],\n", + " *rasterio.transform.array_bounds(mosaic.shape[1], mosaic.shape[2], transform)\n", + " )\n", + " dest = np.zeros((1, height, width), dtype=mosaic.dtype)\n", + " reproject(\n", + " source=mosaic,\n", + " destination=dest,\n", + " src_transform=transform,\n", + " src_crs=f'EPSG:{epsg}',\n", + " dst_transform=dst_transform,\n", + " dst_crs=dst_crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " out_meta = {\n", + " 'driver': 'GTiff',\n", + " 'height': height,\n", + " 'width': width,\n", + " 'count': 1,\n", + " 'dtype': mosaic.dtype,\n", + " 'crs': dst_crs,\n", + " 'transform': dst_transform,\n", + " }\n", + " with rasterio.open(out_path, 'w', **out_meta) as dst:\n", + " dst.write(dest)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f072529", + "metadata": {}, + "outputs": [], + "source": [ + "# ESA WorldCover 2021 water mask (GDAL-only)\n", + "\n", + "ESA_WC_GRID_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/esa_worldcover_grid.fgb\"\n", + "ESA_WC_BASE_URL = \"https://esa-worldcover.s3.eu-central-1.amazonaws.com/v200/2021/map\"\n", + "# ESA WorldCover class codes: 80 = Permanent water bodies\n", + "ESA_WC_WATER_CLASSES = {80}\n", + "\n", + "\n", + "def build_worldcover_water_mask(aoi_wkt, out_path, target_res_deg=None):\n", + " # Create a binary land mask from ESA WorldCover (1=land, 0=water).\n", + " out_path = Path(out_path)\n", + " out_path.parent.mkdir(parents=True, exist_ok=True)\n", + " if out_path.exists():\n", + " return out_path\n", + "\n", + "\n", + " aoi_geom = shapely.wkt.loads(aoi_wkt)\n", + " # Load tile grid and select intersecting tiles\n", + " grid = gpd.read_file(ESA_WC_GRID_URL)\n", + " grid = grid.to_crs(\"EPSG:4326\")\n", + " # Find tile id column (varies by grid version)\n", + " tile_col = next((c for c in grid.columns if 'tile' in c.lower()), None)\n", + " if tile_col is None:\n", + " raise RuntimeError(f\"No tile column found in grid columns: {list(grid.columns)}\")\n", + " tiles = grid[grid.intersects(aoi_geom)][tile_col].tolist()\n", + " if not tiles:\n", + " raise RuntimeError(\"No WorldCover tiles intersect AOI\")\n", + "\n", + " print(f\"Selected tiles: {tiles}\")\n", + "\n", + " tile_urls = [\n", + " f\"{ESA_WC_BASE_URL}/ESA_WorldCover_10m_2021_v200_{t}_Map.tif\"\n", + " for t in tiles\n", + " ]\n", + "\n", + " # Quick URL check for first tile\n", + " first_url = tile_urls[0]\n", + " try:\n", + " _ = gdal.Open(first_url)\n", + " except Exception as e:\n", + " raise RuntimeError(f\"GDAL cannot open first tile URL: {first_url}\\n{e}\")\n", + "\n", + " vrt_path = out_path.with_suffix(\".vrt\")\n", + " gdal.BuildVRT(str(vrt_path), tile_urls)\n", + "\n", + " minx, miny, maxx, maxy = aoi_geom.bounds\n", + " warp_kwargs = dict(\n", + " format=\"GTiff\",\n", + " outputBounds=[minx, miny, maxx, maxy],\n", + " multithread=True,\n", + " )\n", + " if target_res_deg is not None:\n", + " warp_kwargs.update(dict(xRes=target_res_deg, yRes=target_res_deg, targetAlignedPixels=True))\n", + "\n", + " tmp_map = out_path.with_name(out_path.stem + \"_map.tif\")\n", + " warp_ds = gdal.Warp(str(tmp_map), str(vrt_path), **warp_kwargs)\n", + " if warp_ds is None:\n", + " raise RuntimeError(\"GDAL Warp returned None. Check network access/URL.\")\n", + " warp_ds = None\n", + "\n", + " with rasterio.open(tmp_map) as src:\n", + " data = src.read(1)\n", + " profile = src.profile\n", + "\n", + " mask = (~np.isin(data, list(ESA_WC_WATER_CLASSES))).astype(\"uint8\")\n", + " profile.update(dtype=\"uint8\", count=1, nodata=0)\n", + "\n", + " with rasterio.open(out_path, \"w\", **profile) as dst:\n", + " dst.write(mask, 1)\n", + "\n", + " return out_path\n", + "\n", + "\n", + "if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " WATER_MASK_PATH = build_worldcover_water_mask(aoi, WATER_MASK_PATH)" + ] + }, + { + "cell_type": "markdown", + "id": "98916bef", + "metadata": {}, + "source": [ + "## 4. Query OPERA CSLCs using `asf_search`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5e33d72e-2e75-4099-93d6-5cd058643e35", + "metadata": {}, + "outputs": [], + "source": [ + "## Search for OPERA CSLC data in ASF DAAC\n", + "try:\n", + " search_params = dict(\n", + " intersectsWith= aoi,\n", + " dataset='OPERA-S1',\n", + " processingLevel='CSLC',\n", + " flightDirection = orbitPass,\n", + " start=dateStart,\n", + " end=dateEnd)\n", + "\n", + " ## Return results\n", + " results = asf.search(**search_params)\n", + " print(f\"Length of Results: {len(results)}\")\n", + "\n", + "except TypeError:\n", + " search_params = dict(\n", + " intersectsWith= aoi.wkt,\n", + " dataset='OPERA-S1',\n", + " processingLevel='CSLC',\n", + " flightDirection = orbitPass,\n", + " start=dateStart,\n", + " end=dateEnd)\n", + "\n", + " ## Return results\n", + " results = asf.search(**search_params)\n", + " print(f\"Length of Results: {len(results)}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b2f5b5df", + "metadata": {}, + "outputs": [], + "source": [ + "## Save the results in a geopandas dataframe\n", + "gf = gpd.GeoDataFrame.from_features(results.geojson(), crs='EPSG:4326')\n", + "\n", + "## Filter data based on specified track number\n", + "gf = gf[gf.pathNumber==pathNumber]\n", + "# gf = gf[gf.pgeVersion==\"2.1.1\"] " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fe3a0963-1d13-4b99-955a-b2c0fa45f1e3", + "metadata": {}, + "outputs": [], + "source": [ + "# Get only relevant metadata\n", + "cslc_df = gf[['operaBurstID', 'fileID', 'startTime', 'stopTime', 'url', 'geometry', 'pgeVersion']]\n", + "cslc_df['startTime'] = pd.to_datetime(cslc_df.startTime).dt.date\n", + "cslc_df['stopTime'] = pd.to_datetime(cslc_df.stopTime).dt.date\n", + "\n", + "# Extract production time from fileID (2nd date token)\n", + "def _prod_time_from_fileid(file_id):\n", + " # Example: OPERA_L2_CSLC-S1_..._20221122T161650Z_20240504T081640Z_...\n", + " parts = str(file_id).split('_')\n", + " return parts[5] if len(parts) > 5 else None\n", + "\n", + "cslc_df['productionTime'] = pd.to_datetime(cslc_df['fileID'].apply(_prod_time_from_fileid), format='%Y%m%dT%H%M%SZ', errors='coerce')\n", + "\n", + "# Keep newest duplicate by productionTime (fallback to pgeVersion, stopTime)\n", + "cslc_df = cslc_df.sort_values(by=['operaBurstID', 'startTime', 'productionTime', 'pgeVersion', 'stopTime'])\n", + "cslc_df = cslc_df.drop_duplicates(subset=['operaBurstID', 'startTime'], keep='last', ignore_index=True)\n", + "\n", + "\n", + "def _subswath_from_fileid(file_id):\n", + " # Example: ...-IW2_... -> IW2\n", + " m = re.search(r\"-IW[1-3]_\", str(file_id))\n", + " return m.group(0)[1:4] if m else None\n", + "\n", + "cslc_df['burstSubswath'] = cslc_df['fileID'].apply(_subswath_from_fileid)\n", + "\n", + "# Optional filtering by subswath or specific burst IDs\n", + "if BURST_SUBSWATH:\n", + " if isinstance(BURST_SUBSWATH, (list, tuple, set)):\n", + " subswaths = {str(s).upper() for s in BURST_SUBSWATH}\n", + " else:\n", + " subswaths = {str(BURST_SUBSWATH).upper()}\n", + " cslc_df = cslc_df[cslc_df['burstSubswath'].str.upper().isin(subswaths)]\n", + "\n", + "if BURST_ID:\n", + " if isinstance(BURST_ID, (list, tuple, set)):\n", + " burst_ids = {str(b) for b in BURST_ID}\n", + " else:\n", + " burst_ids = {str(BURST_ID)}\n", + " cslc_df = cslc_df[cslc_df['operaBurstID'].isin(burst_ids)]\n", + "cslc_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "76cb1007", + "metadata": {}, + "outputs": [], + "source": [ + "# Build AOI geometry\n", + "aoi_geom = wkt.loads(aoi)\n", + "aoi_gdf = gpd.GeoDataFrame(geometry=[aoi_geom], crs=\"EPSG:4326\")\n", + "\n", + "# Map center\n", + "centroid = aoi_gdf.geometry[0].centroid\n", + "m = folium.Map(location=[centroid.y, centroid.x], zoom_start=9, tiles=\"Esri.WorldImagery\")\n", + "\n", + "# Add CSLC footprints\n", + "folium.GeoJson(\n", + " data=cslc_df[['operaBurstID','geometry']].set_geometry('geometry').to_crs(\"EPSG:4326\").__geo_interface__,\n", + " name=\"CSLC footprints\",\n", + " style_function=lambda x: {\n", + " \"fillColor\": \"blue\",\n", + " \"color\": \"blue\",\n", + " \"weight\": 2,\n", + " \"fillOpacity\": 0.1,\n", + " },\n", + ").add_to(m)\n", + "\n", + "# Add AOI\n", + "folium.GeoJson(\n", + " data=aoi_gdf.__geo_interface__,\n", + " name=\"AOI\",\n", + " style_function=lambda x: {\n", + " \"fillColor\": \"red\",\n", + " \"color\": \"red\",\n", + " \"weight\": 2,\n", + " \"fillOpacity\": 0.1,\n", + " },\n", + ").add_to(m)\n", + "\n", + "folium.LayerControl().add_to(m)\n", + "\n", + "m" + ] + }, + { + "cell_type": "markdown", + "id": "9ca8a611", + "metadata": {}, + "source": [ + "## 5. Download the CSLC-S1 locally" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aea6a5b7", + "metadata": {}, + "outputs": [], + "source": [ + "## Download step skipped: CSLC subsets are streamed via opera-utils\n", + "print('Skipping full CSLC downloads; using opera-utils HTTP subsetting.')\n", + "\n", + "# Sort the CSLC-S1 by burstID and date\n", + "cslc_df = cslc_df.sort_values(by=[\"operaBurstID\", \"startTime\"], ignore_index=True)\n", + "# cslc_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa4801fc", + "metadata": {}, + "outputs": [], + "source": [ + "# Enforce date range on dataframe (useful when re-running with narrower dates)\n", + "date_start_day = dateStart.date()\n", + "date_end_day = dateEnd.date()\n", + "cslc_df = cslc_df[(cslc_df['startTime'] >= date_start_day) & (cslc_df['startTime'] <= date_end_day)]\n", + "cslc_df = cslc_df.reset_index(drop=True)\n", + "cslc_df" + ] + }, + { + "cell_type": "markdown", + "id": "48add9c3", + "metadata": {}, + "source": [ + "## 6. Read each CSLC-S1 and stack them together\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01891a96", + "metadata": {}, + "outputs": [], + "source": [ + "cslc_stack = []; cslc_dates = []; bbox_stack = []; xcoor_stack = []; ycoor_stack = []\n", + "\n", + "\n", + "subset_dir = f\"{savedir}/subset_cslc\"\n", + "os.makedirs(subset_dir, exist_ok=True)\n", + "\n", + "\n", + "\n", + "def _clamp_chunks(chunks, shape):\n", + " # Ensure chunk dims do not exceed data shape\n", + " return tuple(max(1, min(c, s)) for c, s in zip(chunks, shape))\n", + "\n", + "def _extract_subset(input_obj, outpath, rows, cols, chunks=(1,256,256)):\n", + " X0, X1 = (cols.start, cols.stop) if cols is not None else (None, None)\n", + " Y0, Y1 = (rows.start, rows.stop) if rows is not None else (None, None)\n", + " ds = xr.open_dataset(input_obj, engine=\"h5netcdf\", group=\"data\")\n", + " if 'VV' not in ds.data_vars:\n", + " raise ValueError('Source missing VV data')\n", + " subset = ds.isel(y_coordinates=slice(Y0, Y1), x_coordinates=slice(X0, X1))\n", + " # clamp chunks to data shape\n", + " data_shape = subset[\"VV\"].shape\n", + " safe_chunks = _clamp_chunks(chunks, data_shape)\n", + " subset.to_netcdf(\n", + " outpath,\n", + " engine=\"h5netcdf\",\n", + " group=\"data\",\n", + " encoding=_get_netcdf_encoding(subset, chunks=safe_chunks),\n", + " )\n", + " for group in (\"metadata\", \"identification\"):\n", + " with h5py.File(input_obj) as hf, h5py.File(outpath, \"a\") as dest_hf:\n", + " hf.copy(group, dest_hf, name=group)\n", + " with h5py.File(outpath, \"a\") as hf:\n", + " ctype = h5py.h5t.py_create(np.complex64)\n", + " ctype.commit(hf[\"/\"].id, np.bytes_(\"complex64\"))\n", + "\n", + "def _subset_h5_to_disk(url, aoi_wkt, out_dir):\n", + " outpath = Path(out_dir) / Path(url).name\n", + " if outpath.exists():\n", + " if outpath.stat().st_size < 100 * 1024:\n", + " outpath.unlink()\n", + " else:\n", + " return outpath\n", + "\n", + " # determine row/col slices by reading coords\n", + " with open_file(url) as in_f:\n", + " ds = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", + " xcoor = ds[\"x_coordinates\"].values\n", + " ycoor = ds[\"y_coordinates\"].values\n", + " epsg = int(ds[\"projection\"].values)\n", + "\n", + " aoi_geom = wkt.loads(aoi_wkt)\n", + " if epsg != 4326:\n", + " transformer = Transformer.from_crs('EPSG:4326', f'EPSG:{epsg}', always_xy=True)\n", + " aoi_geom = shp_transform(transformer.transform, aoi_geom)\n", + " minx, miny, maxx, maxy = aoi_geom.bounds\n", + " x_mask = (xcoor >= minx) & (xcoor <= maxx)\n", + " y_mask = (ycoor >= miny) & (ycoor <= maxy)\n", + " if not x_mask.any() or not y_mask.any():\n", + " raise ValueError('AOI does not intersect this CSLC extent')\n", + " ix = np.where(x_mask)[0]\n", + " iy = np.where(y_mask)[0]\n", + " rows = slice(iy.min(), iy.max()+1)\n", + " cols = slice(ix.min(), ix.max()+1)\n", + "\n", + " if url.startswith('s3://'):\n", + " with open_file(url) as in_f:\n", + " _extract_subset(in_f, outpath, rows, cols)\n", + " else:\n", + " # HTTPS: download to temp then subset\n", + " with tempfile.NamedTemporaryFile(suffix='.h5') as tf:\n", + " if url.startswith('http'):\n", + " session = requests.Session()\n", + " username, password = get_earthdata_username_password()\n", + " session.auth = (username, password)\n", + " resp = session.get(url, stream=True)\n", + " resp.raise_for_status()\n", + " content_type = resp.headers.get('Content-Type', '').lower()\n", + " if 'text/html' in content_type:\n", + " raise ValueError('Got HTML response instead of HDF5; check Earthdata login/auth')\n", + " for chunk in resp.iter_content(chunk_size=1024 * 1024):\n", + " if chunk:\n", + " tf.write(chunk)\n", + " tf.flush()\n", + " if tf.tell() < 100 * 1024:\n", + " raise ValueError('Downloaded file too small; likely auth/redirect issue')\n", + " _extract_subset(tf.name, outpath, rows, cols)\n", + " # validate output has VV; remove tiny/invalid outputs\n", + " try:\n", + " with h5py.File(outpath, 'r') as h5:\n", + " if '/data/VV' not in h5:\n", + " raise ValueError('Subset missing /data/VV')\n", + " if outpath.stat().st_size < 100 * 1024:\n", + " raise ValueError('Subset file too small')\n", + " except Exception:\n", + " if Path(outpath).exists():\n", + " Path(outpath).unlink()\n", + " raise\n", + " return outpath\n", + "\n", + "def _load_subset(file_id, url, start_date):\n", + " try:\n", + " outpath = _subset_h5_to_disk(url, aoi, subset_dir)\n", + " except FileNotFoundError:\n", + " return None # skip missing products\n", + " # now read subset locally with h5py (fast)\n", + " with h5py.File(outpath, 'r') as h5:\n", + " cslc = h5['/data/VV'][:]\n", + " xcoor = h5['/data/x_coordinates'][:]\n", + " ycoor = h5['/data/y_coordinates'][:]\n", + " dx = int(h5['/data/x_spacing'][()])\n", + " dy = int(h5['/data/y_spacing'][()])\n", + " epsg = int(h5['/data/projection'][()])\n", + " sensing_start = h5['/metadata/processing_information/input_burst_metadata/sensing_start'][()].astype(str)\n", + " sensing_stop = h5['/metadata/processing_information/input_burst_metadata/sensing_stop'][()].astype(str)\n", + " dims = h5['/metadata/processing_information/input_burst_metadata/shape'][:]\n", + " bounding_polygon = h5['/identification/bounding_polygon'][()].astype(str)\n", + " orbit_direction = h5['/identification/orbit_pass_direction'][()].astype(str)\n", + " center_lon, center_lat = h5['/metadata/processing_information/input_burst_metadata/center']\n", + " wavelength = h5['/metadata/processing_information/input_burst_metadata/wavelength'][()].astype(str)\n", + " subset_bbox = [float(xcoor.min()), float(xcoor.max()), float(ycoor.min()), float(ycoor.max())]\n", + " return cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox\n", + "\n", + "# Subset with progress (parallel)\n", + "\n", + "items = list(zip(cslc_df.fileID, cslc_df.url, cslc_df.startTime))\n", + "import xarray as xr\n", + "# Diagnostic: check pixel spacing before multilooking\n", + "with open_file(items[0][1]) as in_f:\n", + " ds0 = xr.open_dataset(in_f, engine=\"h5netcdf\", group=\"data\")\n", + " dx0 = float(ds0[\"x_spacing\"].values)\n", + " dy0 = float(ds0[\"y_spacing\"].values)\n", + "print(f\"Pixel spacing (dx, dy) = ({dx0}, {dy0})\")\n", + "\n", + "# Derive anisotropic coherence window from TARGET_PIXEL_M or MULTILOOK\n", + "def _odd_at_least_one(n):\n", + " n = max(1, int(round(n)))\n", + " return n if n % 2 == 1 else n + 1\n", + "\n", + "# Aim for ~60 m window; if multilook pixel size exceeds 60 m, use that instead\n", + "if TARGET_PIXEL_M is not None:\n", + " base_win_m = TARGET_PIXEL_M\n", + "else:\n", + " base_win_m = max(abs(dx0) * MULTILOOK[1], abs(dy0) * MULTILOOK[0])\n", + "\n", + "COH_WIN_M = max(60.0, base_win_m)\n", + "COH_WIN_X = _odd_at_least_one(COH_WIN_M / abs(dx0))\n", + "COH_WIN_Y = _odd_at_least_one(COH_WIN_M / abs(dy0))\n", + "print(f\"Using anisotropic COH_WIN (Y,X)=({COH_WIN_Y},{COH_WIN_X}) from COH_WIN_M={COH_WIN_M} m\")\n", + "\n", + "# Derive MULTILOOK from TARGET_PIXEL_M if provided\n", + "if TARGET_PIXEL_M is not None:\n", + " looks_x = max(1, int(round(TARGET_PIXEL_M / abs(dx0))))\n", + " looks_y = max(1, int(round(TARGET_PIXEL_M / abs(dy0))))\n", + " MULTILOOK = (looks_y, looks_x)\n", + " print(f\"Using MULTILOOK={MULTILOOK} for TARGET_PIXEL_M={TARGET_PIXEL_M} (dx={dx0}, dy={dy0})\")\n", + "\n", + "\n", + "results = [None] * len(items)\n", + "_t0 = time.perf_counter()\n", + "with ThreadPoolExecutor(max_workers=DOWNLOAD_PROCESSES) as ex:\n", + " futures = {ex.submit(_load_subset, fileID, url, start_date): i for i, (fileID, url, start_date) in enumerate(items)}\n", + " for fut in tqdm(as_completed(futures), total=len(futures), desc='Subsetting CSLC'):\n", + " i = futures[fut]\n", + " results[i] = fut.result()\n", + "_t1 = time.perf_counter()\n", + "print(f\"Subset/download time: {_t1 - _t0:.1f} s\")\n", + "\n", + "valid_idx = [i for i, res in enumerate(results) if res is not None]\n", + "if len(valid_idx) != len(results):\n", + " print(f\"Skipping {len(results) - len(valid_idx)} failed subsets\")\n", + " cslc_df = cslc_df.iloc[valid_idx].reset_index(drop=True)\n", + " results = [results[i] for i in valid_idx]\n", + "\n", + "for (fileID, start_date), res in zip(zip(cslc_df.fileID, cslc_df.startTime), results):\n", + " cslc, xcoor, ycoor, dx, dy, epsg, sensing_start, sensing_stop, dims, bounding_polygon, orbit_direction, center_lon, center_lat, wavelength, subset_bbox = res\n", + " cslc_stack.append(cslc)\n", + " cslc_dates.append(pd.to_datetime(sensing_start).date())\n", + " if subset_bbox is not None:\n", + " bbox = subset_bbox\n", + " else:\n", + " cslc_poly = wkt.loads(bounding_polygon)\n", + " bbox = [cslc_poly.bounds[0], cslc_poly.bounds[2], cslc_poly.bounds[1], cslc_poly.bounds[3]]\n", + " bbox_stack.append(bbox)\n", + " xcoor_stack.append(xcoor)\n", + " ycoor_stack.append(ycoor)" + ] + }, + { + "cell_type": "markdown", + "id": "f5e97e45", + "metadata": {}, + "source": [ + "
\n", + "7. Generate the interferograms, compute for the coherence, save the files as GeoTiffs\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d87c1f9", + "metadata": {}, + "outputs": [], + "source": [ + "import h5py, os, glob\n", + "f = sorted(glob.glob(f\"{savedir}/subset_cslc/*.h5\"))[0]\n", + "print(f, os.path.getsize(f))\n", + "with h5py.File(f, \"r\") as h5:\n", + " print(list(h5[\"/data\"].keys()))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57c74ac4", + "metadata": {}, + "outputs": [], + "source": [ + "def colorize(array=[], cmap='RdBu', cmin=[], cmax=[]):\n", + " normed_data = (array - cmin) / (cmax - cmin) \n", + " cm = plt.cm.get_cmap(cmap)\n", + " return cm(normed_data) " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25c56034", + "metadata": {}, + "outputs": [], + "source": [ + "def goldstein_filter(ifg_cpx, alpha=0.5, pad=32, edge_trim=16):\n", + " # Goldstein filter with padding + taper + mask to reduce edge effects\n", + " mask = np.isfinite(ifg_cpx)\n", + " data = np.nan_to_num(ifg_cpx, nan=0.0)\n", + " if pad and pad > 0:\n", + " data = np.pad(data, ((pad, pad), (pad, pad)), mode=\"reflect\")\n", + " mask = np.pad(mask, ((pad, pad), (pad, pad)), mode=\"constant\", constant_values=False)\n", + " # Apply 2D Hann window (taper)\n", + " wy = np.hanning(data.shape[0])\n", + " wx = np.hanning(data.shape[1])\n", + " window = wy[:, None] * wx[None, :]\n", + " f = np.fft.fft2(data * window)\n", + " s = np.abs(f)\n", + " s = s / (s.max() + 1e-8)\n", + " f_filt = f * (s ** alpha)\n", + " out = np.fft.ifft2(f_filt)\n", + " if pad and pad > 0:\n", + " out = out[pad:-pad, pad:-pad]\n", + " mask = mask[pad:-pad, pad:-pad]\n", + " # restore NaNs outside valid mask\n", + " out[~mask] = np.nan\n", + " if edge_trim and edge_trim > 0:\n", + " out[:edge_trim, :] = np.nan\n", + " out[-edge_trim:, :] = np.nan\n", + " out[:, :edge_trim] = np.nan\n", + " out[:, -edge_trim:] = np.nan\n", + " return out" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82bda5ae", + "metadata": {}, + "outputs": [], + "source": [ + "def rasterWrite(outtif,arr,transform,epsg,dtype='float32'):\n", + " #writing geotiff using rasterio\n", + " \n", + " new_dataset = rasterio.open(outtif, 'w', driver='GTiff',\n", + " height = arr.shape[0], width = arr.shape[1],\n", + " count=1, dtype=dtype,\n", + " crs=CRS.from_epsg(epsg),\n", + " transform=transform,nodata=np.nan)\n", + " new_dataset.write(arr, 1)\n", + " new_dataset.close() " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b47f82a5", + "metadata": {}, + "outputs": [], + "source": [ + "## Build date pairs per burstID\n", + "cslc_dates = cslc_df[[\"startTime\"]]\n", + "burstID = cslc_df.operaBurstID.drop_duplicates(ignore_index=True)\n", + "n_unique_burstID = len(burstID)\n", + "\n", + "def _lag_list(lag):\n", + " if lag is None:\n", + " return []\n", + " if isinstance(lag, (list, tuple, set)):\n", + " return sorted({int(x) for x in lag})\n", + " return [int(lag)]\n", + "\n", + "pair_lags = _lag_list(pair_t_span_days)\n", + "pair_indices = [] # list of (ref_idx, sec_idx) in cslc_df order\n", + "\n", + "for bid, group in cslc_df.groupby('operaBurstID'):\n", + " group = group.sort_values('startTime')\n", + " idx = group.index.to_list()\n", + " dates = group['startTime'].to_list()\n", + "\n", + " burst_pairs = []\n", + " if pair_mode == 'all':\n", + " for i in range(len(idx)):\n", + " for j in range(i+1, len(idx)):\n", + " delta = (dates[j] - dates[i]).days\n", + " if delta < min_t_span_days:\n", + " continue\n", + " if max_t_span_days is not None and delta > max_t_span_days:\n", + " continue\n", + " burst_pairs.append((idx[i], idx[j]))\n", + " elif pair_mode == 't_span':\n", + " for i in range(len(idx)):\n", + " for j in range(i+1, len(idx)):\n", + " delta = (dates[j] - dates[i]).days\n", + " if delta in pair_lags and delta >= min_t_span_days and (max_t_span_days is None or delta <= max_t_span_days):\n", + " burst_pairs.append((idx[i], idx[j]))\n", + " else:\n", + " raise ValueError(\"pair_mode must be 'all' or 't_span'\")\n", + "\n", + " if max_pairs_per_burst is not None:\n", + " burst_pairs = burst_pairs[:int(max_pairs_per_burst)]\n", + "\n", + " pair_indices.extend(burst_pairs)\n", + "\n", + "if max_pairs_total is not None:\n", + " pair_indices = pair_indices[:int(max_pairs_total)]\n", + "\n", + "# Sort pairs by date, then burstID (so same dates group together)\n", + "def _pair_sort_key(pair):\n", + " ref_idx, sec_idx = pair\n", + " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", + " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", + " burst = cslc_df.operaBurstID.iloc[ref_idx]\n", + " return (ref_date, sec_date, burst)\n", + "pair_indices = sorted(pair_indices, key=_pair_sort_key)\n", + "print(f'Pair count: {len(pair_indices)}')\n", + "\n", + "# Seasonal filtering based on pair endpoints\n", + "from datetime import date\n", + "\n", + "# Normalize exclude ranges to date objects\n", + "_excl_ranges = []\n", + "for s, e in EXCLUDE_DATE_RANGES:\n", + " try:\n", + " s_d = pd.to_datetime(s).date()\n", + " e_d = pd.to_datetime(e).date()\n", + " except Exception:\n", + " continue\n", + " _excl_ranges.append((s_d, e_d))\n", + "\n", + "# Normalize month-day ranges\n", + "if EXCLUDE_MONTHDAY_RANGES:\n", + " if len(EXCLUDE_MONTHDAY_RANGES) == 2 and all(isinstance(x, str) for x in EXCLUDE_MONTHDAY_RANGES):\n", + " EXCLUDE_MONTHDAY_RANGES = [(EXCLUDE_MONTHDAY_RANGES[0], EXCLUDE_MONTHDAY_RANGES[1])]\n", + "\n", + "# Filter pairs: exclude if either endpoint falls in excluded months or ranges\n", + "filtered_pairs = []\n", + "for r, s in pair_indices:\n", + " d1 = pd.to_datetime(cslc_dates.iloc[r].values[0]).date()\n", + " d2 = pd.to_datetime(cslc_dates.iloc[s].values[0]).date()\n", + "\n", + " # Date-range exclusion\n", + " excluded = False\n", + " for rs, re in _excl_ranges:\n", + " if rs <= d1 <= re or rs <= d2 <= re:\n", + " excluded = True\n", + " break\n", + " if excluded:\n", + " continue\n", + "\n", + " # Month-day exclusion (recurring each year)\n", + " if EXCLUDE_MONTHDAY_RANGES:\n", + " md1 = (d1.month, d1.day)\n", + " md2 = (d2.month, d2.day)\n", + " for rng in EXCLUDE_MONTHDAY_RANGES:\n", + " if not (isinstance(rng, (list, tuple)) and len(rng) == 2):\n", + " continue\n", + " s_md, e_md = rng\n", + " try:\n", + " s_m, s_d = map(int, s_md.split('-'))\n", + " e_m, e_d = map(int, e_md.split('-'))\n", + " except Exception:\n", + " continue\n", + " start = (s_m, s_d)\n", + " end = (e_m, e_d)\n", + " # handle ranges that wrap year end (e.g., 12-15 to 02-15)\n", + " def _in_range(md):\n", + " if start <= end:\n", + " return start <= md <= end\n", + " return md >= start or md <= end\n", + " if _in_range(md1) or _in_range(md2):\n", + " excluded = True\n", + " break\n", + " if excluded:\n", + " continue\n", + "\n", + " filtered_pairs.append((r, s))\n", + "\n", + "pair_indices = filtered_pairs\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6852f778", + "metadata": {}, + "outputs": [], + "source": [ + "def take_looks(arr, row_looks, col_looks, func_type=\"nanmean\", edge_strategy=\"cutoff\"):\n", + " if row_looks == 1 and col_looks == 1:\n", + " return arr\n", + " if arr.ndim != 2:\n", + " raise ValueError(\"take_looks expects 2D array\")\n", + " rows, cols = arr.shape\n", + " if edge_strategy == \"cutoff\":\n", + " rows = (rows // row_looks) * row_looks\n", + " cols = (cols // col_looks) * col_looks\n", + " arr = arr[:rows, :cols]\n", + " elif edge_strategy == \"pad\":\n", + " pad_r = (-rows) % row_looks\n", + " pad_c = (-cols) % col_looks\n", + " if pad_r or pad_c:\n", + " arr = np.pad(arr, ((0, pad_r), (0, pad_c)), mode=\"constant\", constant_values=np.nan)\n", + " rows, cols = arr.shape\n", + " else:\n", + " raise ValueError(\"edge_strategy must be 'cutoff' or 'pad'\")\n", + "\n", + " new_rows = rows // row_looks\n", + " new_cols = cols // col_looks\n", + " func = getattr(np, func_type)\n", + " with warnings.catch_warnings():\n", + " warnings.simplefilter(\"ignore\", category=RuntimeWarning)\n", + " return func(arr.reshape(new_rows, row_looks, new_cols, col_looks), axis=(1, 3))\n", + "\n", + "\n", + "def _multilook(arr, looks_y=1, looks_x=1):\n", + " return take_looks(arr, looks_y, looks_x, func_type=\"nanmean\", edge_strategy=\"cutoff\")\n", + "\n", + "\n", + "def _box_mean(arr, win_y, win_x):\n", + " pad_y = win_y // 2\n", + " pad_x = win_x // 2\n", + " arr_p = np.pad(arr, ((pad_y, pad_y), (pad_x, pad_x)), mode='reflect')\n", + " windows = sliding_window_view(arr_p, (win_y, win_x))\n", + " return windows.mean(axis=(-2, -1))\n", + "\n", + "def get_kernel(size_y, size_x, num_conv):\n", + " if not isinstance(num_conv, int):\n", + " raise ValueError('num_conv must be an integer')\n", + " k0 = np.ones((size_y, size_x), dtype=np.float32)\n", + " k = k0\n", + " for i in range(num_conv):\n", + " if i > 3:\n", + " k = convolve(k, k0, mode='same')\n", + " else:\n", + " k = convolve(k, k0)\n", + " k = k / np.sum(k)\n", + " return k.astype(np.float32)\n", + "\n", + "\n", + "def _weighted_mean(arr, k):\n", + " # supports complex or real arrays\n", + " return convolve(arr, k, mode='same')\n", + "\n", + "\n", + "\n", + "def lee_filter(img, win_y=5, win_x=5):\n", + " mean = _box_mean(img, win_y, win_x)\n", + " mean_sq = _box_mean(img**2, win_y, win_x)\n", + " var = mean_sq - mean**2\n", + " noise_var = np.nanmedian(var)\n", + " w = var / (var + noise_var + 1e-8)\n", + " return mean + w * (img - mean)\n", + "\n", + "\n", + "def goldstein(\n", + " phase: NDArray[np.complex64] | NDArray[np.float64], alpha: float, psize: int = 32\n", + ") -> np.ndarray:\n", + " \"\"\"Apply the Goldstein adaptive filter to the given data.\"\"\"\n", + "\n", + " def apply_pspec(data: NDArray[np.complex64]) -> np.ndarray:\n", + " if alpha < 0:\n", + " raise ValueError(f\"alpha must be >= 0, got {alpha = }\")\n", + " weight = np.power(np.abs(data) ** 2, alpha / 2)\n", + " data = weight * data\n", + " return data\n", + "\n", + " def make_weight(nxp: int, nyp: int) -> np.ndarray:\n", + " wx = 1.0 - np.abs(np.arange(nxp // 2) - (nxp / 2.0 - 1.0)) / (nxp / 2.0 - 1.0)\n", + " wy = 1.0 - np.abs(np.arange(nyp // 2) - (nyp / 2.0 - 1.0)) / (nyp / 2.0 - 1.0)\n", + " quadrant = np.outer(wy, wx)\n", + " weight = np.block(\n", + " [\n", + " [quadrant, np.flip(quadrant, axis=1)],\n", + " [np.flip(quadrant, axis=0), np.flip(np.flip(quadrant, axis=0), axis=1)],\n", + " ]\n", + " )\n", + " return weight\n", + "\n", + " def patch_goldstein_filter(\n", + " data: NDArray[np.complex64], weight: NDArray[np.float64], psize: int\n", + " ) -> np.ndarray:\n", + " data = np.fft.fft2(data, s=(psize, psize))\n", + " data = apply_pspec(data)\n", + " data = np.fft.ifft2(data, s=(psize, psize))\n", + " return weight * data\n", + "\n", + " def apply_goldstein_filter(data: NDArray[np.complex64]) -> np.ndarray:\n", + " empty_mask = np.isnan(data) | (data == 0)\n", + " if np.all(empty_mask):\n", + " return data\n", + "\n", + " nrows, ncols = data.shape\n", + " step = psize // 2\n", + "\n", + " pad_top = step\n", + " pad_left = step\n", + " pad_bottom = step + (step - (nrows % step)) % step\n", + " pad_right = step + (step - (ncols % step)) % step\n", + " data_padded = np.pad(\n", + " data, ((pad_top, pad_bottom), (pad_left, pad_right)), mode=\"reflect\"\n", + " )\n", + "\n", + " out = np.zeros(data_padded.shape, dtype=np.complex64)\n", + " weight_sum = np.zeros(data_padded.shape, dtype=np.float64)\n", + " weight_matrix = make_weight(psize, psize)\n", + "\n", + " padded_rows, padded_cols = data_padded.shape\n", + " for i in range(0, padded_rows - psize + 1, step):\n", + " for j in range(0, padded_cols - psize + 1, step):\n", + " data_window = data_padded[i : i + psize, j : j + psize]\n", + " filtered_window = patch_goldstein_filter(\n", + " data_window, weight_matrix, psize\n", + " )\n", + " out[i : i + psize, j : j + psize] += filtered_window\n", + " weight_sum[i : i + psize, j : j + psize] += weight_matrix\n", + "\n", + " valid = weight_sum > 0\n", + " out[valid] /= weight_sum[valid]\n", + "\n", + " out = out[pad_top : pad_top + nrows, pad_left : pad_left + ncols]\n", + " out[empty_mask] = 0\n", + " return out\n", + "\n", + " if np.iscomplexobj(phase):\n", + " return apply_goldstein_filter(phase)\n", + " else:\n", + " return apply_goldstein_filter(np.exp(1j * phase))\n", + "\n", + "\n", + "def calc_ifg_coh_filtered(reference, secondary, goldstein_alpha=0.5, coh_win_y=5, coh_win_x=5, looks_y=1, looks_x=1, coh_method=\"standard\", ifg_apply_filter=True, coh_apply_lee=False, coh_kernel=\"boxcar\", coh_kernel_num_conv=5):\n", + " reference = _multilook(reference, looks_y, looks_x)\n", + " secondary = _multilook(secondary, looks_y, looks_x)\n", + " phase = reference * np.conjugate(secondary)\n", + " amp = np.sqrt((reference * np.conjugate(reference)) * (secondary * np.conjugate(secondary)))\n", + " nan_mask = np.isnan(phase)\n", + " ifg_cpx = np.exp(1j * np.nan_to_num(np.angle(phase/amp)))\n", + " if ifg_apply_filter:\n", + " ifg_cpx_f = goldstein(ifg_cpx, alpha=goldstein_alpha, psize=32)\n", + " else:\n", + " ifg_cpx_f = ifg_cpx\n", + " ifg = np.angle(ifg_cpx_f)\n", + " ifg[nan_mask] = np.nan\n", + "\n", + " ifg_cpx_used = ifg_cpx # coherence never uses filtered IFG\n", + "\n", + " if coh_kernel == 'weighted':\n", + " k = get_kernel(coh_win_y, coh_win_x, coh_kernel_num_conv)\n", + " elif coh_kernel == 'boxcar':\n", + " k = None\n", + " else:\n", + " raise ValueError(\"coh_kernel must be 'boxcar' or 'weighted'\")\n", + " if coh_method == \"phase_only\":\n", + " if coh_kernel == 'weighted':\n", + " coh = np.abs(_weighted_mean(ifg_cpx_used, k))\n", + " else:\n", + " coh = np.abs(_box_mean(ifg_cpx_used, coh_win_y, coh_win_x))\n", + " elif coh_method == \"standard\":\n", + " if coh_kernel == 'weighted':\n", + " num = np.abs(_weighted_mean(phase, k))\n", + " den = np.sqrt(_weighted_mean(np.abs(reference)**2, k) * _weighted_mean(np.abs(secondary)**2, k))\n", + " else:\n", + " num = np.abs(_box_mean(phase, coh_win_y, coh_win_x))\n", + " den = np.sqrt(_box_mean(np.abs(reference)**2, coh_win_y, coh_win_x) * _box_mean(np.abs(secondary)**2, coh_win_y, coh_win_x))\n", + " coh = np.where(den > 0, num / den, 0)\n", + " else:\n", + " raise ValueError(\"coh_method must be 'standard' or 'phase_only'\")\n", + " coh = np.clip(coh, 0, 1)\n", + " if coh_apply_lee:\n", + " coh = lee_filter(coh, win_y=coh_win_y, win_x=coh_win_x)\n", + " coh = np.clip(coh, 0, 1)\n", + " zero_mask = phase == 0\n", + " coh[nan_mask] = np.nan\n", + " coh[zero_mask] = 0\n", + " return ifg, coh, amp\n", + "\n", + "\n", + "def calc_ifg_coh(reference, secondary, goldstein_alpha=0.5, coh_win_y=5, coh_win_x=5, looks_y=1, looks_x=1, coh_method=\"standard\", ifg_apply_filter=True, coh_apply_lee=False, coh_kernel=\"boxcar\", coh_kernel_num_conv=5):\n", + " return calc_ifg_coh_filtered(reference, secondary, goldstein_alpha=goldstein_alpha, coh_win_y=coh_win_y, coh_win_x=coh_win_x, looks_y=looks_y, looks_x=looks_x, coh_method=coh_method, ifg_apply_filter=ifg_apply_filter, coh_apply_lee=coh_apply_lee, coh_kernel=coh_kernel, coh_kernel_num_conv=coh_kernel_num_conv)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2775556c", + "metadata": {}, + "outputs": [], + "source": [ + "## For each date-pair, calculate the ifg, coh. Save the results as GeoTiffs.\n", + "for ref_idx, sec_idx in pair_indices:\n", + " ref_date = cslc_dates.iloc[ref_idx].values[0]\n", + " sec_date = cslc_dates.iloc[sec_idx].values[0]\n", + " print(f\"Reference: {ref_date} Secondary: {sec_date}\")\n", + "\n", + " # Calculate ifg, coh, amp\n", + " if \"calc_ifg_coh_filtered\" not in globals():\n", + " raise RuntimeError(\"calc_ifg_coh_filtered is not defined. Run the filter definition cell first.\")\n", + " looks_y, looks_x = MULTILOOK\n", + "\n", + " # Save each interferogram as GeoTiff (no per-burst plotting)\n", + " transform = from_origin(xcoor_stack[ref_idx][0], ycoor_stack[ref_idx][0], dx, np.abs(dy))" + ] + }, + { + "cell_type": "markdown", + "id": "d57d1a71", + "metadata": {}, + "source": [ + "
\n", + "8. Merge the burst-wise interferograms and coherence and save as GeoTiff.\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e98baf1f", + "metadata": {}, + "outputs": [], + "source": [ + "def custom_merge(old_data, new_data, old_nodata, new_nodata, **kwargs):\n", + " # Feather overlaps to reduce burst seams\n", + " if MERGE_BLEND_OVERLAP:\n", + " overlap = np.logical_and(~old_nodata, ~new_nodata)\n", + " if np.any(overlap):\n", + " # distance to nodata inside each valid mask\n", + " dist_old = distance_transform_edt(~old_nodata)\n", + " dist_new = distance_transform_edt(~new_nodata)\n", + " w_new = dist_new / (dist_new + dist_old + MERGE_BLEND_EPS)\n", + " w_new = np.clip(w_new, 0, 1)\n", + " blended = old_data * (1 - w_new) + new_data * w_new\n", + " old_data[overlap] = blended[overlap]\n", + " # fill empty pixels\n", + " mask = np.logical_and(old_nodata, ~new_nodata)\n", + " old_data[mask] = new_data[mask]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "897e44dd", + "metadata": {}, + "outputs": [], + "source": [ + "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", + "# Merge burst-wise interferograms per date-pair\n", + "\n", + "# Group pair indices by date tag\n", + "pairs_by_tag = {}\n", + "for r, s in pair_indices:\n", + " ref_date = cslc_dates.iloc[r].values[0]\n", + " sec_date = cslc_dates.iloc[s].values[0]\n", + " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", + " pairs_by_tag.setdefault(tag, []).append((r, s))\n", + "\n", + "for tag, pairs in pairs_by_tag.items():\n", + " srcs = []\n", + " for r, s in pairs:\n", + " looks_y, looks_x = MULTILOOK\n", + " ifg, coh, amp = calc_ifg_coh(\n", + " cslc_stack[r], cslc_stack[s],\n", + " goldstein_alpha=GOLDSTEIN_ALPHA, coh_win_y=COH_WIN_Y, coh_win_x=COH_WIN_X,\n", + " looks_y=looks_y, looks_x=looks_x,\n", + " coh_method=COH_METHOD,\n", + " ifg_apply_filter=IFG_APPLY_FILTER,\n", + " coh_apply_lee=COH_APPLY_LEE,\n", + " coh_kernel=COH_KERNEL, coh_kernel_num_conv=COH_KERNEL_NUM_CONV,\n", + " )\n", + " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", + " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", + " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", + " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", + " mem = MemoryFile()\n", + " ds = mem.open(\n", + " driver='GTiff', height=ifg.shape[0], width=ifg.shape[1], count=1, dtype=ifg.dtype,\n", + " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " )\n", + " ds.write(ifg, 1)\n", + " srcs.append(ds)\n", + " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", + " dest, output_transform = _trim_nan_border(dest, output_transform)\n", + " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", + " if mask is not None:\n", + " dest[0] = _apply_water_mask(dest[0], mask)\n", + " out_meta = srcs[0].meta.copy()\n", + " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", + " out_path = f\"{savedir}/tifs/merged_ifg_{tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path)):\n", + " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", + " dest1.write(dest)\n", + " if SAVE_WGS84:\n", + " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_ifg_WGS84_{tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_wgs84)):\n", + " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", + " for ds in srcs:\n", + " ds.close()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7c41924c", + "metadata": {}, + "outputs": [], + "source": [ + "# Merge burst-wise coherence per date-pair\n", + "\n", + "# Group pair indices by date tag\n", + "pairs_by_tag = {}\n", + "for r, s in pair_indices:\n", + " ref_date = cslc_dates.iloc[r].values[0]\n", + " sec_date = cslc_dates.iloc[s].values[0]\n", + " tag = f\"{ref_date.strftime('%Y%m%d')}-{sec_date.strftime('%Y%m%d')}\"\n", + " pairs_by_tag.setdefault(tag, []).append((r, s))\n", + "\n", + "for tag, pairs in pairs_by_tag.items():\n", + " srcs = []\n", + " for r, s in pairs:\n", + " looks_y, looks_x = MULTILOOK\n", + " ifg, coh, amp = calc_ifg_coh(\n", + " cslc_stack[r], cslc_stack[s],\n", + " goldstein_alpha=GOLDSTEIN_ALPHA, coh_win_y=COH_WIN_Y, coh_win_x=COH_WIN_X,\n", + " looks_y=looks_y, looks_x=looks_x,\n", + " coh_method=COH_METHOD,\n", + " ifg_apply_filter=IFG_APPLY_FILTER,\n", + " coh_apply_lee=COH_APPLY_LEE,\n", + " coh_kernel=COH_KERNEL, coh_kernel_num_conv=COH_KERNEL_NUM_CONV,\n", + " )\n", + " dy_signed = (ycoor_stack[r][1] - ycoor_stack[r][0]) if len(ycoor_stack[r]) > 1 else -dy\n", + " x0 = xcoor_stack[r][0] + (looks_x - 1) * dx / 2\n", + " y0 = ycoor_stack[r][0] + (looks_y - 1) * dy_signed / 2\n", + " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", + " mem = MemoryFile()\n", + " ds = mem.open(\n", + " driver='GTiff', height=coh.shape[0], width=coh.shape[1], count=1, dtype=coh.dtype,\n", + " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " )\n", + " ds.write(coh, 1)\n", + " srcs.append(ds)\n", + " dest, output_transform = merge.merge(srcs, method=custom_merge)\n", + " dest, output_transform = _trim_nan_border(dest, output_transform)\n", + " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " mask = _load_water_mask_match(WATER_MASK_PATH, dest.shape[1:], output_transform, CRS.from_epsg(epsg))\n", + " if mask is not None:\n", + " dest[0] = _apply_water_mask(dest[0], mask)\n", + " out_meta = srcs[0].meta.copy()\n", + " out_meta.update({\"driver\": \"GTiff\", \"height\": dest.shape[1], \"width\": dest.shape[2], \"transform\": output_transform})\n", + " out_path = f\"{savedir}/tifs/merged_coh_{tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path)):\n", + " with rasterio.open(out_path, \"w\", **out_meta) as dest1:\n", + " dest1.write(dest)\n", + " if SAVE_WGS84:\n", + " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_coh_WGS84_{tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_wgs84)):\n", + " _save_mosaic_utm_to_wgs84(out_path_wgs84, dest, output_transform, epsg)\n", + " for ds in srcs:\n", + " ds.close()\n" + ] + }, + { + "cell_type": "markdown", + "id": "858ea831", + "metadata": {}, + "source": [ + "
\n", + "9. Read the merged GeoTiff and Visualize using `matplotlib`\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9cb2a341", + "metadata": {}, + "outputs": [], + "source": [ + "# Read merged IFG/COH files and plot paired grids\n", + "\n", + "\n", + "# Output dir for per-pair PNGs\n", + "pair_png_dir = f\"{savedir}/pairs_png\"\n", + "os.makedirs(pair_png_dir, exist_ok=True)\n", + "\n", + "ifg_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_ifg_*.tif\"))\n", + "coh_norm = mcolors.PowerNorm(gamma=COH_NORM_GAMMA, vmin=0, vmax=1) if COH_USE_GAMMA_NORM else None\n", + "coh_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\"))\n", + "\n", + "ifg_map = {p.split('merged_ifg_')[-1].replace('.tif',''): p for p in ifg_paths}\n", + "coh_map = {p.split('merged_coh_')[-1].replace('.tif',''): p for p in coh_paths}\n", + "\n", + "\n", + "\n", + "def _prep_da(path):\n", + " da = rioxarray.open_rasterio(path)[0]\n", + " data = da.values\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if mask.any():\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " # Trim NaN borders so edges don't show padding\n", + " da = da.isel(y=slice(r0, r1), x=slice(c0, c1))\n", + " return da\n", + "\n", + "pair_tags = sorted(set(ifg_map).intersection(coh_map))\n", + "# Filter pairs by current date range\n", + "date_start_day = dateStart.date()\n", + "date_end_day = dateEnd.date()\n", + "pair_tags = [t for t in pair_tags if (date_start_day <= pd.to_datetime(t.split('-')[0], format='%Y%m%d').date() <= date_end_day and date_start_day <= pd.to_datetime(t.split('-')[1], format='%Y%m%d').date() <= date_end_day)]\n", + "\n", + "if not pair_tags:\n", + " print('No matching IFG/COH pairs found')\n", + "else:\n", + " # Save ALL pairs as PNGs\n", + " for tag in pair_tags:\n", + " fig, axes = plt.subplots(1, 2, figsize=(10, 4), constrained_layout=True)\n", + " ax_ifg, ax_coh = axes\n", + "\n", + " # IFG\n", + " merged_ifg = _prep_da(ifg_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", + " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", + " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", + " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", + " ax_ifg.set_xticks([])\n", + " ax_ifg.set_yticks([])\n", + " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", + "\n", + " # COH\n", + " merged_coh = _prep_da(coh_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", + " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, norm=coh_norm, vmin=None if COH_USE_GAMMA_NORM else 0, vmax=None if COH_USE_GAMMA_NORM else 1.0)\n", + " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", + " ax_coh.set_xticks([])\n", + " ax_coh.set_yticks([])\n", + " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')\n", + "\n", + " out_png = os.path.join(pair_png_dir, f\"pair_{tag}.png\")\n", + " fig.savefig(out_png, dpi=150)\n", + " plt.close(fig)\n", + "\n", + " # Display only last 5 pairs in notebook\n", + " display_tags = pair_tags[-5:]\n", + " n = len(display_tags)\n", + " ncols = 2\n", + " nrows = math.ceil(n / 1) # one pair per row\n", + " fig, axes = plt.subplots(nrows, ncols, figsize=(6*ncols, 3*nrows), constrained_layout=True)\n", + " if nrows == 1:\n", + " axes = [axes]\n", + "\n", + " for i, tag in enumerate(display_tags):\n", + " ax_ifg, ax_coh = axes[i]\n", + "\n", + " # IFG\n", + " merged_ifg = _prep_da(ifg_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_ifg.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " colored_ifg = colorize(merged_ifg, 'twilight_shifted', -np.pi, np.pi)\n", + " colored_ifg = np.ma.masked_invalid(colored_ifg)\n", + " im_ifg = ax_ifg.imshow(colored_ifg, cmap='twilight_shifted', interpolation='none', origin='upper', extent=bbox, vmin=-np.pi, vmax=np.pi)\n", + " ax_ifg.set_title(f\"IFG_{tag}\", fontsize=10)\n", + " ax_ifg.set_xticks([])\n", + " ax_ifg.set_yticks([])\n", + " fig.colorbar(im_ifg, ax=ax_ifg, orientation='vertical', fraction=0.046, pad=0.02, label='Wrapped phase (rad)')\n", + "\n", + " # COH\n", + " merged_coh = _prep_da(coh_map[tag])\n", + " minlon, minlat, maxlon, maxlat = merged_coh.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " coh_vals = np.ma.masked_invalid(merged_coh.values)\n", + " im_coh = ax_coh.imshow(coh_vals, cmap='gray', interpolation='none', origin='upper', extent=bbox, norm=coh_norm, vmin=None if COH_USE_GAMMA_NORM else 0, vmax=None if COH_USE_GAMMA_NORM else 1.0)\n", + " ax_coh.set_title(f\"COH_{tag}\", fontsize=10)\n", + " ax_coh.set_xticks([])\n", + " ax_coh.set_yticks([])\n", + " fig.colorbar(im_coh, ax=ax_coh, orientation='vertical', fraction=0.046, pad=0.02, label='Coherence')" + ] + }, + { + "cell_type": "markdown", + "id": "ea1e4f24", + "metadata": {}, + "source": [ + "
\n", + "9.5 Merge and plot backscatter (dB) mosaics (per date)\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7f478f4", + "metadata": {}, + "outputs": [], + "source": [ + "def _load_water_mask_match(mask_path, shape, transform, crs):\n", + " with rasterio.open(mask_path) as src:\n", + " src_mask = src.read(1)\n", + " if src.crs == crs and src.transform == transform and src_mask.shape == shape:\n", + " return src_mask\n", + " dst = np.zeros(shape, dtype=src_mask.dtype)\n", + " reproject(\n", + " source=src_mask,\n", + " destination=dst,\n", + " src_transform=src.transform,\n", + " src_crs=src.crs,\n", + " dst_transform=transform,\n", + " dst_crs=crs,\n", + " resampling=Resampling.nearest,\n", + " )\n", + " return dst\n", + "\n", + "def _apply_water_mask(arr, mask):\n", + " # mask: 1 = keep land, 0 = water\n", + " return np.where(mask == 0, np.nan, arr)\n", + "\n", + "\n", + "def _trim_nan_border(arr, transform):\n", + " data = arr[0] if arr.ndim == 3 else arr\n", + " mask = np.isfinite(data) & (data != 0)\n", + " if not mask.any():\n", + " return arr, transform\n", + " rows = np.where(mask.any(axis=1))[0]\n", + " cols = np.where(mask.any(axis=0))[0]\n", + " r0, r1 = rows[0], rows[-1] + 1\n", + " c0, c1 = cols[0], cols[-1] + 1\n", + " data = data[r0:r1, c0:c1]\n", + " if arr.ndim == 3:\n", + " arr = data[None, ...]\n", + " else:\n", + " arr = data\n", + " new_transform = transform * Affine.translation(c0, r0)\n", + " return arr, new_transform\n", + "\n", + "# Build per-date backscatter (dB) mosaics directly from subset H5 (no per-burst GeoTIFFs)\n", + "os.makedirs(f\"{savedir}/tifs\", exist_ok=True)\n", + "\n", + "date_tags = sorted(cslc_df.startTime.astype(str).str.replace('-', '').unique())\n", + "\n", + "def _save_mosaic_utm(out_path, mosaic, transform, epsg):\n", + " with rasterio.open(\n", + " out_path, \"w\", driver=\"GTiff\", height=mosaic.shape[0], width=mosaic.shape[1],\n", + " count=1, dtype=mosaic.dtype, crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " ) as dst_ds:\n", + " dst_ds.write(mosaic, 1)\n", + "\n", + "def _save_mosaic_utm_to_wgs84(out_path, mosaic, transform, epsg):\n", + " dst_crs = \"EPSG:4326\"\n", + " src_crs = CRS.from_epsg(epsg)\n", + " height, width = mosaic.shape\n", + " dst_transform, dst_width, dst_height = calculate_default_transform(\n", + " src_crs, dst_crs, width, height, *rasterio.transform.array_bounds(height, width, transform)\n", + " )\n", + " dst = np.empty((dst_height, dst_width), dtype=mosaic.dtype)\n", + " reproject(\n", + " source=mosaic,\n", + " destination=dst,\n", + " src_transform=transform,\n", + " src_crs=src_crs,\n", + " dst_transform=dst_transform,\n", + " dst_crs=dst_crs,\n", + " resampling=Resampling.bilinear,\n", + " )\n", + " with rasterio.open(\n", + " out_path, \"w\", driver=\"GTiff\", height=dst_height, width=dst_width, count=1,\n", + " dtype=dst.dtype, crs=dst_crs, transform=dst_transform, nodata=np.nan\n", + " ) as dst_ds:\n", + " dst_ds.write(dst, 1)\n", + "\n", + "# Mosaicking helper (in memory)\n", + "def _mosaic_arrays(arrays, transforms, epsg):\n", + " # Convert arrays to in-memory rasterio datasets via MemoryFile\n", + " srcs = []\n", + " for arr, transform in zip(arrays, transforms):\n", + " mem = MemoryFile()\n", + " ds = mem.open(\n", + " driver='GTiff', height=arr.shape[0], width=arr.shape[1], count=1, dtype=arr.dtype,\n", + " crs=CRS.from_epsg(epsg), transform=transform, nodata=np.nan\n", + " )\n", + " ds.write(arr, 1)\n", + " srcs.append(ds)\n", + " dest, out_transform = merge.merge(srcs, method=custom_merge)\n", + " for ds in srcs:\n", + " ds.close()\n", + " return dest[0], out_transform\n", + "\n", + "\n", + "def _reproject_calibration_to_data_grid(sigma, cal_x, cal_y, cal_dx, cal_dy, xcoor, ycoor, dx, dy, epsg):\n", + " cal_dy_signed = (cal_y[1] - cal_y[0]) if len(cal_y) > 1 else -cal_dy\n", + " data_dy_signed = (ycoor[1] - ycoor[0]) if len(ycoor) > 1 else -dy\n", + " if sigma.shape == (len(ycoor), len(xcoor)) and cal_dx == dx and abs(cal_dy_signed) == abs(data_dy_signed):\n", + " return sigma\n", + " src_transform = from_origin(cal_x[0], cal_y[0], cal_dx, abs(cal_dy_signed))\n", + " dst_transform = from_origin(xcoor[0], ycoor[0], dx, abs(data_dy_signed))\n", + " dst = np.empty((len(ycoor), len(xcoor)), dtype=sigma.dtype)\n", + " reproject(\n", + " source=sigma,\n", + " destination=dst,\n", + " src_transform=src_transform,\n", + " src_crs=CRS.from_epsg(epsg),\n", + " dst_transform=dst_transform,\n", + " dst_crs=CRS.from_epsg(epsg),\n", + " resampling=Resampling.bilinear,\n", + " )\n", + " return dst\n", + "\n", + "looks_y, looks_x = MULTILOOK\n", + "# Backscatter calibration mode tag for filenames\n", + "bsc_mode_tag = CALIBRATION_MODE\n", + "beta_naught = None\n", + "for date_tag in date_tags:\n", + " # collect subset H5 files for this date\n", + " rows = cslc_df[cslc_df.startTime.astype(str).str.replace('-', '') == date_tag]\n", + " arrays = []\n", + " transforms = []\n", + " epsg = None\n", + " for fileID in rows.fileID:\n", + " subset_path = f\"{savedir}/subset_cslc/{fileID}.h5\"\n", + " with h5py.File(subset_path, 'r') as h5:\n", + " cslc = h5['/data/VV'][:]\n", + " xcoor = h5['/data/x_coordinates'][:]\n", + " ycoor = h5['/data/y_coordinates'][:]\n", + " dx = int(h5['/data/x_spacing'][()])\n", + " dy = int(h5['/data/y_spacing'][()])\n", + " epsg = int(h5['/data/projection'][()])\n", + " sigma = h5['/metadata/calibration_information/sigma_naught'][:]\n", + " cal_x = h5['/metadata/calibration_information/x_coordinates'][:]\n", + " cal_y = h5['/metadata/calibration_information/y_coordinates'][:]\n", + " cal_dx = int(h5['/metadata/calibration_information/x_spacing'][()])\n", + " cal_dy = int(h5['/metadata/calibration_information/y_spacing'][()])\n", + " beta_naught = h5['/metadata/calibration_information/beta_naught'][()]\n", + " power_ml = _multilook(np.abs(cslc)**2, looks_y, looks_x)\n", + " if CALIBRATION_MODE == 'sigma0':\n", + " sigma_on_data = _reproject_calibration_to_data_grid(\n", + " sigma, cal_x, cal_y, cal_dx, cal_dy, xcoor, ycoor, dx, dy, epsg\n", + " )\n", + " sigma_ml = _multilook(sigma_on_data, looks_y, looks_x)\n", + " sigma_ml = np.where(sigma_ml <= 0, np.nan, sigma_ml)\n", + " sigma_factor = sigma_ml**2 if CALIBRATION_FACTOR_IS_AMPLITUDE else sigma_ml\n", + " bsc = 10*np.log10(power_ml / sigma_factor)\n", + " elif CALIBRATION_MODE == 'beta0':\n", + " beta = beta_naught\n", + " if beta is None or beta <= 0:\n", + " bsc = np.full(power_ml.shape, np.nan, dtype=power_ml.dtype)\n", + " else:\n", + " beta_factor = beta**2 if CALIBRATION_FACTOR_IS_AMPLITUDE else beta\n", + " bsc = 10*np.log10(power_ml / beta_factor)\n", + " else:\n", + " raise ValueError(\"CALIBRATION_MODE must be 'sigma0' or 'beta0'\")\n", + " dy_signed = (ycoor[1] - ycoor[0]) if len(ycoor) > 1 else -dy\n", + " x0 = xcoor[0] + (looks_x - 1) * dx / 2\n", + " y0 = ycoor[0] + (looks_y - 1) * dy_signed / 2\n", + " transform = from_origin(x0, y0, dx*looks_x, np.abs(dy_signed)*looks_y)\n", + " arrays.append(bsc)\n", + " transforms.append(transform)\n", + "\n", + " if not arrays:\n", + " continue\n", + " mosaic, out_transform = _mosaic_arrays(arrays, transforms, epsg)\n", + " out_path_utm = f\"{savedir}/tifs/merged_bsc_{bsc_mode_tag}_{date_tag}.tif\"\n", + " mosaic, out_transform = _trim_nan_border(mosaic, out_transform)\n", + " if APPLY_WATER_MASK and WATER_MASK_PATH:\n", + " mask = _load_water_mask_match(WATER_MASK_PATH, mosaic.shape, out_transform, CRS.from_epsg(epsg))\n", + " mosaic = _apply_water_mask(mosaic, mask)\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_utm)):\n", + " _save_mosaic_utm(out_path_utm, mosaic, out_transform, epsg)\n", + " if SAVE_WGS84:\n", + " out_path_wgs84 = f\"{savedir}/tifs/WGS84/merged_bsc_{bsc_mode_tag}_WGS84_{date_tag}.tif\"\n", + " if (not SKIP_EXISTING_TIFS) or (not os.path.exists(out_path_wgs84)):\n", + " _save_mosaic_utm_to_wgs84(out_path_wgs84, mosaic, out_transform, epsg)\n", + "\n", + "# Plot merged backscatter (dB) mosaics in a grid (native CRS from saved GeoTIFFs)\n", + "\n", + "# Output dir for backscatter PNGs\n", + "bsc_png_dir = f\"{savedir}/bsc_png\"\n", + "os.makedirs(bsc_png_dir, exist_ok=True)\n", + "\n", + "paths = sorted(glob.glob(f\"{savedir}/tifs/merged_bsc_{bsc_mode_tag}_*.tif\"))\n", + "paths = [p for p in paths if 'WGS84' not in p]\n", + "all_vals = []\n", + "for p in paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " all_vals.append(da.values.ravel())\n", + "if all_vals:\n", + " all_vals = np.concatenate(all_vals)\n", + " gmin = np.nanpercentile(all_vals, 2)\n", + " gmax = np.nanpercentile(all_vals, 90)\n", + "else:\n", + " gmin, gmax = None, None\n", + "n = len(paths)\n", + "if n == 0:\n", + " print('No merged backscatter files found')\n", + "else:\n", + " # Save ALL backscatter PNGs\n", + " for path in paths:\n", + " src = rioxarray.open_rasterio(path)\n", + " bsc = src[0]\n", + " minlon, minlat, maxlon, maxlat = bsc.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " fig, ax = plt.subplots(figsize=(5,4))\n", + " im = ax.imshow(bsc.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", + " tag = path.split(f'merged_bsc_{bsc_mode_tag}_')[-1].replace('.tif','')\n", + " ax.set_title(f\"{tag}\", fontsize=10)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " fig.colorbar(im, ax=ax, orientation='vertical', fraction=0.046, pad=0.02, label=f\"{bsc_mode_tag} (dB)\")\n", + " out_png = os.path.join(bsc_png_dir, f\"bsc_{tag}.png\")\n", + " fig.savefig(out_png, dpi=150)\n", + " plt.close(fig)\n", + "\n", + " # Show only last 5 in notebook\n", + " display_paths = paths[-5:]\n", + " n = len(display_paths)\n", + " ncols = 3\n", + " nrows = math.ceil(n / ncols)\n", + " fig, axes = plt.subplots(nrows, ncols, figsize=(4*ncols, 3*nrows), constrained_layout=True)\n", + " axes = axes.ravel()\n", + " for ax, path in zip(axes, display_paths):\n", + " src = rioxarray.open_rasterio(path)\n", + " bsc = src[0]\n", + " minlon, minlat, maxlon, maxlat = bsc.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " im = ax.imshow(bsc.values, cmap='gray', interpolation='none', origin='upper', extent=bbox, vmin=gmin, vmax=gmax)\n", + " tag = path.split(f'merged_bsc_{bsc_mode_tag}_')[-1].replace('.tif','')\n", + " ax.set_title(f\"{tag}\", fontsize=10)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " for ax in axes[n:]:\n", + " ax.axis('off')\n", + " fig.colorbar(im, ax=axes.tolist(), orientation='vertical', fraction=0.02, pad=0.02, label=f\"{bsc_mode_tag} (dB)\")" + ] + }, + { + "cell_type": "markdown", + "id": "df66a59f", + "metadata": {}, + "source": [ + "
\n", + "10. Monthly mean coherence calendar (per year)\n", + "\n", + "This calendar bins each pair into a month using the midpoint date between the reference and secondary scenes.\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c31a947a", + "metadata": {}, + "outputs": [], + "source": [ + "# # CALENDAR_CMAP = ROCKET_CMAP # previous\n", + "CALENDAR_CMAP = cmc.bilbao\n", + "# CALENDAR_CMAP = cmc.bilbao\n", + "\n", + "# Build an index of merged coherence files by midpoint year-month\n", + "records = []\n", + "for path in sorted(glob.glob(f\"{savedir}/tifs/merged_coh_*.tif\")):\n", + " tag = path.split('merged_coh_')[-1].replace('.tif','')\n", + " try:\n", + " ref_str, sec_str = tag.split('-')\n", + " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", + " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", + " mid_date = ref_date + (sec_date - ref_date) / 2\n", + " except Exception:\n", + " continue\n", + " records.append({\"path\": path, \"mid_date\": mid_date})\n", + "\n", + "df_paths = pd.DataFrame(records)\n", + "if df_paths.empty:\n", + " print('No merged coherence files found for calendar')\n", + " raise SystemExit\n", + "\n", + "# Apply current date range using midpoint date\n", + "date_start_day = dateStart.date()\n", + "date_end_day = dateEnd.date()\n", + "df_paths = df_paths[(df_paths['mid_date'].dt.date >= date_start_day) & (df_paths['mid_date'].dt.date <= date_end_day)]\n", + "\n", + "# Calendar year labeling\n", + "if USE_WATER_YEAR:\n", + " # Water year starts Oct (10) and ends Sep (9)\n", + " df_paths['year'] = df_paths['mid_date'].dt.year + (df_paths['mid_date'].dt.month >= 10).astype(int)\n", + " month_order = [10,11,12,1,2,3,4,5,6,7,8,9]\n", + " month_labels = ['Oct','Nov','Dec','Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep']\n", + "else:\n", + " df_paths['year'] = df_paths['mid_date'].dt.year\n", + " month_order = list(range(1,13))\n", + " month_labels = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']\n", + "\n", + "years = sorted(df_paths['year'].unique())\n", + "\n", + "# Contrast stretch for low coherence (red = low)\n", + "norm = mcolors.PowerNorm(gamma=0.3, vmin=0, vmax=1)\n", + "\n", + "# One row per year, 12 columns\n", + "fig, axes = plt.subplots(len(years), 12, figsize=(24, 2.5*len(years)), constrained_layout=True)\n", + "if len(years) == 1:\n", + " axes = np.array([axes])\n", + "\n", + "for row_idx, y in enumerate(years):\n", + " # pick a template for consistent grid within the year (first available file)\n", + " year_paths = df_paths[df_paths['year'] == y]['path'].tolist()\n", + " if not year_paths:\n", + " continue\n", + " template = rioxarray.open_rasterio(year_paths[0])\n", + "\n", + " for col_idx, m in enumerate(month_order):\n", + " ax = axes[row_idx, col_idx]\n", + " month_paths = df_paths[(df_paths['year'] == y) & (df_paths['mid_date'].dt.month == m)]['path'].tolist()\n", + " if USE_WATER_YEAR:\n", + " year_for_month = y - 1 if m in (10, 11, 12) else y\n", + " else:\n", + " year_for_month = y\n", + " title = f\"{month_labels[col_idx]} {year_for_month}\"\n", + " if not month_paths:\n", + " ax.set_title(title, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " # keep a visible box for empty months\n", + " for spine in ax.spines.values():\n", + " spine.set_visible(True)\n", + " spine.set_linewidth(0.8)\n", + " spine.set_color('0.5')\n", + " continue\n", + " stacks = []\n", + " for p in month_paths:\n", + " da = rioxarray.open_rasterio(p)\n", + " da = da.rio.reproject_match(template)\n", + " stacks.append(da)\n", + " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", + " minlon, minlat, maxlon, maxlat = da_month.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " im = ax.imshow(da_month.values.squeeze(), cmap=CALENDAR_CMAP, norm=norm, origin='upper', extent=bbox, interpolation='none')\n", + " ax.set_title(title, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " for spine in ax.spines.values():\n", + " spine.set_visible(True)\n", + " spine.set_linewidth(0.8)\n", + " spine.set_color('0.5')\n", + " # left-side year label\n", + " if USE_WATER_YEAR:\n", + " label = f\"WY {y}\"\n", + " else:\n", + " label = str(y)\n", + " axes[row_idx, 0].set_ylabel(label, rotation=90, labelpad=6, fontsize=9)\n", + " axes[row_idx, 0].yaxis.set_label_coords(-0.06, 0.5)\n", + "\n", + "fig.colorbar(im, ax=axes, orientation='vertical', fraction=0.02, pad=0.02, label='Mean coherence')" + ] + }, + { + "cell_type": "markdown", + "id": "169216f0", + "metadata": {}, + "source": [ + "
\n", + "11. Create GIF animations (Backscatter (dB) + Coherence)\n", + "\n", + "
" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "89109813", + "metadata": {}, + "outputs": [], + "source": [ + "# GIF helper functions\n", + "\n", + "def _global_bounds(paths):\n", + " bounds = []\n", + " for p in paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " minx, miny, maxx, maxy = da.rio.bounds()\n", + " bounds.append((minx, miny, maxx, maxy))\n", + " minx = min(b[0] for b in bounds)\n", + " miny = min(b[1] for b in bounds)\n", + " maxx = max(b[2] for b in bounds)\n", + " maxy = max(b[3] for b in bounds)\n", + " return [minx, maxx, miny, maxy]\n", + "\n", + "def _extract_date_from_tag(tag):\n", + " m = re.search(r\"(\\d{8})\", tag)\n", + " if not m:\n", + " return None\n", + " try:\n", + " return pd.to_datetime(m.group(1), format=\"%Y%m%d\")\n", + " except Exception:\n", + " return None\n", + "\n", + "def _format_title(tag, date=None):\n", + " if date is not None:\n", + " return date.strftime('%Y-%m-%d')\n", + " m = re.findall(r\"(\\d{8})\", tag)\n", + " if len(m) >= 2:\n", + " try:\n", + " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\")\n", + " d1 = pd.to_datetime(m[1], format=\"%Y%m%d\")\n", + " return f\"{d0.strftime('%Y-%m-%d')} to {d1.strftime('%Y-%m-%d')}\"\n", + " except Exception:\n", + " pass\n", + " if len(m) == 1:\n", + " try:\n", + " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\")\n", + " return d0.strftime('%Y-%m-%d')\n", + " except Exception:\n", + " pass\n", + " return tag\n", + "\n", + "def _filter_paths_by_date(paths, date_start, date_end):\n", + " if not paths:\n", + " return paths\n", + " ds = pd.to_datetime(date_start).date()\n", + " de = pd.to_datetime(date_end).date()\n", + " kept = []\n", + " for p in paths:\n", + " tag = os.path.basename(p).replace('.tif','')\n", + " m = re.findall(r\"(\\d{8})\", tag)\n", + " if len(m) >= 2:\n", + " try:\n", + " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\").date()\n", + " d1 = pd.to_datetime(m[1], format=\"%Y%m%d\").date()\n", + " mid = d0 + (d1 - d0) / 2\n", + " mid_date = mid if hasattr(mid, 'year') else mid.date()\n", + " if ds <= mid_date <= de:\n", + " kept.append(p)\n", + " except Exception:\n", + " kept.append(p)\n", + " elif len(m) == 1:\n", + " try:\n", + " d0 = pd.to_datetime(m[0], format=\"%Y%m%d\").date()\n", + " if ds <= d0 <= de:\n", + " kept.append(p)\n", + " except Exception:\n", + " kept.append(p)\n", + " else:\n", + " kept.append(p)\n", + " return kept\n", + "\n", + "def _render_frames(tif_paths, out_dir, cmap, vmin=None, vmax=None, title_prefix=None, extent=None, cbar_label=None, cbar_ticks=None, template=None, dates=None, title_dates=None, show_time_bar=False, norm=None):\n", + " os.makedirs(out_dir, exist_ok=True)\n", + " frames = []\n", + " if template is None and tif_paths:\n", + " template = rioxarray.open_rasterio(tif_paths[0])[0]\n", + " if extent is None and template is not None:\n", + " minlon, minlat, maxlon, maxlat = template.rio.bounds()\n", + " extent = [minlon, maxlon, minlat, maxlat]\n", + "\n", + " date_min = date_max = None\n", + " if show_time_bar and dates:\n", + " valid_dates = [d for d in dates if d is not None]\n", + " if valid_dates:\n", + " date_min = min(valid_dates)\n", + " date_max = max(valid_dates)\n", + "\n", + " for i, p in enumerate(tif_paths):\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " if template is not None:\n", + " da = da.rio.reproject_match(template)\n", + " fig, ax = plt.subplots(figsize=(6,4))\n", + " im = ax.imshow(da.values, cmap=cmap, origin='upper', extent=extent, norm=norm, vmin=None if norm is not None else vmin, vmax=None if norm is not None else vmax)\n", + " tag = os.path.basename(p).replace('.tif','')\n", + " if title_prefix is None:\n", + " title_date = title_dates[i] if title_dates and i < len(title_dates) else None\n", + " title = _format_title(tag, title_date)\n", + " else:\n", + " title = f\"{title_prefix}{tag}\"\n", + " ax.set_title(title, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", + " if cbar_label:\n", + " cb.set_label(cbar_label)\n", + " if cbar_ticks is not None:\n", + " cb.set_ticks(cbar_ticks)\n", + "\n", + " if show_time_bar and date_min is not None and date_max is not None:\n", + " ax_time = fig.add_axes([0.12, 0.04, 0.76, 0.05])\n", + " ax_time.plot([0, 1], [0.5, 0.5], color=\"0.6\", linewidth=3, solid_capstyle='round')\n", + " cur_date = dates[i] if dates and i < len(dates) else None\n", + " if cur_date is not None and date_max != date_min:\n", + " pos = (cur_date - date_min) / (date_max - date_min)\n", + " pos = max(0, min(1, float(pos)))\n", + " else:\n", + " pos = 0.0 if date_max == date_min else 0.5\n", + " ax_time.plot([pos, pos], [0.2, 0.8], color=\"crimson\", linewidth=2, zorder=5)\n", + " ax_time.scatter([pos], [0.5], s=90, color=\"crimson\", zorder=6)\n", + "\n", + " if date_max == date_min:\n", + " tick_dates = [date_min]\n", + " else:\n", + " tick_dates = pd.date_range(date_min, date_max, periods=5)\n", + " for d in tick_dates:\n", + " t = (d - date_min) / (date_max - date_min)\n", + " t = max(0, min(1, float(t)))\n", + " ax_time.plot([t, t], [0.35, 0.65], color=\"0.4\", linewidth=1)\n", + " ax_time.text(t, -0.05, d.strftime('%Y-%m-%d'), ha='center', va='top', fontsize=6)\n", + "\n", + " ax_time.set_xlim(0, 1)\n", + " ax_time.set_ylim(0, 1)\n", + " ax_time.axis('off')\n", + "\n", + " frame_path = os.path.join(out_dir, f\"{tag}.png\")\n", + " fig.savefig(frame_path, dpi=150)\n", + " plt.close(fig)\n", + " frames.append(frame_path)\n", + " return frames\n", + "\n", + "def _pad_frames(frame_paths):\n", + " imgs = [imageio.imread(f) for f in frame_paths]\n", + " max_h = max(im.shape[0] for im in imgs)\n", + " max_w = max(im.shape[1] for im in imgs)\n", + " padded = []\n", + " for im in imgs:\n", + " pad_h = max_h - im.shape[0]\n", + " pad_w = max_w - im.shape[1]\n", + " top = pad_h // 2\n", + " bottom = pad_h - top\n", + " left = pad_w // 2\n", + " right = pad_w - left\n", + " padded.append(np.pad(im, ((top, bottom), (left, right), (0, 0)), mode='edge'))\n", + " return padded\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8397a42", + "metadata": {}, + "outputs": [], + "source": [ + "# Prepare coherence extent/template for GIFs\n", + "coh_template = None\n", + "coh_extent = None\n", + "if coh_paths:\n", + " coh_template = rioxarray.open_rasterio(coh_paths[0])[0]\n", + " minlon, minlat, maxlon, maxlat = coh_template.rio.bounds()\n", + " coh_extent = [minlon, maxlon, minlat, maxlat]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15d63323", + "metadata": {}, + "outputs": [], + "source": [ + "# GIF prep (paths, dates, extent)\n", + "gif_dir = f\"{savedir}/gifs\"\n", + "os.makedirs(gif_dir, exist_ok=True)\n", + "\n", + "coh_paths = _filter_paths_by_date(coh_paths, dateStart, dateEnd)\n", + "coh_dates = []\n", + "for p in coh_paths:\n", + " tag = Path(p).name.replace('merged_coh_', '').replace('.tif', '')\n", + " try:\n", + " ref_str, sec_str = tag.split('-')\n", + " ref_date = pd.to_datetime(ref_str, format='%Y%m%d')\n", + " sec_date = pd.to_datetime(sec_str, format='%Y%m%d')\n", + " coh_dates.append(ref_date + (sec_date - ref_date) / 2)\n", + " except Exception:\n", + " coh_dates.append(None)\n", + "\n", + "coh_template = None\n", + "coh_extent = None\n", + "if coh_paths:\n", + " coh_template = rioxarray.open_rasterio(coh_paths[0])[0]\n", + " minlon, minlat, maxlon, maxlat = coh_template.rio.bounds()\n", + " coh_extent = [minlon, maxlon, minlat, maxlat]\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60b7c196", + "metadata": {}, + "outputs": [], + "source": [ + "# Coherence GIF\n", + "# Adjust GIF speed based on number of frames\n", + "\n", + "def _gif_duration(n_frames, min_sec=0.3, max_sec=1.2, target_total_sec=12.0):\n", + " if n_frames <= 0:\n", + " return max_sec\n", + " return max(min_sec, min(max_sec, target_total_sec / n_frames))\n", + "\n", + "coh_norm = mcolors.PowerNorm(gamma=COH_NORM_GAMMA, vmin=0, vmax=1) if COH_USE_GAMMA_NORM else None\n", + "if coh_paths:\n", + " coh_frames = _render_frames(\n", + " coh_paths, f\"{gif_dir}/coh_frames\", cmap=cmc.bilbao, vmin=0, vmax=1,\n", + " title_prefix=None, extent=coh_extent, cbar_label='Coherence', cbar_ticks=[0,0.5,1],\n", + " template=coh_template, dates=coh_dates, title_dates=None, show_time_bar=True, norm=coh_norm\n", + " )\n", + " coh_gif = f\"{gif_dir}/coherence.gif\"\n", + " coh_imgs = _pad_frames(coh_frames)\n", + " imageio.mimsave(coh_gif, coh_imgs, duration=_gif_duration(len(coh_imgs)))\n", + " print(f\"Wrote {coh_gif}\")\n", + "else:\n", + " print('No merged coherence files found for GIF')\n", + "\n", + "\n", + "# Monthly mean coherence GIF\n", + "if coh_paths:\n", + " monthly_dir = f\"{gif_dir}/coh_monthly_frames\"\n", + " os.makedirs(monthly_dir, exist_ok=True)\n", + " # Build month index using midpoint dates\n", + " monthly = {}\n", + " for p, d in zip(coh_paths, coh_dates):\n", + " if d is None:\n", + " continue\n", + " key = d.strftime('%Y-%m')\n", + " monthly.setdefault(key, []).append(p)\n", + "\n", + " monthly_frames = []\n", + " for key in sorted(monthly.keys()):\n", + " stacks = []\n", + " for p in monthly[key]:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " if coh_template is not None:\n", + " da = da.rio.reproject_match(coh_template)\n", + " stacks.append(da)\n", + " if not stacks:\n", + " continue\n", + " da_month = xr.concat(stacks, dim='stack').mean(dim='stack', skipna=True)\n", + " minlon, minlat, maxlon, maxlat = da_month.rio.bounds()\n", + " bbox = [minlon, maxlon, minlat, maxlat]\n", + " fig, ax = plt.subplots(figsize=(6,4))\n", + " im = ax.imshow(da_month.values, cmap=cmc.bilbao, origin='upper', extent=bbox, interpolation='none', norm=coh_norm, vmin=None if COH_USE_GAMMA_NORM else 0, vmax=None if COH_USE_GAMMA_NORM else 1)\n", + " ax.set_title(key, fontsize=9)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " cb = fig.colorbar(im, ax=ax, fraction=0.046, pad=0.02)\n", + " cb.set_label('Mean coherence')\n", + " frame_path = os.path.join(monthly_dir, f\"{key}.png\")\n", + " fig.savefig(frame_path, dpi=150)\n", + " plt.close(fig)\n", + " monthly_frames.append(frame_path)\n", + "\n", + " if monthly_frames:\n", + " monthly_gif = f\"{gif_dir}/coherence_monthly.gif\"\n", + " monthly_imgs = _pad_frames(monthly_frames)\n", + " imageio.mimsave(monthly_gif, monthly_imgs, duration=_gif_duration(len(monthly_imgs)))\n", + " print(f\"Wrote {monthly_gif}\")\n", + " else:\n", + " print('No monthly coherence frames found for GIF')\n", + "\n", + "# Backscatter GIF (sigma0/beta0)\n", + "bsc_paths = sorted(glob.glob(f\"{savedir}/tifs/merged_bsc_{CALIBRATION_MODE}_*.tif\"))\n", + "bsc_paths = _filter_paths_by_date(bsc_paths, dateStart, dateEnd)\n", + "bsc_dates = []\n", + "\n", + "# Fixed color range across the series\n", + "bsc_all = []\n", + "for p in bsc_paths:\n", + " da = rioxarray.open_rasterio(p)[0]\n", + " bsc_all.append(da.values.ravel())\n", + "if bsc_all:\n", + " bsc_all = np.concatenate(bsc_all)\n", + " bsc_vmin = np.nanpercentile(bsc_all, 2)\n", + " bsc_vmax = np.nanpercentile(bsc_all, 90)\n", + "else:\n", + " bsc_vmin, bsc_vmax = None, None\n", + "for p in bsc_paths:\n", + " tag = Path(p).name.replace('.tif','')\n", + " bsc_dates.append(_extract_date_from_tag(tag))\n", + "\n", + "if bsc_paths:\n", + " bsc_template = rioxarray.open_rasterio(bsc_paths[0])[0]\n", + " minlon, minlat, maxlon, maxlat = bsc_template.rio.bounds()\n", + " bsc_extent = [minlon, maxlon, minlat, maxlat]\n", + " bsc_frames = _render_frames(\n", + " bsc_paths, f\"{gif_dir}/bsc_frames_{CALIBRATION_MODE}\", cmap=cmc.bilbao,\n", + " vmin=bsc_vmin, vmax=bsc_vmax, title_prefix=None, extent=bsc_extent,\n", + " cbar_label=f\"Backscatter ({CALIBRATION_MODE}) dB\", cbar_ticks=None,\n", + " template=bsc_template, dates=bsc_dates, title_dates=None, show_time_bar=True\n", + " )\n", + " bsc_gif = f\"{gif_dir}/backscatter_{CALIBRATION_MODE}.gif\"\n", + " bsc_imgs = _pad_frames(bsc_frames)\n", + " imageio.mimsave(bsc_gif, bsc_imgs, duration=_gif_duration(len(bsc_imgs)))\n", + " print(f\"Wrote {bsc_gif}\")\n", + "else:\n", + " print('No merged backscatter files found for GIF')\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "opera_cslc_slides", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.14.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}