From 6e625694d6ae13000b548518161d278d78a142f9 Mon Sep 17 00:00:00 2001 From: rzinke Date: Tue, 23 Sep 2025 01:04:37 +0000 Subject: [PATCH 1/3] Check whether ifg exclusions specified in my_sites --- .../secular/Secular_Requirement_Validation.ipynb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/methods/secular/Secular_Requirement_Validation.ipynb b/methods/secular/Secular_Requirement_Validation.ipynb index fd82084..d5be3c9 100644 --- a/methods/secular/Secular_Requirement_Validation.ipynb +++ b/methods/secular/Secular_Requirement_Validation.ipynb @@ -287,7 +287,20 @@ }, "outputs": [], "source": [ - "command = 'smallbaselineApp.py ' + str(config_file) + ' --dostep modify_network'\n", + "# Modify network - base command\n", + "command = f\"modify_network.py inputs/ifgramStack.h5 -t {config_file}\"\n", + "\n", + "# Check whether exclusions specified in my_sites file\n", + "if site_info.get('ifgExcludePair') not in [None, 'auto', 'no']:\n", + " command += f\" --exclude-ifg {site_info.get('ifgExcludePair')}\"\n", + "\n", + "if site_info.get('ifgExcludeDate') not in [None, 'auto', 'no']:\n", + " command += f\" --exclude-date {site_info.get('ifgExcludeDate')}\"\n", + "\n", + "if site_info.get('ifgExcludeIndex') not in [None, 'auto', 'no']:\n", + " command += f\" --exclude-ifg-index {site_info.get('ifgExcludeIndex')} \"\n", + "\n", + "# Run command\n", "process = subprocess.run(command, shell=True)\n", "plot_network.main(['inputs/ifgramStack.h5'])" ] From 9459b3aa658501b13a0b03ac362088db967ebfbb Mon Sep 17 00:00:00 2001 From: rzinke Date: Wed, 24 Sep 2025 00:14:26 +0000 Subject: [PATCH 2/3] Dropped dates, ifgs, indices are no longer stored in MintPy cfg files, but are applied dynamically by the method notebooks --- .../Coseismic_Requirement_Validation.ipynb | 20 ++++++- .../Secular_Requirement_Validation.ipynb | 9 ++- .../Transient_Requirement_Validation.ipynb | 56 +++++++++++++++---- prep/ARIA_prep.ipynb | 7 +-- 4 files changed, 70 insertions(+), 22 deletions(-) diff --git a/methods/coseismic/Coseismic_Requirement_Validation.ipynb b/methods/coseismic/Coseismic_Requirement_Validation.ipynb index cf91f1f..64b2909 100644 --- a/methods/coseismic/Coseismic_Requirement_Validation.ipynb +++ b/methods/coseismic/Coseismic_Requirement_Validation.ipynb @@ -288,9 +288,25 @@ }, "outputs": [], "source": [ - "command = 'smallbaselineApp.py ' + str(config_file) + ' --dostep modify_network'\n", + "# Formulate ifgramStack file\n", + "ifgramStack_file = os.path.join(mintpy_dir, 'inputs/ifgramStack.h5')\n", + "\n", + "# Modify network - base command\n", + "command = f\"modify_network.py {ifgramStack_file} -t {config_file}\"\n", + "\n", + "# Check whether exclusions specified in my_sites file\n", + "if site_info.get('ifgExcludePair') not in [None, 'auto', 'no']:\n", + " command += f\" --exclude-ifg {site_info.get('ifgExcludePair')}\"\n", + "\n", + "if site_info.get('ifgExcludeDate') not in [None, 'auto', 'no']:\n", + " command += f\" --exclude-date {site_info.get('ifgExcludeDate')}\"\n", + "\n", + "if site_info.get('ifgExcludeIndex') not in [None, 'auto', 'no']:\n", + " command += f\" --exclude-ifg-index {site_info.get('ifgExcludeIndex')} \"\n", + "\n", + "# Run command\n", "process = subprocess.run(command, shell=True)\n", - "plot_network.main(['inputs/ifgramStack.h5'])" + "plot_network.main([ifgramStack_file])" ] }, { diff --git a/methods/secular/Secular_Requirement_Validation.ipynb b/methods/secular/Secular_Requirement_Validation.ipynb index d5be3c9..612141b 100644 --- a/methods/secular/Secular_Requirement_Validation.ipynb +++ b/methods/secular/Secular_Requirement_Validation.ipynb @@ -150,7 +150,7 @@ "aria_gunw_version = \"3_0_1\"\n", "\n", "rundate = \"20250826\" # Date of this Cal/Val run\n", - "version = \"1\" # Version of this Cal/Val run\n", + "version = \"1b\" # Version of this Cal/Val run\n", "custom_sites = \"/home/jovyan/my_sites.txt\" # Path to custom site metadata\n", "\n", "# === Username Detection / Creation ===\n", @@ -287,8 +287,11 @@ }, "outputs": [], "source": [ + "# Formulate ifgramStack file\n", + "ifgramStack_file = os.path.join(mintpy_dir, 'inputs/ifgramStack.h5')\n", + "\n", "# Modify network - base command\n", - "command = f\"modify_network.py inputs/ifgramStack.h5 -t {config_file}\"\n", + "command = f\"modify_network.py {ifgramStack_file} -t {config_file}\"\n", "\n", "# Check whether exclusions specified in my_sites file\n", "if site_info.get('ifgExcludePair') not in [None, 'auto', 'no']:\n", @@ -302,7 +305,7 @@ "\n", "# Run command\n", "process = subprocess.run(command, shell=True)\n", - "plot_network.main(['inputs/ifgramStack.h5'])" + "plot_network.main([ifgramStack_file])" ] }, { diff --git a/methods/transient/Transient_Requirement_Validation.ipynb b/methods/transient/Transient_Requirement_Validation.ipynb index 66f79e4..07dc7c2 100644 --- a/methods/transient/Transient_Requirement_Validation.ipynb +++ b/methods/transient/Transient_Requirement_Validation.ipynb @@ -111,7 +111,7 @@ "from matplotlib import pyplot as plt\n", "from mintpy import smallbaselineApp\n", "from mintpy.objects import gnss\n", - "from mintpy.utils import readfile, utils as ut\n", + "from mintpy.utils import readfile, utils as ut, network\n", "from mintpy.cli import view\n", "\n", "from solid_utils.sampling import load_geo, samp_pair\n", @@ -138,13 +138,13 @@ "outputs": [], "source": [ "# === Basic Configuration ===\n", - "site = \"CalVal_S1_LosAngelesA64\"\n", + "site = \"test\"\n", "requirement = \"Transient\"\n", "dataset = 'ARIA_S1_new' # For Sentinel-1 testing with aria-tools\n", "aria_gunw_version = \"3_0_1\"\n", "\n", - "rundate = \"20250820\" # Date of this Cal/Val run\n", - "version = \"1\" # Version of this Cal/Val run\n", + "rundate = \"20250826\" # Date of this Cal/Val run\n", + "version = \"1b\" # Version of this Cal/Val run\n", "custom_sites = \"/home/jovyan/my_sites.txt\" # Path to custom site metadata\n", "\n", "# === Username Detection / Creation ===\n", @@ -278,6 +278,39 @@ "Load the date of interferograms into Python:" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Formulate ifgramStack file\n", + "ifgramStack_file = os.path.join(mintpy_dir, 'inputs/ifgramStack.h5')\n", + "\n", + "# Modify network - base command\n", + "command = f\"modify_network.py {ifgramStack_file} -t {config_file} \"\n", + "\n", + "# Check whether exclusions specified in my_sites file\n", + "if site_info.get('ifgExcludePair') not in [None, 'auto', 'no']:\n", + " command += f\" --exclude-ifg {site_info.get('ifgExcludePair')}\"\n", + "\n", + "if site_info.get('ifgExcludeDate') not in [None, 'auto', 'no']:\n", + " command += f\" --exclude-date {site_info.get('ifgExcludeDate')}\"\n", + "\n", + "if site_info.get('ifgExcludeIndex') not in [None, 'auto', 'no']:\n", + " command += f\" --exclude-ifg-index {site_info.get('ifgExcludeIndex')} \"\n", + "\n", + "# Run command\n", + "process = subprocess.run(command, shell=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Retrieve the valid date pairs." + ] + }, { "cell_type": "code", "execution_count": null, @@ -287,12 +320,12 @@ "outputs": [], "source": [ "# Retrieve available interferogram date pairs\n", - "ifgs_date = readfile.read(ifgs_file,datasetName='date')[0]\n", + "ifgs_date = network.get_date12_list(ifgs_file, dropIfgram=True)\n", "\n", "# Report all available date pairs\n", "print(f\"Total {len(ifgs_date)} interferograms available\")\n", "for pair in ifgs_date:\n", - " print(f\"{pair[0].decode()}-{pair[1].decode()}\")" + " print(f\"{'-'.join(pair.split('_'))}\")" ] }, { @@ -305,9 +338,8 @@ "source": [ "# Format date strings into Python datetime objects\n", "_ifgs_date = np.empty_like(ifgs_date, dtype=dt)\n", - "for i in range(ifgs_date.shape[0]):\n", - " start_date = ifgs_date[i,0].decode()\n", - " end_date = ifgs_date[i,1].decode()\n", + "for i, pair in enumerate(ifgs_date):\n", + " start_date, end_date = pair.split(\"_\")\n", " start_date = dt.strptime(start_date, \"%Y%m%d\")\n", " end_date = dt.strptime(end_date, \"%Y%m%d\")\n", " _ifgs_date[i] = [start_date, end_date]\n", @@ -337,11 +369,11 @@ "# Determine which interferograms to exclude based on 12-day criterion\n", "del_row_index = []\n", "for i in range(ifgs_date.shape[0]):\n", - " time_interval = (ifgs_date[i,1]-ifgs_date[i,0]).days\n", + " time_interval = (ifgs_date[i][1]-ifgs_date[i][0]).days\n", " if time_interval != 12:\n", " del_row_index.append(i)\n", "while i Date: Mon, 6 Oct 2025 20:19:03 +0000 Subject: [PATCH 3/3] Fixed remaining indices (e.g., ifgs_date[i,0] -> ifgs_date[i][0]) --- .../Transient_Requirement_Validation.ipynb | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/methods/transient/Transient_Requirement_Validation.ipynb b/methods/transient/Transient_Requirement_Validation.ipynb index 07dc7c2..265d87a 100644 --- a/methods/transient/Transient_Requirement_Validation.ipynb +++ b/methods/transient/Transient_Requirement_Validation.ipynb @@ -899,8 +899,8 @@ "E = W+lon_step*(width-1)\n", "\n", "# Temporal metadata\n", - "start_date_gnss = ifgs_date[0,0]\n", - "end_date_gnss = ifgs_date[-1,-1]" + "start_date_gnss = ifgs_date[0][0]\n", + "end_date_gnss = ifgs_date[-1][-1]" ] }, { @@ -998,8 +998,8 @@ " # Count number of dates in time range by looping through interferograms\n", " for ifg_ndx in range(insar_displacement.shape[0]):\n", " # Days in interferogram range (should be 12 based on above filtering)\n", - " start_date = ifgs_date[ifg_ndx,0]\n", - " end_date = ifgs_date[ifg_ndx,-1]\n", + " start_date = ifgs_date[ifg_ndx][0]\n", + " end_date = ifgs_date[ifg_ndx][-1]\n", " range_days = (end_date - start_date).days\n", "\n", " # Count number of GNSS epochs in IFG date range\n", @@ -1235,7 +1235,7 @@ " img1 = ax.imshow(insar_displacement[ifg_ndx],\n", " cmap=cmap_obj, vmin=vmin, vmax=vmax, interpolation='nearest',\n", " extent=(W, E, S, N))\n", - " ax.set_title(f\"{ifgs_date[ifg_ndx,0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx,1].strftime('%Y%m%d')}\")\n", + " ax.set_title(f\"{ifgs_date[ifg_ndx][0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx][1].strftime('%Y%m%d')}\")\n", " cbar1 = fig.colorbar(img1, ax=ax, orientation='horizontal')\n", " cbar1.set_label('LOS displacement [mm]')\n", "\n", @@ -1365,8 +1365,8 @@ "site_loc = sitedata['sites'][site]['calval_location']\n", "for ifg_ndx in displacement.index.get_level_values(0).unique():\n", " # Start and end dates as strings\n", - " start_date = ifgs_date[ifg_ndx,0].strftime('%Y%m%d')\n", - " end_date = ifgs_date[ifg_ndx,1].strftime('%Y%m%d')\n", + " start_date = ifgs_date[ifg_ndx][0].strftime('%Y%m%d')\n", + " end_date = ifgs_date[ifg_ndx][1].strftime('%Y%m%d')\n", "\n", " # Validation figure and assessment\n", " _, validation_fig_method1 = display_transient_validation(ddiff_dist[ifg_ndx], abs_ddiff_disp[ifg_ndx],\n", @@ -1518,7 +1518,7 @@ "\n", "index = []\n", "for i in range(len(ifgs_date_ap1)):\n", - " index.append(ifgs_date_ap1[i,0].strftime('%Y%m%d')+'-'+ifgs_date_ap1[i,1].strftime('%Y%m%d'))\n", + " index.append(ifgs_date_ap1[i][0].strftime('%Y%m%d')+'-'+ifgs_date_ap1[i][1].strftime('%Y%m%d'))\n", "\n", "n_all_pd = pd.DataFrame(n_all,columns=columns,index=index)\n", "n_pass_pd = pd.DataFrame(n_pass,columns=columns,index=index)\n", @@ -1761,7 +1761,7 @@ " fig, ax = plt.subplots(figsize=[18, 5.5])\n", " img1 = ax.imshow(insar_coherence[ifg_ndx], cmap=cmap_obj, interpolation='nearest', extent=(W, E, S, N))\n", " ax.set_title(f\"Coherence \"\n", - " f\"\\n Date range {ifgs_date[ifg_ndx,0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx,1].strftime('%Y%m%d')}\")\n", + " f\"\\n Date range {ifgs_date[ifg_ndx][0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx][1].strftime('%Y%m%d')}\")\n", " cbar1 = fig.colorbar(img1, ax=ax)\n", " cbar1.set_label('coherence')" ] @@ -1780,7 +1780,7 @@ " fig, ax = plt.subplots(figsize=[18, 5.5])\n", " img1 = ax.imshow(insar_displacement[ifg_ndx], cmap=cmap_obj, interpolation='nearest', extent=(W, E, S, N))\n", " ax.set_title(f\"Interferogram \"\n", - " f\"\\n Date range {ifgs_date[i,0].strftime('%Y%m%d')}-{ifgs_date[i,1].strftime('%Y%m%d')}\")\n", + " f\"\\n Date range {ifgs_date[i][0].strftime('%Y%m%d')}-{ifgs_date[i][1].strftime('%Y%m%d')}\")\n", " cbar1 = fig.colorbar(img1, ax=ax)\n", " cbar1.set_label('LOS displacement [mm]')" ] @@ -1853,7 +1853,7 @@ " fig, ax = plt.subplots(figsize=[18, 5.5])\n", " img1 = ax.hist(dist[ifg_ndx], bins=100)\n", " ax.set_title(f\"Histogram of distance \"\n", - " f\"\\n Date range {ifgs_date[ifg_ndx,0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx,1].strftime('%Y%m%d')}\")\n", + " f\"\\n Date range {ifgs_date[ifg_ndx][0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx][1].strftime('%Y%m%d')}\")\n", " ax.set_xlabel(r'Distance ($km$)')\n", " ax.set_ylabel('Frequency')\n", " ax.set_xlim(0, 50)" @@ -1872,7 +1872,7 @@ " fig, ax = plt.subplots(figsize=[18, 5.5])\n", " img1 = ax.hist(rel_measure[ifg_ndx], bins=100)\n", " ax.set_title(f\"Histogram of Relative Measurement \"\n", - " f\"\\n Date range {ifgs_date[ifg_ndx,0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx,1].strftime('%Y%m%d')}\")\n", + " f\"\\n Date range {ifgs_date[ifg_ndx][0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx][1].strftime('%Y%m%d')}\")\n", " ax.set_xlabel(r'Relative Measurement ($mm$)')\n", " ax.set_ylabel('Frequency')" ] @@ -1995,8 +1995,8 @@ "method2_validation_figs = []\n", "for ifg_ndx in displacement.index.get_level_values(0).unique():\n", " # Start and end dates as strings\n", - " start_date = ifgs_date[ifg_ndx,0].strftime('%Y%m%d')\n", - " end_date = ifgs_date[ifg_ndx,1].strftime('%Y%m%d')\n", + " start_date = ifgs_date[ifg_ndx][0].strftime('%Y%m%d')\n", + " end_date = ifgs_date[ifg_ndx][1].strftime('%Y%m%d')\n", "\n", " # Validation figure and assessment\n", " _, validation_fig_method2 = display_transient_validation(dist[ifg_ndx], rel_measure[ifg_ndx],\n", @@ -2049,7 +2049,7 @@ "\n", "index = []\n", "for i in range(len(ifgs_date)):\n", - " index.append(f\"{ifgs_date[i,0].strftime('%Y%m%d')}-{ifgs_date[i,1].strftime('%Y%m%d')}\")\n", + " index.append(f\"{ifgs_date[i][0].strftime('%Y%m%d')}-{ifgs_date[i][1].strftime('%Y%m%d')}\")\n", "\n", "n_all_pd = pd.DataFrame(n_all, columns=columns, index=index)\n", "n_pass_pd = pd.DataFrame(n_pass, columns=columns, index=index)\n", @@ -2228,7 +2228,7 @@ " # Format figure\n", " plt.legend(loc='upper right')\n", " plt.title(f\"Displacements \"\n", - " f\"\\n Date range {ifgs_date[ifg_ndx,0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx,1].strftime('%Y%m%d')} \"\n", + " f\"\\n Date range {ifgs_date[ifg_ndx][0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx][1].strftime('%Y%m%d')} \"\n", " f\"\\n Number of station pairs used: {len(insar_disp[ifg_ndx])}\")\n", " plt.xlabel('LOS Displacement (mm)')\n", " plt.ylabel('Number of Station Pairs')\n", @@ -2252,7 +2252,7 @@ " # Format figure\n", " plt.legend(loc='upper right')\n", " plt.title(f\"Residuals\"\n", - " f\"\\n Date range {ifgs_date[ifg_ndx,0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx,1].strftime('%Y%m%d')}\"\n", + " f\"\\n Date range {ifgs_date[ifg_ndx][0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx][1].strftime('%Y%m%d')}\"\n", " f\"\\n Number of stations pairs used: {len(ddiff_disp[i])}\")\n", " plt.xlabel('Displacement Residual (mm)')\n", " plt.ylabel('N Stations')\n", @@ -2282,7 +2282,7 @@ " plt.xlabel(\"Distance (km)\")\n", " plt.ylabel(\"Amplitude of Displacement Residuals (mm)\")\n", " plt.title(f\"Residuals \"\n", - " f\"\\n Date range {ifgs_date[ifg_ndx,0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx,1].strftime('%Y%m%d')} \"\n", + " f\"\\n Date range {ifgs_date[ifg_ndx][0].strftime('%Y%m%d')}-{ifgs_date[ifg_ndx][1].strftime('%Y%m%d')} \"\n", " f\"\\n Number of stations pairs used: {len(ddiff_dist[ifg_ndx])}\")\n", " plt.legend([\"Measurement\", \"Mission Reqiurement\"])\n", " plt.show()" @@ -2310,8 +2310,8 @@ "# Loop through interferograms\n", "for ifg_ndx in range(insar_displacement.shape[0]):\n", " # Define start and end dates\n", - " start_time_str = ifgs_date[ifg_ndx,0].strftime('%Y%m%d')\n", - " end_time_str = ifgs_date[ifg_ndx,1].strftime('%Y%m%d')\n", + " start_time_str = ifgs_date[ifg_ndx][0].strftime('%Y%m%d')\n", + " end_time_str = ifgs_date[ifg_ndx][1].strftime('%Y%m%d')\n", "\n", " # Loop through stations in IFG\n", " for site_name in gnss_time_series[ifg_ndx].columns:\n", @@ -2324,7 +2324,7 @@ " plt.figure(figsize=(15,5))\n", "\n", " # Plot time-series data\n", - " plt.scatter(pd.date_range(start=ifgs_date[ifg_ndx,0], end=ifgs_date[ifg_ndx,1]),series)\n", + " plt.scatter(pd.date_range(start=ifgs_date[ifg_ndx][0], end=ifgs_date[ifg_ndx][1]),series)\n", "\n", " # Foramt figure\n", " plt.title(f\"station name: {site_name}\")\n",