From 66715efeda2ebde3ee5b54650d5e980ee6ad089d Mon Sep 17 00:00:00 2001
From: Monimu <monimu@gmx.de>
Date: Fri, 20 Feb 2026 15:50:13 +0100
Subject: [PATCH] hierarchical clustering dealing with nans when averaging SI
 and CH, change to python 3.11 only

---
 .DS_Store                       | Bin 0 -> 6148 bytes
 .github/workflows/ci.yml        |   2 +-
 jale/.DS_Store                  | Bin 0 -> 6148 bytes
 jale/core/.DS_Store             | Bin 0 -> 6148 bytes
 jale/core/utils/hierarchical.py |  44 ++++++++++++++++++++++++--------
 pyproject.toml                  |   3 +--
 6 files changed, 35 insertions(+), 14 deletions(-)
 create mode 100644 .DS_Store
 create mode 100644 jale/.DS_Store
 create mode 100644 jale/core/.DS_Store

diff --git a/.DS_Store b/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..e2231bcefdb97d62a461d4c89292daf61bd72ebb
GIT binary patch
literal 6148
zcmeHK%}T>S5Z-O0CKMqD1-%8l7Hny)h?fxS3mDOZN=-=6Xv~%-HHT8jU0=u-@p+ut
z-H63{6|pn0`_1oe_JiyXV~l&#sLPnm7&D<Ea#WTGx|fC;IvJ7U7-5-@!&HP|zv<Xt
z2mE$}J+OeKEdKia;W$n5qSJZjji#}-zG0eX%e?XLMd=qoG0nYTa)s8p5NTNJL3okG
z<;dPT5m^x=Sv-*iNfblM^<|PpqV&Wxi*i}(tAJ@)*2vzS%?^8mw$two=51%z?H+W%
zIO@;mmbtyRe|$E2OrJ9Gq8Q~sxRgzeHN1lIxu9o%lBFWM17DqA=MfSE!~iis46G~z
z`baQaE6V^aoERVmeqsRk2L~FWtFcljw+`s=`iyY{5e0O7OCUNLU5%AOh=6ca3aCoC
zeqwM{4t_`Hxf&~ls+@5-GmN8Wt{*R4&JKP@rZes;q@EZc1{N7;D$~aE{}g_irH}l@
z60(Q^V&I=Kz>T3l^k7l?Z2h)8JZl|jx6n{9uR;X`^o2_R7`Tt@$e@Zls6(Esu~LYm
TpkI~)(nUZLLLD*i3k-Y!gWgJV

literal 0
HcmV?d00001

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a14bda4..175c399 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -16,7 +16,7 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.11, 3.12]
+        python-version: [3.11]
 
     steps:
       - uses: actions/checkout@v3
diff --git a/jale/.DS_Store b/jale/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..0b27785aa8d28ab4c57d76604da104f8cfb8dcfd
GIT binary patch
literal 6148
zcmeHK%}T>S5Z-O0CKMqD1-%8l7Hny)h?h|73mDOZN=;1AV9b^zHHT8jU0=u-@p+ut
z-H63{6|pn0`_1oe_JiyXV~h`{VV5zRF=j$T<ftqWbT17xbTT5xF@iFm1c?a1e$%nP
z4*2Z`yJLc7Ec*KW!6b?EqSJZjji#}-zG0eX%e?U(MClcNG0lg5c7@it5J^z$esB>-
z<=Eaj5ozJaX_U!=IE*0W`Z7*KQ4Ym44RcxRseox&*4W;i%?^8mw$two=51%z?H+W%
zIO@;mmbtyRe|$E6OrBElq8Q~sxRgzeHN1lIxu9n+OB0dagRjo7^9YFnVt^PR23D2<
zeI%Hzm1TeyP7DwOKQVy&g98oG)mSN%TL*M_ea5(fhypskB@i8ruEt6sL_oMI1yrS6
zKQXu}2fw59T#c1NRnEAa8OG5w*N+!2X9vF{(;0UaQcnyJ1B(nam1*Poe+s|M(ntPc
z30cGdG4RhA;Ks-s4PjCGZ2h)8JZl|jx6n{9uR;X`^o2_R7`Tt@$e@Zls6(Esu~LYm
TpkI~)(nUZLLLD*i3k-Y!&813=

literal 0
HcmV?d00001

diff --git a/jale/core/.DS_Store b/jale/core/.DS_Store
new file mode 100644
index 0000000000000000000000000000000000000000..57d6dcf3dfe95e0ce384e5b46eb1450a7edfa522
GIT binary patch
literal 6148
zcmeHK%SyvQ6rJfq6N-?9g6;xt3%0aY#7&6x2aM=Kr6#0kFlI`Vm_;dMuRr9E_&we`
zGl3R^OA&i7%$)n2$sEW$7-QVc!#-mcW6Xet$Wf^gbk~L&IvJ76G18z&XF)82nqmH?
z3%|Y1a+a}(#ccKaKZ02trCG1{$s5g9du!V;jE-^dJ&D50{47she}0SBl@M`I>3(n>
zO^b=OdoGg9kCN$J7DV9`Qtob|Bou`!@+3@Ut)~KpX_^!3V6iwEjJo!4Fj{u)MZbUC
z1LJhKT$;w-;nCUU<T-vx#G7K01Lab7G*<8q%F2>ny?GLg<Pp48MU_cN3=jjv05PzU
z4Cv#)=xpTbY0<<0G4K-uxIfs?5FL%BLb-K7hu3F}w-8Z4$F~Hct<lj~Duf6KSEYce
zl<Ox3SLNWhb)KWKRH(`smovjSdgki!!sYDXw`Drxjza2*0b*dCfu=GoJpV7?m#KZ^
zua}TT3=jkVi~-&ndt(<CrO(!H%fqwQhV}pr1@lT&KtNx+1b~72$i57!xQ#mGIT}la
UI12h@IUrpGBq7uh1HZt)7xJS^SO5S3

literal 0
HcmV?d00001

diff --git a/jale/core/utils/hierarchical.py b/jale/core/utils/hierarchical.py
index 6279683..fdc3652 100644
--- a/jale/core/utils/hierarchical.py
+++ b/jale/core/utils/hierarchical.py
@@ -89,8 +89,10 @@ def hierarchical_clustering_pipeline(
         project_path=project_path,
         meta_name=meta_name,
         silhouette_scores=silhouette_scores,
+        null_silhouette_scores=null_silhouette_scores,
         silhouette_scores_z=silhouette_scores_z,
         calinski_harabasz_scores=calinski_harabasz_scores,
+        null_calinski_harabasz_scores=null_calinski_harabasz_scores,
         calinski_harabasz_scores_z=calinski_harabasz_scores_z,
         exp_separation_density=exp_separation_density,
         correlation_type=correlation_type,
@@ -328,21 +330,21 @@ def compute_hc_metrics_z(
     def pooled_std(sample1, sample2):
         """Compute the pooled standard deviation of two samples."""
         n1, n2 = sample1.shape[1], sample2.shape[1]
-        var1, var2 = np.var(sample1, axis=1, ddof=1), np.var(sample2, axis=1, ddof=1)
+        var1, var2 = np.nanvar(sample1, axis=1, ddof=1), np.nanvar(sample2, axis=1, ddof=1)
         return np.sqrt(((n1 - 1) * var1 + (n2 - 1) * var2) / (n1 + n2 - 2))
 
-    silhouette_scores_avg = np.average(silhouette_scores, axis=1)
-    null_silhouette_scores_avg = np.average(null_silhouette_scores, axis=1)
+    silhouette_scores_avg = np.nanmean(silhouette_scores, axis=1)
+    null_silhouette_scores_avg = np.nanmean(null_silhouette_scores, axis=1)
 
     if use_pooled_std:
         silhouette_std = pooled_std(silhouette_scores, null_silhouette_scores)
     else:
-        silhouette_std = np.std(null_silhouette_scores, axis=1, ddof=1)
+        silhouette_std = np.nanstd(null_silhouette_scores, axis=1, ddof=1)
 
     silhouette_z = (silhouette_scores_avg - null_silhouette_scores_avg) / silhouette_std
 
-    calinski_harabasz_scores_avg = np.average(calinski_harabasz_scores, axis=1)
-    null_calinski_harabasz_scores_avg = np.average(
+    calinski_harabasz_scores_avg = np.nanmean(calinski_harabasz_scores, axis=1)
+    null_calinski_harabasz_scores_avg = np.nanmean(
         null_calinski_harabasz_scores, axis=1
     )
 
@@ -351,7 +353,7 @@ def pooled_std(sample1, sample2):
             calinski_harabasz_scores, null_calinski_harabasz_scores
         )
     else:
-        calinski_harabasz_std = np.std(null_calinski_harabasz_scores, axis=1, ddof=1)
+        calinski_harabasz_std = np.nanstd(null_calinski_harabasz_scores, axis=1, ddof=1)
 
     calinski_harabasz_z = (
         calinski_harabasz_scores_avg - null_calinski_harabasz_scores_avg
@@ -435,8 +437,10 @@ def save_hc_metrics(
     project_path,
     meta_name,
     silhouette_scores,
+    null_silhouette_scores,
     silhouette_scores_z,
     calinski_harabasz_scores,
+    null_calinski_harabasz_scores,
     calinski_harabasz_scores_z,
     exp_separation_density,
     correlation_type,
@@ -446,11 +450,15 @@ def save_hc_metrics(
     metrics_df = pd.DataFrame(
         {
             "Number of Clusters": range(2, max_k + 1),
-            "Silhouette Scores": np.average(silhouette_scores, axis=1),
-            "Silhouette Scores SD": np.std(silhouette_scores, axis=1),
+            "Silhouette Scores": np.nanmean(silhouette_scores, axis=1),
+            "Silhouette Scores SD": np.nanstd(silhouette_scores, axis=1),
+            "Silhouette Score Null": np.nanmean(null_silhouette_scores, axis=1),
+            "Silhouette Scores Null SD": np.nanstd(null_silhouette_scores, axis=1),
             "Silhouette Scores Z": silhouette_scores_z,
-            "Calinski-Harabasz Scores": np.average(calinski_harabasz_scores, axis=1),
-            "Calinski-Harabasz Scores SD": np.std(calinski_harabasz_scores, axis=1),
+            "Calinski-Harabasz Scores": np.nanmean(calinski_harabasz_scores, axis=1),
+            "Calinski-Harabasz Scores Null": np.nanmean(null_calinski_harabasz_scores, axis=1),
+            "Calinski-Harabasz Scoress Null SD": np.nanstd(null_calinski_harabasz_scores, axis=1),
+            "Calinski-Harabasz Scores SD": np.nanstd(calinski_harabasz_scores, axis=1),
             "Calinski-Harabasz Scores Z": calinski_harabasz_scores_z,
             # Pad with NaN for k=2 as metrics start at k=3
             "Experiment Separation Density": np.concatenate(
@@ -471,6 +479,13 @@ def save_hc_metrics(
         header=[f"k={k}" for k in range(2, max_k + 1)],
     )
 
+    pd.DataFrame(null_silhouette_scores.T).to_csv(
+        project_path
+        / f"Results/MA_Clustering/metrics/{meta_name}_Null_silhouette_scores_{correlation_type}_hc_{linkage_method}.csv",
+        index=False,
+        header=[f"k={k}" for k in range(2, max_k + 1)],
+    )
+
     pd.DataFrame(calinski_harabasz_scores.T).to_csv(
         project_path
         / f"Results/MA_Clustering/metrics/{meta_name}_calinski_harabasz_scores_{correlation_type}_hc_{linkage_method}.csv",
@@ -478,6 +493,13 @@ def save_hc_metrics(
         header=[f"k={k}" for k in range(2, max_k + 1)],
     )
 
+    pd.DataFrame(null_calinski_harabasz_scores.T).to_csv(
+        project_path
+        / f"Results/MA_Clustering/metrics/{meta_name}_Null_calinski_harabasz_scores_{correlation_type}_hc_{linkage_method}.csv",
+        index=False,
+        header=[f"k={k}" for k in range(2, max_k + 1)],
+    )
+
     pd.DataFrame(exp_separation_density.T).to_csv(
         project_path
         / f"Results/MA_Clustering/metrics/{meta_name}_exp_separation_density_{correlation_type}_hc_{linkage_method}.csv",
diff --git a/pyproject.toml b/pyproject.toml
index e127593..2f533c3 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,13 +7,12 @@ name = "jale"
 version = "0.2.9"
 description = "Package allowing users to run Activation Likelihood Estimation Meta-Analysis"
 readme = "README.md"
-requires-python = ">=3.9, <3.13"
+requires-python = ">=3.11, <3.12"
 license = "MIT"
 authors = [{ name = "Lennart Frahm", email = "l.frahm@mailbox.org" }]
 classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
     "Operating System :: OS Independent",
 ]
 dependencies = [