diff --git a/docs/tutorial_vla.ipynb b/docs/tutorial_vla.ipynb index 897a79ba..1671be19 100644 --- a/docs/tutorial_vla.ipynb +++ b/docs/tutorial_vla.ipynb @@ -41,12 +41,6 @@ "execution_count": 1, "id": "7db6f868-030c-41ee-8188-c236aa675c27", "metadata": { - "execution": { - "iopub.execute_input": "2025-06-04T16:18:37.911073Z", - "iopub.status.busy": "2025-06-04T16:18:37.910824Z", - "iopub.status.idle": "2025-06-04T16:18:40.926016Z", - "shell.execute_reply": "2025-06-04T16:18:40.923455Z" - }, "tags": [] }, "outputs": [ @@ -54,7 +48,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:38,376\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m viperlog: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n" + "AstroHACK version 0.7.1 already installed.\n" ] }, { @@ -158,12 +152,6 @@ "execution_count": 2, "id": "ffb79bcd", "metadata": { - "execution": { - "iopub.execute_input": "2025-06-04T16:18:40.928267Z", - "iopub.status.busy": "2025-06-04T16:18:40.927773Z", - "iopub.status.idle": "2025-06-04T16:18:43.383806Z", - "shell.execute_reply": "2025-06-04T16:18:43.383273Z" - }, "tags": [] }, "outputs": [ @@ -171,21 +159,37 @@ "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:40,929\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper/utils/data/.dropbox\u001b[0m \n" + "[\u001b[38;2;128;05;128m2025-07-01 16:42:16,030\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper/utils/data/.dropbox\u001b[0m \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:16,031\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Updating file metadata information ... \n", + " " ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:40,929\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Updating file metadata information ... \n" - ] + "data": { + "text/html": [ + "
                                       \n",
+       "  Download List                        \n",
+       " ───────────────────────────────────── \n",
+       "  ea25_cal_small_after_fixed.split.ms  \n",
+       "                                       \n",
+       "
\n" + ], + "text/plain": [ + " \n", + " \u001b[1m \u001b[0m\u001b[1mDownload List \u001b[0m\u001b[1m \u001b[0m \n", + " ───────────────────────────────────── \n", + " \u001b[35mea25_cal_small_after_fixed.split.ms\u001b[0m \n", + " \n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ - " " + "[\u001b[38;2;128;05;128m2025-07-01 16:42:17,241\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: data/ea25_cal_small_after_fixed.split.ms \n" ] }, { @@ -291,78 +295,93 @@ "cell_type": "code", "execution_count": 3, "id": "7fb0902f-f274-4d47-a48e-61c0c2561ca7", - "metadata": { - "execution": { - "iopub.execute_input": "2025-06-04T16:18:43.387848Z", - "iopub.status.busy": "2025-06-04T16:18:43.387343Z", - "iopub.status.idle": "2025-06-04T16:18:50.451784Z", - "shell.execute_reply": "2025-06-04T16:18:50.451381Z" - } - }, + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:43,388\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:48,304\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Finished processing \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:48,348\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Writing distance matrix to /export/home/figs/vdesouza/work/Holography-1022/astrohack/docs/.baseline_distance_matrix.csv ... \n" + "[\u001b[38;2;128;05;128m2025-07-01 16:42:17,246\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:21,526\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Finished processing \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:21,612\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Writing distance matrix to /export/home/figs/vdesouza/work/Holography-1022/astrohack/docs/.baseline_distance_matrix.csv ... \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:21,620\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper/utils/data/.dropbox\u001b[0m \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:21,624\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Updating file metadata information ... \n", + " " ] }, { - "name": "stdout", - "output_type": "stream", - "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:48,352\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m File exists: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper/utils/data/.dropbox\u001b[0m \n" - ] + "data": { + "text/html": [ + "
                   \n",
+       "  Download List    \n",
+       " ───────────────── \n",
+       "  heuristic_model  \n",
+       "                   \n",
+       "
\n" + ], + "text/plain": [ + " \n", + " \u001b[1m \u001b[0m\u001b[1mDownload List \u001b[0m\u001b[1m \u001b[0m \n", + " ───────────────── \n", + " \u001b[35mheuristic_model\u001b[0m \n", + " \n" + ] + }, + "metadata": {}, + "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:48,352\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Updating file metadata information ... \n" + " " ] }, { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - " " + "model/elastic.model: 100%|██████████████████████████████████████████████████████████████████████████████| 556/556 [00:00<00:00, 2.69MiB/s]\n", + "/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/sklearn/base.py:380: InconsistentVersionWarning: Trying to unpickle estimator ElasticNet from version 1.3.2 when using version 1.6.1. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n", + "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n", + " warnings.warn(\n" ] }, { "data": { "text/html": [ - "
                   \n",
-       "  Download List    \n",
-       " ───────────────── \n",
-       "  heuristic_model  \n",
-       "                   \n",
+       "
                                      System Info                                       \n",
+       "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n",
+       "┃ N-cores  Available memory (MB)  Total memory (MB)  Suggested memory per core (MB) ┃\n",
+       "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n",
+       "│       4  18966                  31703                                       11911 │\n",
+       "└─────────┴───────────────────────┴───────────────────┴────────────────────────────────┘\n",
+       "    Available memory: represents the system memory available without going into swap    \n",
        "
\n" ], "text/plain": [ - " \n", - " \u001b[1m \u001b[0m\u001b[1mDownload List \u001b[0m\u001b[1m \u001b[0m \n", - " ───────────────── \n", - " \u001b[35mheuristic_model\u001b[0m \n", - " \n" + "\u001b[3m System Info \u001b[0m\n", + "┏━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓\n", + "┃\u001b[1m \u001b[0m\u001b[1mN-cores\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mAvailable memory (MB)\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mTotal memory (MB)\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mSuggested memory per core (MB)\u001b[0m\u001b[1m \u001b[0m┃\n", + "┡━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩\n", + "│\u001b[34m \u001b[0m\u001b[34m 4\u001b[0m\u001b[34m \u001b[0m│\u001b[35m \u001b[0m\u001b[35m18966 \u001b[0m\u001b[35m \u001b[0m│\u001b[36m \u001b[0m\u001b[36m31703 \u001b[0m\u001b[36m \u001b[0m│\u001b[32m \u001b[0m\u001b[32m 11911\u001b[0m\u001b[32m \u001b[0m│\n", + "└─────────┴───────────────────────┴───────────────────┴────────────────────────────────┘\n", + "\u001b[2;3m Available memory: represents the system memory available without going into swap \u001b[0m\n" ] }, "metadata": {}, "output_type": "display_data" }, + { + "data": { + "text/plain": [ + "11911" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + }, { "name": "stdout", "output_type": "stream", @@ -443,12 +462,6 @@ "execution_count": 4, "id": "10dffc63-1907-497f-b025-392b5813eac9", "metadata": { - "execution": { - "iopub.execute_input": "2025-06-04T16:18:50.453501Z", - "iopub.status.busy": "2025-06-04T16:18:50.453006Z", - "iopub.status.idle": "2025-06-04T16:18:52.068383Z", - "shell.execute_reply": "2025-06-04T16:18:52.067796Z" - }, "tags": [] }, "outputs": [ @@ -456,63 +469,503 @@ "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:50,454\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper\u001b[0m \n" + "[\u001b[38;2;128;05;128m2025-07-01 16:42:23,706\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/toolviper\u001b[0m \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:23,712\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m It is recommended that the local cache directory be set using the \u001b[38;2;50;50;205mdask_local_dir\u001b[0m parameter. \n" ] }, { - "name": "stdout", + "name": "stderr", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:50,461\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m It is recommended that the local cache directory be set using the \u001b[38;2;50;50;205mdask_local_dir\u001b[0m parameter. \n" + "/export/home/figs/vdesouza/mambaforge/envs/casadev/lib/python3.12/site-packages/distributed/node.py:187: UserWarning: Port 8787 is already in use.\n", + "Perhaps you already have a cluster running?\n", + "Hosting the HTTP server on port 35271 instead\n", + " warnings.warn(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,937\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: slurm -- Success \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mdask_jobqueue\u001b[0m is available \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: dask_ssh -- Fail \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205masyncssh\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mjupyter_server_proxy\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mparamiko\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,938\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: CUDA -- Fail \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,939\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mdask_cuda\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,939\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;46;139;87mAvailable functions of this environment\u001b[0m: slurm \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: slurm -- Success \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: slurm -- Success \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mdask_jobqueue\u001b[0m is available \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mdask_jobqueue\u001b[0m is available \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: dask_ssh -- Fail \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205masyncssh\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: dask_ssh -- Fail \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mjupyter_server_proxy\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mparamiko\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,943\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205masyncssh\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: CUDA -- Fail \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mjupyter_server_proxy\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mparamiko\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mdask_cuda\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;46;139;87mAvailable functions of this environment\u001b[0m: slurm \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: CUDA -- Fail \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mdask_cuda\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;46;139;87mChecking functions availability:\u001b[0m \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;46;139;87mAvailable functions of this environment\u001b[0m: slurm \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: slurm -- Success \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,944\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mdask_jobqueue\u001b[0m is available \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,945\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: dask_ssh -- Fail \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,945\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205masyncssh\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,945\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mjupyter_server_proxy\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,945\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mparamiko\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,946\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Loading module: CUDA -- Fail \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,946\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;50;50;205mdask_cuda\u001b[0m is unavailable \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,946\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m \u001b[38;2;46;139;87mAvailable functions of this environment\u001b[0m: slurm \n", - "[\u001b[38;2;128;05;128m2025-06-04 10:18:51,948\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Client \n" + "[\u001b[38;2;128;05;128m2025-07-01 16:42:26,465\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Client \n" ] }, + { + "data": { + "text/html": [ + "
\n", + "
\n", + "
\n", + "

Client

\n", + "

MenrvaClient-a8c7c1b4-56cc-11f0-a921-0c37967bc0e9

\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
Connection method: Cluster objectCluster type: distributed.LocalCluster
\n", + " Dashboard: http://127.0.0.1:35271/status\n", + "
\n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "

Cluster Info

\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

LocalCluster

\n", + "

594d734b

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "\n", + " \n", + "
\n", + " Dashboard: http://127.0.0.1:35271/status\n", + " \n", + " Workers: 4\n", + "
\n", + " Total threads: 4\n", + " \n", + " Total memory: 30.96 GiB\n", + "
Status: runningUsing processes: True
\n", + "\n", + "
\n", + " \n", + "

Scheduler Info

\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

Scheduler

\n", + "

Scheduler-9987407a-6cc6-4c6f-b5b1-976f7cc90d32

\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " Comm: tcp://127.0.0.1:46589\n", + " \n", + " Workers: 4 \n", + "
\n", + " Dashboard: http://127.0.0.1:35271/status\n", + " \n", + " Total threads: 4\n", + "
\n", + " Started: Just now\n", + " \n", + " Total memory: 30.96 GiB\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "

Workers

\n", + "
\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 0

\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:37205\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:44653/status\n", + " \n", + " Memory: 7.74 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:46231\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-c9r4n_ir\n", + "
\n", + " Tasks executing: \n", + " \n", + " Tasks in memory: \n", + "
\n", + " Tasks ready: \n", + " \n", + " Tasks in flight: \n", + "
\n", + " CPU usage: 0.0%\n", + " \n", + " Last seen: Just now\n", + "
\n", + " Memory usage: 63.73 MiB\n", + " \n", + " Spilled bytes: 0 B\n", + "
\n", + " Read bytes: 0.0 B\n", + " \n", + " Write bytes: 0.0 B\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 1

\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:43595\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:40663/status\n", + " \n", + " Memory: 7.74 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:39425\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-py5wco57\n", + "
\n", + " Tasks executing: \n", + " \n", + " Tasks in memory: \n", + "
\n", + " Tasks ready: \n", + " \n", + " Tasks in flight: \n", + "
\n", + " CPU usage: 0.0%\n", + " \n", + " Last seen: Just now\n", + "
\n", + " Memory usage: 63.33 MiB\n", + " \n", + " Spilled bytes: 0 B\n", + "
\n", + " Read bytes: 0.0 B\n", + " \n", + " Write bytes: 0.0 B\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 2

\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:43897\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:41813/status\n", + " \n", + " Memory: 7.74 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:40629\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-a1ahw76j\n", + "
\n", + " Tasks executing: \n", + " \n", + " Tasks in memory: \n", + "
\n", + " Tasks ready: \n", + " \n", + " Tasks in flight: \n", + "
\n", + " CPU usage: 0.0%\n", + " \n", + " Last seen: Just now\n", + "
\n", + " Memory usage: 63.75 MiB\n", + " \n", + " Spilled bytes: 0 B\n", + "
\n", + " Read bytes: 0.0 B\n", + " \n", + " Write bytes: 0.0 B\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "

Worker: 3

\n", + "
\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " \n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "
\n", + " Comm: tcp://127.0.0.1:41981\n", + " \n", + " Total threads: 1\n", + "
\n", + " Dashboard: http://127.0.0.1:37365/status\n", + " \n", + " Memory: 7.74 GiB\n", + "
\n", + " Nanny: tcp://127.0.0.1:38545\n", + "
\n", + " Local directory: /tmp/dask-scratch-space/worker-bgmaxbqf\n", + "
\n", + " Tasks executing: \n", + " \n", + " Tasks in memory: \n", + "
\n", + " Tasks ready: \n", + " \n", + " Tasks in flight: \n", + "
\n", + " CPU usage: 0.0%\n", + " \n", + " Last seen: Just now\n", + "
\n", + " Memory usage: 64.02 MiB\n", + " \n", + " Spilled bytes: 0 B\n", + "
\n", + " Read bytes: 0.0 B\n", + " \n", + " Write bytes: 0.0 B\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + " \n", + "\n", + "
\n", + "
" + ], + "text/plain": [ + "" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + }, { "data": { "text/html": [ @@ -632,12 +1085,6 @@ "execution_count": 5, "id": "0f6f37a0-4994-48b6-b930-7fb61e0d8db7", "metadata": { - "execution": { - "iopub.execute_input": "2025-06-04T16:18:52.070545Z", - "iopub.status.busy": "2025-06-04T16:18:52.070349Z", - "iopub.status.idle": "2025-06-04T16:19:09.447310Z", - "shell.execute_reply": "2025-06-04T16:19:09.446741Z" - }, "tags": [] }, "outputs": [ @@ -645,7 +1092,21 @@ "name": "stdout", "output_type": "stream", "text": [ - "[\u001b[38;2;128;05;128m2025-06-04 10:18:52,071\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n" + "[\u001b[38;2;128;05;128m2025-07-01 16:42:26,498\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:26,501\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m data/ea25_cal_small_after_fixed.split.point.zarr will be overwritten. \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,724\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Finished processing \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,760\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Module path: \u001b[38;2;50;50;205m/export/home/figs/vdesouza/work/Holography-1022/astrohack/src/astrohack\u001b[0m \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,768\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Creating output file name: data/ea25_cal_small_after_fixed.split.holog.zarr \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,771\u001b[0m] \u001b[38;2;255;160;0m WARNING\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m data/ea25_cal_small_after_fixed.split.holog.zarr will be overwritten. \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,826\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Processing ddi: 0, scans: [8 ... 57] \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:30,828\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Processing ddi: 1, scans: [8 ... 57] \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:50,573\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m worker_3: \u001b[0m EA25: DDI 0: Suggested cell size 2.47 amin, FOV: (1.11 deg, 1.11 deg) \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:50,579\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m worker_3: \u001b[0m EA06: DDI 0: Suggested cell size 2.47 amin, FOV: (1.11 deg, 1.11 deg) \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:51,244\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m worker_2: \u001b[0m EA25: DDI 1: Suggested cell size 2.20 amin, FOV: (1.11 deg, 1.11 deg) \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:51,250\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m worker_2: \u001b[0m EA06: DDI 1: Suggested cell size 2.20 amin, FOV: (1.11 deg, 1.11 deg) \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:52,688\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m worker_3: \u001b[0m Finished extracting holography chunk for ddi: 0 holog_map_key: map_0 \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:53,485\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m worker_2: \u001b[0m Finished extracting holography chunk for ddi: 1 holog_map_key: map_0 \n", + "[\u001b[38;2;128;05;128m2025-07-01 16:42:53,495\u001b[0m] \u001b[38;2;50;50;205m INFO\u001b[0m\u001b[38;2;112;128;144m astrohack: \u001b[0m Finished processing \n" ] }, { @@ -791,12 +1252,6 @@ "execution_count": 6, "id": "f4ce2575-5925-4822-b426-fc8b67580e9e", "metadata": { - "execution": { - "iopub.execute_input": "2025-06-04T16:19:09.449502Z", - "iopub.status.busy": "2025-06-04T16:19:09.448979Z", - "iopub.status.idle": "2025-06-04T16:19:09.480920Z", - "shell.execute_reply": "2025-06-04T16:19:09.480484Z" - }, "tags": [] }, "outputs": [ @@ -1186,17 +1641,11 @@ " VIS (time, chan, pol) complex128 37MB dask.array<chunksize=(2229, 16, 2), meta=np.ndarray>\n", " WEIGHT (time, chan, pol) float64 18MB dask.array<chunksize=(2229, 16, 2), meta=np.ndarray>\n", "Attributes:\n", - " antenna_name: ea25\n", " ddi: 0\n", - " grid_params: {'cell_size': 0.000719484970141879, 'n_pix': 729}\n", " holog_map_key: map_0\n", - " l_max: 0.009675024871632517\n", - " l_min: -0.009674930411087166\n", - " m_max: 0.009698666161358812\n", - " m_min: -0.00975229251000802\n", " parallactic_samples: [5.308157433326323, 5.357028871639436, 5.502977...\n", - " telescope_name: EVLA\n", - " time_smoothing_interval: 1.0