Skip to content

Commit bf820a4

Browse files
jrhoskjrhosk
authored
Merge pull request #41
39 profiling tools
2 parents c04641d + e60bd81 commit bf820a4

37 files changed

Lines changed: 3238 additions & 1066 deletions

docs/add_entry.ipynb renamed to add_entry.ipynb

Lines changed: 12 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,16 @@
2323
"toolviper.utils.data.update(path=str(path))"
2424
]
2525
},
26+
{
27+
"cell_type": "code",
28+
"execution_count": null,
29+
"id": "0dda04ad-1ecc-4925-92bb-9a608f81d7e1",
30+
"metadata": {},
31+
"outputs": [],
32+
"source": [
33+
"toolviper.utils.data.get_file_size(str(path))"
34+
]
35+
},
2636
{
2737
"cell_type": "code",
2838
"execution_count": null,
@@ -33,11 +43,11 @@
3343
"entries = []\n",
3444
"\n",
3545
"entry = {\n",
36-
" \"file\": str(path.joinpath(\"ngc5921-lsrk-cube.psf.zip\")),\n",
46+
" \"file\": str(path.joinpath(\"upload/casa_no_sky_to_xds_true.zarr.zip\")),\n",
3747
" \"path\": \"radps/image\",\n",
3848
" \"dtype\": \"CASA image\",\n",
3949
" \"telescope\": \"VLA\",\n",
40-
" \"mode\": \"Interferometric\"\n",
50+
" \"mode\": \"Simulated\"\n",
4151
"}\n",
4252
"\n",
4353
"entries.append(entry)"
@@ -56,26 +66,6 @@
5666
" versioning=\"patch\"\n",
5767
")"
5868
]
59-
},
60-
{
61-
"cell_type": "code",
62-
"execution_count": null,
63-
"id": "a113bb2b-fed4-4ede-8d8b-353958f59602",
64-
"metadata": {},
65-
"outputs": [],
66-
"source": [
67-
"#toolviper.utils.data.update()\n",
68-
"\n",
69-
"#toolviper.utils.data.download(file=\"ngc5921-lsrk-cube.psf\", folder=\"test\")"
70-
]
71-
},
72-
{
73-
"cell_type": "code",
74-
"execution_count": null,
75-
"id": "7df91b02-3e25-4989-a76e-20ca4b7a9cb2",
76-
"metadata": {},
77-
"outputs": [],
78-
"source": []
7969
}
8070
],
8171
"metadata": {

‎docs/api.rst‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,4 @@ API
44
.. toctree::
55
:maxdepth: 2
66

7-
_api/autoapi/toolviper/dask/client/index
8-
_api/autoapi/toolviper/graph_tools/coordinate_utils/index
9-
_api/autoapi/toolviper/graph_tools/map/index
10-
_api/autoapi/toolviper/graph_tools/reduce/index
7+
_api/autoapi/toolviper/dask/client/index

‎docs/client_tutorial.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@
4040
},
4141
"outputs": [],
4242
"source": [
43-
"toolviper.utils.data.download(file=\"AA2-Mid-sim_00000.ms\")"
43+
"toolviper.utils.data.download(file=\"AA2-Mid-sim_00000.ms\", folder=\"data\")"
4444
]
4545
},
4646
{
@@ -197,7 +197,7 @@
197197
"name": "python",
198198
"nbconvert_exporter": "python",
199199
"pygments_lexer": "ipython3",
200-
"version": "3.11.13"
200+
"version": "3.12.12"
201201
}
202202
},
203203
"nbformat": 4,

‎docs/download_example.ipynb‎

Lines changed: 23 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -10,15 +10,6 @@
1010
"import toolviper"
1111
]
1212
},
13-
{
14-
"cell_type": "markdown",
15-
"id": "01bc8397-af47-48df-8391-733bba286588",
16-
"metadata": {},
17-
"source": [
18-
"## Getting download metadata version\n",
19-
"#### This will retireve the current file download metadata version; if the file is not found it will attempt to update to the most recent version."
20-
]
21-
},
2213
{
2314
"cell_type": "code",
2415
"execution_count": null,
@@ -31,50 +22,58 @@
3122
},
3223
{
3324
"cell_type": "markdown",
34-
"id": "fe18a0d9-bd21-4ce5-91fd-6348e5f1369e",
25+
"id": "0351dda7-7559-449d-9ebb-c853beb0861e",
3526
"metadata": {},
3627
"source": [
37-
"### Manually update metdata info."
28+
"## Getting available downloadable file in a python list.\n",
29+
"#### This will return an unordered list of the available file on the remote dropbox in a python list. This can be used as an input to thie download function as well."
3830
]
3931
},
4032
{
4133
"cell_type": "code",
4234
"execution_count": null,
43-
"id": "deeee662-94d9-44a0-95e7-f062f55223ca",
35+
"id": "2912a5ae-7a4b-42f3-a633-cbe5ecaffc74",
4436
"metadata": {},
4537
"outputs": [],
4638
"source": [
47-
"toolviper.utils.data.update()"
39+
"files = toolviper.utils.data.get_files()"
40+
]
41+
},
42+
{
43+
"cell_type": "code",
44+
"execution_count": null,
45+
"id": "28c8b343-3a05-4217-a581-e51c7db95d02",
46+
"metadata": {},
47+
"outputs": [],
48+
"source": [
49+
"toolviper.utils.data.download(file=files[3:8], folder=\"data\")"
4850
]
4951
},
5052
{
5153
"cell_type": "markdown",
52-
"id": "0351dda7-7559-449d-9ebb-c853beb0861e",
54+
"id": "01bc8397-af47-48df-8391-733bba286588",
5355
"metadata": {},
5456
"source": [
55-
"## Getting available downloadable file in a python list.\n",
56-
"#### This will return an unordered list of the available file on the remote dropbox in a python list. This can be used as an input to thie download function as well."
57+
"## Getting download metadata version\n",
58+
"#### This will retireve the current file download metadata version; if the file is not found it will attempt to update to the most recent version."
5759
]
5860
},
5961
{
60-
"cell_type": "code",
61-
"execution_count": null,
62-
"id": "2912a5ae-7a4b-42f3-a633-cbe5ecaffc74",
62+
"cell_type": "markdown",
63+
"id": "fe18a0d9-bd21-4ce5-91fd-6348e5f1369e",
6364
"metadata": {},
64-
"outputs": [],
6565
"source": [
66-
"files = toolviper.utils.data.get_files()\n",
67-
"files"
66+
"### Manually update metdata info."
6867
]
6968
},
7069
{
7170
"cell_type": "code",
7271
"execution_count": null,
73-
"id": "28c8b343-3a05-4217-a581-e51c7db95d02",
72+
"id": "d521c952-7f30-4454-8548-de5ac5b98d82",
7473
"metadata": {},
7574
"outputs": [],
7675
"source": [
77-
"toolviper.utils.data.download(file=files[6:8], folder=\"data\")"
76+
"toolviper.utils.data.update()"
7877
]
7978
},
8079
{

docs/Example/ascii/snek.txt renamed to docs/example/ascii/snek.txt

File renamed without changes.

docs/Example/config/viper.param.json renamed to docs/example/config/viper.param.json

File renamed without changes.

docs/Example/viper.py renamed to docs/example/viper.py

File renamed without changes.

‎docs/file-manifest-update.ipynb‎

Lines changed: 35 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,19 @@
22
"cells": [
33
{
44
"cell_type": "code",
5-
"execution_count": null,
65
"id": "33d17704-1b0c-40dc-929c-75338f83c3c2",
76
"metadata": {},
8-
"outputs": [],
97
"source": [
108
"import toolviper\n",
119
"import pathlib"
12-
]
10+
],
11+
"outputs": [],
12+
"execution_count": null
1313
},
1414
{
1515
"cell_type": "code",
16-
"execution_count": null,
1716
"id": "7b0ca451-df41-4d13-ac4c-3e2b7601eba4",
1817
"metadata": {},
19-
"outputs": [],
2018
"source": [
2119
"def make_random_file(file):\n",
2220
" import random\n",
@@ -27,46 +25,57 @@
2725
" handle.write(random.randbytes(1024))\n",
2826
"\n",
2927
" subprocess.run([\"zip\", \"-r\", f\"{file}.zip\", file])"
30-
]
28+
],
29+
"outputs": [],
30+
"execution_count": null
3131
},
3232
{
3333
"cell_type": "code",
34-
"execution_count": null,
3534
"id": "937a9e64-cc8a-4283-a3d8-1f5118592d52",
3635
"metadata": {},
37-
"outputs": [],
3836
"source": [
3937
"path = pathlib.Path().cwd()\n",
4038
"\n",
4139
"toolviper.utils.data.update(path=str(path))"
42-
]
40+
],
41+
"outputs": [],
42+
"execution_count": null
4343
},
4444
{
4545
"cell_type": "code",
46-
"execution_count": null,
47-
"id": "8e66f912-d356-486b-8992-6e3ce62ad672",
46+
"id": "8589d72e-c594-400a-8660-f54336b9c4d6",
4847
"metadata": {},
48+
"source": [
49+
"_json = toolviper.utils.tools.open_json(\"file.download.json\")\n",
50+
"toolviper.utils.display.DataDict.html(_json[\"metadata\"])"
51+
],
4952
"outputs": [],
53+
"execution_count": null
54+
},
55+
{
56+
"cell_type": "code",
57+
"id": "8e66f912-d356-486b-8992-6e3ce62ad672",
58+
"metadata": {},
5059
"source": [
5160
"make_random_file(file=\"single-dish.ultra.calibrated.ms\")"
52-
]
61+
],
62+
"outputs": [],
63+
"execution_count": null
5364
},
5465
{
5566
"cell_type": "code",
56-
"execution_count": null,
5767
"id": "05c07272-88ee-424a-9219-78e3ea52a0d7",
5868
"metadata": {},
59-
"outputs": [],
6069
"source": [
6170
"make_random_file(file=\"alma.mega.uncalibrated.ms\")"
62-
]
71+
],
72+
"outputs": [],
73+
"execution_count": null
6374
},
6475
{
6576
"cell_type": "code",
66-
"execution_count": null,
6777
"id": "7fd8b0b1-302b-454d-921b-9929af36e44e",
6878
"metadata": {},
69-
"outputs": [],
7079
"source": [
7180
"entries = []\n",
7281
"\n",
@@ -79,14 +88,14 @@
7988
"}\n",
8089
"\n",
8190
"entries.append(entry)"
82-
]
91+
],
92+
"outputs": [],
93+
"execution_count": null
8394
},
8495
{
8596
"cell_type": "code",
86-
"execution_count": null,
8797
"id": "d2bc0fd4-64c3-42e4-9422-9c6dd69a9c32",
8898
"metadata": {},
89-
"outputs": [],
9099
"source": [
91100
"entry = {\n",
92101
" \"file\": str(path.joinpath(\"alma.mega.uncalibrated.ms.zip\")),\n",
@@ -97,21 +106,23 @@
97106
"}\n",
98107
"\n",
99108
"entries.append(entry)"
100-
]
109+
],
110+
"outputs": [],
111+
"execution_count": null
101112
},
102113
{
103114
"cell_type": "code",
104-
"execution_count": null,
105115
"id": "0e98fb83-eb71-408d-b476-1d821d14b7f0",
106116
"metadata": {},
107-
"outputs": [],
108117
"source": [
109118
"_ = toolviper.utils.tools.add_entry(\n",
110119
" entries=entries,\n",
111120
" manifest=str(path.joinpath(\"file.download.json\")),\n",
112121
" versioning=\"patch\"\n",
113122
")"
114-
]
123+
],
124+
"outputs": [],
125+
"execution_count": null
115126
}
116127
],
117128
"metadata": {

0 commit comments

Comments
 (0)