Skip to content

Commit b994239

Browse files
ascibiszascibisz
authored
Feature/client upload script (#417)
* add upload script * add example data and more documentation * point to correct collection * return firebase IDs from upload_recipe adn upload_config * add documentation * add installation step * lint fixes * lint fix * fix typos * keep all upload functionality in upload.py, and rename client to studio * remove accidential line * update example editable fields to match updates we've done * add output param * lint fixes * small wording fixes * make config_path optional
1 parent 1f8e7e4 commit b994239

5 files changed

Lines changed: 142 additions & 7 deletions

File tree

‎cellpack/autopack/DBRecipeHandler.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -515,6 +515,7 @@ def upload_recipe(self, recipe_meta_data, recipe_data):
515515
recipe_to_save = self.upload_collections(recipe_meta_data, recipe_data)
516516
recipe_to_save["recipe_path"] = self.db.create_path("recipes", recipe_id)
517517
self.upload_data("recipes", recipe_to_save, recipe_id)
518+
return recipe_id
518519

519520
def upload_config(self, config_data, source_path):
520521
"""
@@ -526,7 +527,7 @@ def upload_config(self, config_data, source_path):
526527
# update the config data with the firebase doc path
527528
config_data["config_path"] = doc_path
528529
self.db.update_doc("configs", id, config_data)
529-
return
530+
return id
530531

531532
def upload_result_metadata(self, file_name, url, job_id=None):
532533
"""

‎cellpack/autopack/upy/simularium/simularium_helper.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1397,21 +1397,23 @@ def post_and_open_file(self, file_name, open_results_in_browser):
13971397
simulariumHelper.open_in_simularium(url)
13981398

13991399
@staticmethod
1400-
def store_result_file(file_path, storage=None, batch_job_id=None):
1400+
def store_result_file(
1401+
file_path, storage=None, batch_job_id=None, sub_folder="simularium"
1402+
):
14011403
if storage == "aws":
14021404
handler = DATABASE_IDS.handlers().get(storage)
14031405
# if batch_job_id is not None, then we are in a batch job and should use the temp bucket
14041406
# TODO: use cellpack-results bucket for batch jobs once we have the correct permissions
14051407
if batch_job_id:
14061408
initialized_handler = handler(
14071409
bucket_name="cellpack-demo",
1408-
sub_folder_name="simularium",
1410+
sub_folder_name=sub_folder,
14091411
region_name="us-west-2",
14101412
)
14111413
else:
14121414
initialized_handler = handler(
14131415
bucket_name="cellpack-results",
1414-
sub_folder_name="simularium",
1416+
sub_folder_name=sub_folder,
14151417
region_name="us-west-2",
14161418
)
14171419
file_name, url = initialized_handler.save_file_and_get_url(file_path)

‎cellpack/bin/upload.py‎

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
import sys
22
import fire
3+
import json
34

45
from cellpack.autopack.FirebaseHandler import FirebaseHandler
56
from cellpack.autopack.DBRecipeHandler import DBUploader, DBMaintenance
6-
7+
from cellpack.autopack.upy.simularium.simularium_helper import simulariumHelper
78
from cellpack.autopack.interface_objects.database_ids import DATABASE_IDS
89
from cellpack.autopack.loaders.config_loader import ConfigLoader
910
from cellpack.autopack.loaders.recipe_loader import RecipeLoader
@@ -36,12 +37,40 @@ def upload(
3637
recipe_path=None,
3738
config_path=None,
3839
db_id=DATABASE_IDS.FIREBASE,
40+
studio: bool = False,
41+
fields: str = None,
42+
name: str = None,
43+
output_file: str = None,
3944
):
4045
"""
4146
Uploads a recipe to the database
4247
48+
:param recipe: string argument
49+
path to local recipe file to upload to firebase
50+
:param config: string argument
51+
path to local config file to upload to firebase
52+
:param db_id: string argument
53+
database ID to use for the upload
54+
:param studio: boolean argument
55+
upload for use in cellPACK studio if True
56+
:param fields: string argument
57+
path to local editable fields file to upload to firebase
58+
:param name: string argument
59+
display name for recipe in studio selection menu
60+
:param output_file: string argument
61+
path to local simularium output file to upload
62+
4363
:return: void
4464
"""
65+
if studio and (not name or not recipe_path or not fields):
66+
sys.exit(
67+
"To upload a recipe for cellPACK studio, please provide --name, --recipe_path, and --fields arguments."
68+
)
69+
70+
recipe_id = ""
71+
config_id = ""
72+
result_url = ""
73+
editable_fields_ids = []
4574
if db_id == DATABASE_IDS.FIREBASE:
4675
# fetch the service key json file
4776
db_handler = FirebaseHandler()
@@ -51,10 +80,29 @@ def upload(
5180
recipe_loader = RecipeLoader(recipe_path)
5281
recipe_full_data = recipe_loader._read(resolve_inheritance=False)
5382
recipe_meta_data = get_recipe_metadata(recipe_loader)
54-
db_handler.upload_recipe(recipe_meta_data, recipe_full_data)
83+
recipe_id = db_handler.upload_recipe(recipe_meta_data, recipe_full_data)
5584
if config_path:
5685
config_data = ConfigLoader(config_path).config
57-
db_handler.upload_config(config_data, config_path)
86+
config_id = db_handler.upload_config(config_data, config_path)
87+
if fields:
88+
editable_fields_data = json.load(open(fields, "r"))
89+
for field in editable_fields_data.get("editable_fields", []):
90+
id, _ = db_handler.upload_data("editable_fields", field)
91+
editable_fields_ids.append(id)
92+
if output_file:
93+
_, result_url = simulariumHelper.store_result_file(
94+
output_file, storage="aws", sub_folder="client"
95+
)
96+
if studio:
97+
recipe_metadata = {
98+
"name": name,
99+
"recipe": recipe_id,
100+
"config": config_id,
101+
"editable_fields": editable_fields_ids,
102+
"result_path": result_url,
103+
}
104+
# Upload the combined recipe metadata to example_packings collection for studio
105+
db_handler.upload_data("example_packings", recipe_metadata)
58106

59107
else:
60108
db_maintainer = DBMaintenance(db_handler)

‎docs/STUDIO_SITE.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# cellPACK Studio Site
2+
3+
cellPACK studio allows users to edit example recipes, run packings, and visualize results in the browser.
4+
5+
* Production site: https://cellpack.allencell.org/
6+
* GitHub repository: https://github.com/AllenCell/cellpack-client
7+
8+
### Publishing Recipes to cellPACK Studio
9+
To add new example recipes to the client site, the recipe and its associated metadata must be uploaded to Firebase.
10+
11+
1. Set up access to the staging Firebase account, following [these instructions](https://github.com/mesoscope/cellpack/blob/main/docs/REMOTE_DATABASES.md#firebase-firestore)
12+
2. Determine which recipe fields should be editable in the client, and create a JSON file specifying these editable fields following the schema outlined in [this example](https://github.com/mesoscope/cellpack/blob/0f140859824086d73edab008ff381b5e5717db8b/examples/client-data/example_editable_fields.json)
13+
3. Follow cellPACK installation requirements [in README](https://github.com/mesoscope/cellpack?tab=readme-ov-file#installation)
14+
4. Run the following script to upload necessary files to Firebase: `upload -s -r <path_to_recipe_file> -c <path_to_config_file> -f <path_to_editable_fields_file> -n "Name of Recipe" -o <local_path_to_simularium_result_file>`

‎examples/client-data/example_editable_fields.json‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
{
2+
"editable_fields": [
3+
{
4+
"name": "Display Name",
5+
"description": "Description text",
6+
"path": "path.to.field.in.recipe.json",
7+
"data_type": "integer/float/string",
8+
"input_type": "slider/dropdown/gradient"
9+
},
10+
{
11+
"name": "Endosome Count",
12+
"description": "Number of spheres to be packed",
13+
"path": "composition.membrane.regions.interior.1.count",
14+
"data_type": "integer",
15+
"input_type": "slider",
16+
"min": 0,
17+
"max": 300
18+
},
19+
{
20+
"name": "Endosome Radius",
21+
"description": "Radius of spheres",
22+
"path": "objects.endosome.radius",
23+
"data_type": "float",
24+
"input_type": "slider",
25+
"min": 0,
26+
"max": 10,
27+
"unit": "µm",
28+
"conversion_factor": 0.108
29+
},
30+
{
31+
"name": "Spatial Distribution Rule",
32+
"description": "Choose from one of the listed gradient types",
33+
"path": "objects.endosome.gradient",
34+
"data_type": "gradient",
35+
"input_type": "gradient",
36+
"gradient_options": [
37+
{
38+
"display_name": "Nuclear Bias",
39+
"value": "nucleus_gradient",
40+
"path": "objects.endosome.gradient",
41+
"packing_mode": "gradient",
42+
"packing_mode_path": "objects.endosome.packing_mode",
43+
"strength_path": "gradients.nucleus_gradient.weight_mode_settings.decay_length",
44+
"strength_display_name": "Decay Length",
45+
"strength_description": "Smaller decay length indicates stronger bias",
46+
"strength_min": 0,
47+
"strength_max": 1
48+
},
49+
{
50+
"display_name": "Membrane Bias",
51+
"value": "membrane_gradient",
52+
"path": "objects.endosome.gradient",
53+
"packing_mode": "gradient",
54+
"packing_mode_path": "objects.endosome.packing_mode",
55+
"strength_path": "gradients.membrane_gradient.weight_mode_settings.decay_length",
56+
"strength_display_name": "Decay Length",
57+
"strength_description": "Smaller decay length indicates stronger bias",
58+
"strength_min": 0,
59+
"strength_max": 1
60+
},
61+
{
62+
"display_name": "Unbiased",
63+
"value": "random",
64+
"path": "objects.endosome.packing_mode",
65+
"packing_mode": "random"
66+
}
67+
]
68+
}
69+
]
70+
}

0 commit comments

Comments
 (0)