From 694e79f11d29b1782fd05b74edd9e02e41398d1e Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Thu, 21 Jul 2022 10:26:36 -0400 Subject: [PATCH 01/10] WIP: implement orion, doesn't work yet --- bigearthnet/configs/sweeper/orion.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 bigearthnet/configs/sweeper/orion.yaml diff --git a/bigearthnet/configs/sweeper/orion.yaml b/bigearthnet/configs/sweeper/orion.yaml new file mode 100644 index 0000000..7023d85 --- /dev/null +++ b/bigearthnet/configs/sweeper/orion.yaml @@ -0,0 +1,24 @@ +hydra: +sweeper: + orion: + name: 'experiment' + version: '1' + + algorithm: + type: random + config: + seed: 1 + + worker: + n_workers: -1 + max_broken: 3 + max_trials: 100 + + storage: + type: pickledb + host: 'database.pkl' + + # default parametrization of the search space + parametrization: + optimizer: + lr: "uniform(0.00001, 0.01)" From 5c81777a7dad5be3911c66ff1fac9da256343458 Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Thu, 21 Jul 2022 10:26:53 -0400 Subject: [PATCH 02/10] update requirements --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index a498ca7..0f4b799 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,6 @@ "gitpython", "hub", "hydra-core>=1.2", - "hydra-joblib-launcher", "jupyter", "matplotlib", "pyyaml>=5.3", From e2efee8c980e90e3abc124e35e173e5a6cb44c61 Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Thu, 21 Jul 2022 13:09:42 -0400 Subject: [PATCH 03/10] first working pass --- bigearthnet/configs/config.yaml | 28 +++++++++++++++++++++++++- bigearthnet/configs/sweeper/orion.yaml | 2 +- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/bigearthnet/configs/config.yaml b/bigearthnet/configs/config.yaml index 1218cca..b6a6119 100644 --- a/bigearthnet/configs/config.yaml +++ b/bigearthnet/configs/config.yaml @@ -8,7 +8,8 @@ datamodule: optimizer: name: 'adam' # adam or sgd - lr: 0.0001 + # lr: 0.0001 + lr: ${lr} # set by orion logger: _target_: pytorch_lightning.loggers.TensorBoardLogger @@ -47,6 +48,8 @@ trainer: max_epochs: 3 profiler: "pytorch" +lr: 0.0001 + hydra: run: dir: outputs/${datamodule.dataset_name}/${now:%Y-%m-%dT%H:%M:%S} @@ -54,8 +57,31 @@ hydra: dir: outputs/${datamodule.dataset_name}/${now:%Y-%m-%dT%H:%M:%S}/multirun/ job: chdir: true + sweeper: + orion: + name: 'experiment' + version: '1' + + algorithm: + type: random + config: + seed: 1 + + worker: + n_workers: -1 + max_broken: 3 + max_trials: 3 + + storage: + type: pickledb + host: 'database.pkl' + + # default parametrization of the search space + parametrization: + lr: "uniform(0, 1)" defaults: - _self_ - model: baseline.yaml - transforms: norm.yaml + - override hydra/sweeper: orion diff --git a/bigearthnet/configs/sweeper/orion.yaml b/bigearthnet/configs/sweeper/orion.yaml index 7023d85..6e5a10b 100644 --- a/bigearthnet/configs/sweeper/orion.yaml +++ b/bigearthnet/configs/sweeper/orion.yaml @@ -12,7 +12,7 @@ sweeper: worker: n_workers: -1 max_broken: 3 - max_trials: 100 + max_trials: 10 storage: type: pickledb From d8c392a189e8f636db2c16b8973fd6dc3f2fa6c2 Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Fri, 22 Jul 2022 23:22:07 -0400 Subject: [PATCH 04/10] return best metric for orion --- bigearthnet/train.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigearthnet/train.py b/bigearthnet/train.py index 8921d9b..68141e7 100644 --- a/bigearthnet/train.py +++ b/bigearthnet/train.py @@ -36,6 +36,8 @@ def main(cfg: DictConfig): trainer.test(ckpt_path=ckpt_path, datamodule=datamodule) log.info("Test evaluation Done.") + return model.val_best_metric + if __name__ == "__main__": main() From c287695f7ced94db85c07d020c918aea5f400d3b Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Sat, 23 Jul 2022 09:52:44 -0400 Subject: [PATCH 05/10] move orion config to proper folder --- bigearthnet/configs/config.yaml | 24 +------------------- bigearthnet/configs/hydra/sweeper/orion.yaml | 21 +++++++++++++++++ bigearthnet/configs/sweeper/orion.yaml | 24 -------------------- 3 files changed, 22 insertions(+), 47 deletions(-) create mode 100644 bigearthnet/configs/hydra/sweeper/orion.yaml delete mode 100644 bigearthnet/configs/sweeper/orion.yaml diff --git a/bigearthnet/configs/config.yaml b/bigearthnet/configs/config.yaml index b6a6119..d6ba395 100644 --- a/bigearthnet/configs/config.yaml +++ b/bigearthnet/configs/config.yaml @@ -8,7 +8,6 @@ datamodule: optimizer: name: 'adam' # adam or sgd - # lr: 0.0001 lr: ${lr} # set by orion logger: @@ -48,6 +47,7 @@ trainer: max_epochs: 3 profiler: "pytorch" +# Orion doesn't seem to support overriding nested attributes lr: 0.0001 hydra: @@ -57,28 +57,6 @@ hydra: dir: outputs/${datamodule.dataset_name}/${now:%Y-%m-%dT%H:%M:%S}/multirun/ job: chdir: true - sweeper: - orion: - name: 'experiment' - version: '1' - - algorithm: - type: random - config: - seed: 1 - - worker: - n_workers: -1 - max_broken: 3 - max_trials: 3 - - storage: - type: pickledb - host: 'database.pkl' - - # default parametrization of the search space - parametrization: - lr: "uniform(0, 1)" defaults: - _self_ diff --git a/bigearthnet/configs/hydra/sweeper/orion.yaml b/bigearthnet/configs/hydra/sweeper/orion.yaml new file mode 100644 index 0000000..238e143 --- /dev/null +++ b/bigearthnet/configs/hydra/sweeper/orion.yaml @@ -0,0 +1,21 @@ +orion: + name: 'experiment' + version: '1' + +algorithm: + type: random + config: + seed: 1 + +worker: + n_workers: -1 + max_broken: 3 + max_trials: 3 + +storage: + type: pickledb + host: 'database.pkl' + +# default parametrization of the search space +parametrization: + lr: "uniform(0, 1)" diff --git a/bigearthnet/configs/sweeper/orion.yaml b/bigearthnet/configs/sweeper/orion.yaml deleted file mode 100644 index 6e5a10b..0000000 --- a/bigearthnet/configs/sweeper/orion.yaml +++ /dev/null @@ -1,24 +0,0 @@ -hydra: -sweeper: - orion: - name: 'experiment' - version: '1' - - algorithm: - type: random - config: - seed: 1 - - worker: - n_workers: -1 - max_broken: 3 - max_trials: 10 - - storage: - type: pickledb - host: 'database.pkl' - - # default parametrization of the search space - parametrization: - optimizer: - lr: "uniform(0.00001, 0.01)" From 7b84fce6bab35141ea574f70730014e9cd106806 Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Tue, 26 Jul 2022 11:37:19 -0400 Subject: [PATCH 06/10] update orion config for latest version --- bigearthnet/configs/config.yaml | 2 +- bigearthnet/configs/hydra/sweeper/orion.yaml | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/bigearthnet/configs/config.yaml b/bigearthnet/configs/config.yaml index d6ba395..353a723 100644 --- a/bigearthnet/configs/config.yaml +++ b/bigearthnet/configs/config.yaml @@ -8,7 +8,7 @@ datamodule: optimizer: name: 'adam' # adam or sgd - lr: ${lr} # set by orion + lr: 0.0001 logger: _target_: pytorch_lightning.loggers.TensorBoardLogger diff --git a/bigearthnet/configs/hydra/sweeper/orion.yaml b/bigearthnet/configs/hydra/sweeper/orion.yaml index 238e143..8936d56 100644 --- a/bigearthnet/configs/hydra/sweeper/orion.yaml +++ b/bigearthnet/configs/hydra/sweeper/orion.yaml @@ -13,9 +13,16 @@ worker: max_trials: 3 storage: - type: pickledb - host: 'database.pkl' + type: legacy -# default parametrization of the search space + database: + type: pickleddb + host: 'database.pkl' + +# parametrization of the hyperparameter space parametrization: - lr: "uniform(0, 1)" + datamodule: + batch_size: "choices([64, 128, 256])" + optimizer: + lr: "uniform(0, 1)" + name: "choices(['adam', 'sgd'])" From 1ed6f018843d1b3d3a226deae4d25c7a8e92eaa5 Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Tue, 26 Jul 2022 13:20:14 -0400 Subject: [PATCH 07/10] remove unused parameter --- bigearthnet/configs/config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/bigearthnet/configs/config.yaml b/bigearthnet/configs/config.yaml index 353a723..b1b29ac 100644 --- a/bigearthnet/configs/config.yaml +++ b/bigearthnet/configs/config.yaml @@ -47,8 +47,6 @@ trainer: max_epochs: 3 profiler: "pytorch" -# Orion doesn't seem to support overriding nested attributes -lr: 0.0001 hydra: run: From ff5f2f467cf105b9cbe0fb6a5d406a5fc7fae664 Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Wed, 19 Oct 2022 13:50:32 -0400 Subject: [PATCH 08/10] Add orion dependency --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 1deec00..c3a77d7 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ "jupyter", "matplotlib", "numpy>=1.23", + "orion", "pyyaml>=5.3", "pytest", "pytorch_lightning==1.6.4", From 3c24f0ce78449048ce48d6ae289b09cad49c799e Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Wed, 19 Oct 2022 13:50:53 -0400 Subject: [PATCH 09/10] Fix default order --- bigearthnet/configs/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bigearthnet/configs/config.yaml b/bigearthnet/configs/config.yaml index d960004..0de3e08 100644 --- a/bigearthnet/configs/config.yaml +++ b/bigearthnet/configs/config.yaml @@ -65,6 +65,6 @@ hydra: defaults: - _self_ - - override hydra/sweeper: orion - model: baseline.yaml # uses the default baseline model when training - - transforms: norm.yaml # performs normalization as default transform \ No newline at end of file + - transforms: norm.yaml # performs normalization as default transform + - override hydra/sweeper: orion From 4896806c935c841775da202c7c35e52160a7b8b8 Mon Sep 17 00:00:00 2001 From: Jeremy Pinto Date: Thu, 20 Oct 2022 09:20:32 -0400 Subject: [PATCH 10/10] return metric to optimize --- bigearthnet/train.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bigearthnet/train.py b/bigearthnet/train.py index d60cd94..485d302 100644 --- a/bigearthnet/train.py +++ b/bigearthnet/train.py @@ -29,6 +29,8 @@ def main(cfg: DictConfig): trainer.fit(model, datamodule=datamodule) log.info("Training Done.") + return model.val_best_metric + if __name__ == "__main__": main()