deploy changes

Author: github-actions[bot]

asparagus/pipeline/run/pretrain.py

@@ -113,15 +113,6 @@ def main(cfg: DictConfig) -> None:
         log_images_every_n_epoch=cfg.logger.log_images_every_n_epoch,
     )
 
-    print("Training duration configured as:")
-    print(f"  - Steps: {cfg.training.steps}")
-    print(f"  - Steps per pseudo epoch: {cfg.training.steps_per_epoch}")
-    print(f"  - Validation steps per pseudo epoch: {cfg.training.val_steps_per_epoch}")
-    print(
-        f"  - Pseudo Epochs: {cfg.training.steps / (cfg.training.steps_per_epoch * cfg.training.accumulate_grad_batches):.1f}"
-    )
-    print(f"  - Warmup Pseudo Epochs: {cfg.training.warmup_epochs} (ratio {cfg.training.warmup_ratio})")
-
     trainer = instantiate(
         cfg.lightning._trainer,
         callbacks=callbacks,
@@ -136,6 +127,19 @@ def main(cfg: DictConfig) -> None:
         accumulate_grad_batches=cfg.training.accumulate_grad_batches,
     )
 
+    if trainer.is_global_zero:
+        print("Training duration configured as:")
+        print(f"  - Steps: {cfg.training.steps}")
+        print(f"  - Global batch size: {cfg.training.global_batch_size}")
+        print(f"  - Steps per pseudo epoch: {cfg.training.steps_per_epoch}")
+        print(f"  - Validation steps per pseudo epoch: {cfg.training.val_steps_per_epoch}")
+        print(
+            "  - Pseudo Epochs: {:.1f}".format(
+                cfg.training.steps / (cfg.training.steps_per_epoch * cfg.training.accumulate_grad_batches)
+            )
+        )
+        print(f"  - Warmup Pseudo Epochs: {cfg.training.warmup_epochs} (ratio {cfg.training.warmup_ratio})")
+
     trainer.fit(
         model=model_module,
         datamodule=data_module,

configs/default_pretrain.yaml

@@ -39,11 +39,12 @@ training:
   # when we increase the number of devices _or_ use a bigger dataset.
   steps_per_epoch: 1890  # <--- should be constant ... but note that if gradient accumulation is used, then steps_per_epoch > number of backwards passes.
   val_steps_per_epoch: ${eval:"${training.steps_per_epoch} // 100"}
-  steps: ${eval:"${training.max_samples} // (${hardware.num_devices} * ${hardware.num_nodes} * ${training.batch_size} * ${training.accumulate_grad_batches})"}
+  global_batch_size: ${eval:"${training.batch_size} * ${hardware.num_devices} * ${hardware.num_nodes} * ${training.accumulate_grad_batches}"}
+  steps: ${eval:"${training.max_samples} // (${training.global_batch_size})"}
   warmup_epochs: ${eval:"max(1, int((${training.steps} // ${training.steps_per_epoch}) * ${training.warmup_ratio}))"}
   decoder_warmup_epochs: 0
   rec_loss_masked_only: False
-  check_val_every_n_epoch: 1
+  check_val_every_n_epoch: 3
 
 # num_devices 4
 # num_nodes 2