Lightning is overwriting best models.

[aaprasad]

Hi there! I'm not sure if I've configured my model checkpointing incorrectly but for some reason lightning is overwriting my best models with models that are not as good? This is how I set up the checkpointing

def get_checkpointing(self) -> pl.callbacks.ModelCheckpoint:
        """Getter for lightning checkpointing callback.

        Returns:
            A lightning checkpointing callback with specified params
        """
        # convert to dict to enable extracting/removing params
        checkpoint_params = self.cfg.checkpointing
        logging_params = self.cfg.logging
        dirpath = checkpoint_params.pop("dirpath", None)
        if dirpath is None:
            if "group" in logging_params:
                dirpath = f"./models/{logging_params.group}/{logging_params.name}"
            else:
                dirpath = f"./models/{logging_params.name}"

        dirpath = Path(dirpath).resolve()
        if not Path(dirpath).exists():
            try:
                Path(dirpath).mkdir(parents=True, exist_ok=True)
            except OSError as e:
                logger.exception(
                    f"Cannot create a new folder. Check the permissions to the given Checkpoint directory. \n {e}"
                )
        with open_dict(checkpoint_params):
            _ = checkpoint_params.pop("dirpath", None)
            monitor = checkpoint_params.pop("monitor", ["val_loss"])
        checkpointers = []

        for metric in monitor:
            checkpointer = pl.callbacks.ModelCheckpoint(
                monitor=metric,
                dirpath=dirpath,
                filename=f"{{epoch}}-{{{metric}}}",
                **checkpoint_params,
            )
            checkpointer.CHECKPOINT_NAME_LAST = f"{{epoch}}-best-{{{metric}}}"
            checkpointers.append(checkpointer)
        return checkpointers

Along with the params I use:

checkpointing:
  monitor: ["val_loss","val_num_switches"]
  verbose: true
  save_last: true
  dirpath: null
  auto_insert_metric_name: true
  every_n_epochs: 10
  save_top_k: -1