22# Copyright (c) Microsoft Corporation. All rights reserved.
33# Licensed under the MIT License (MIT). See LICENSE in the repo root for license information.
44# ------------------------------------------------------------------------------------------
5+ from __future__ import annotations
6+
57import abc
6- import logging
7- import re
88from datetime import datetime
99from enum import Enum , unique
1010from pathlib import Path
11- from typing import Any , Dict , List , Optional , Tuple
12-
13- import numpy as np
11+ from typing import Any , Dict , List
1412
1513DATASET_CSV_FILE_NAME = "dataset.csv"
1614CHECKPOINT_SUFFIX = ".ckpt"
2624LAST_CHECKPOINT_FILE_NAME = "last"
2725LAST_CHECKPOINT_FILE_NAME_WITH_SUFFIX = LAST_CHECKPOINT_FILE_NAME + CHECKPOINT_SUFFIX
2826
27+ FINAL_MODEL_FOLDER = "final_model"
28+ FINAL_ENSEMBLE_MODEL_FOLDER = "final_ensemble_model"
29+ CHECKPOINT_FOLDER = "checkpoints"
30+ VISUALIZATION_FOLDER = "visualizations"
31+ EXTRA_RUN_SUBFOLDER = "extra_run_id"
32+ ARGS_TXT = "args.txt"
33+
2934
3035@unique
3136class ModelExecutionMode (Enum ):
@@ -64,18 +69,14 @@ def get_feature_length(self, column: str) -> int:
6469 raise NotImplementedError ("get_feature_length must be implemented by sub classes" )
6570
6671
67- def get_recovery_checkpoint_path ( path : Path ) -> Path :
72+ def create_unique_timestamp_id ( ) -> str :
6873 """
69- Returns the path to the last recovery checkpoint in the given folder or the provided filename. Raises a
70- FileNotFoundError if no
71- recovery checkpoint file is present.
72- :param path: Path to checkpoint folder
74+ Creates a unique string using the current time in UTC, up to seconds precision, with characters that
75+ are suitable for use in filenames. For example, on 31 Dec 2019 at 11:59:59pm UTC, the result would be
76+ 2019-12-31T235959Z.
7377 """
74- recovery_ckpt_and_epoch = find_recovery_checkpoint_and_epoch (path )
75- if recovery_ckpt_and_epoch is not None :
76- return recovery_ckpt_and_epoch [0 ]
77- files = list (path .glob ("*" ))
78- raise FileNotFoundError (f"No checkpoint files found in { path } { ' ' .join (p .name for p in files )} )
78+ unique_id = datetime .utcnow ().strftime ("%Y-%m-%dT%H%M%SZ" )
79+ return unique_id
7980
8081
8182def get_best_checkpoint_path (path : Path ) -> Path :
@@ -84,73 +85,3 @@ def get_best_checkpoint_path(path: Path) -> Path:
8485 :param path to checkpoint folder
8586 """
8687 return path / BEST_CHECKPOINT_FILE_NAME_WITH_SUFFIX
87-
88-
89- def find_all_recovery_checkpoints (path : Path ) -> Optional [List [Path ]]:
90- """
91- Extracts all file starting with RECOVERY_CHECKPOINT_FILE_NAME in path
92- :param path:
93- :return:
94- """
95- all_recovery_files = [f for f in path .glob (RECOVERY_CHECKPOINT_FILE_NAME + "*" )]
96- if len (all_recovery_files ) == 0 :
97- return None
98- return all_recovery_files
99-
100-
101- PathAndEpoch = Tuple [Path , int ]
102-
103-
104- def extract_latest_checkpoint_and_epoch (available_files : List [Path ]) -> PathAndEpoch :
105- """
106- Checkpoints are saved as recovery_epoch={epoch}.ckpt, find the latest ckpt and epoch number.
107- :param available_files: all available checkpoints
108- :return: path the checkpoint from latest epoch and epoch number
109- """
110- recovery_epochs = [int (re .findall (r"[\d]+" , f .stem )[0 ]) for f in available_files ]
111- idx_max_epoch = int (np .argmax (recovery_epochs ))
112- return available_files [idx_max_epoch ], recovery_epochs [idx_max_epoch ]
113-
114-
115- def find_recovery_checkpoint_and_epoch (path : Path ) -> Optional [PathAndEpoch ]:
116- """
117- Looks at all the recovery files, extracts the epoch number for all of them and returns the most recent (latest
118- epoch)
119- checkpoint path along with the corresponding epoch number. If no recovery checkpoint are found, return None.
120- :param path: The folder to start searching in.
121- :return: None if there is no file matching the search pattern, or a Tuple with Path object and integer pointing to
122- recovery checkpoint path and recovery epoch.
123- """
124- available_checkpoints = find_all_recovery_checkpoints (path )
125- if available_checkpoints is not None :
126- return extract_latest_checkpoint_and_epoch (available_checkpoints )
127- return None
128-
129-
130- def create_best_checkpoint (path : Path ) -> Path :
131- """
132- Creates the best checkpoint file. "Best" is at the moment defined as being the last checkpoint, but could be
133- based on some defined policy.
134- The best checkpoint will be renamed to `best_checkpoint.ckpt`.
135- :param path: The folder that contains all checkpoint files.
136- """
137- logging .debug (f"Files in checkpoint folder: { ' ' .join (p .name for p in path .glob ('*' ))} )
138- last_ckpt = path / LAST_CHECKPOINT_FILE_NAME_WITH_SUFFIX
139- all_files = f"Existing files: { ' ' .join (p .name for p in path .glob ('*' ))}
140- if not last_ckpt .is_file ():
141- raise FileNotFoundError (f"Checkpoint file { LAST_CHECKPOINT_FILE_NAME_WITH_SUFFIX } { all_files } )
142- logging .info (f"Using { LAST_CHECKPOINT_FILE_NAME_WITH_SUFFIX }
143- f"{ BEST_CHECKPOINT_FILE_NAME_WITH_SUFFIX } )
144- best = path / BEST_CHECKPOINT_FILE_NAME_WITH_SUFFIX
145- last_ckpt .rename (best )
146- return best
147-
148-
149- def create_unique_timestamp_id () -> str :
150- """
151- Creates a unique string using the current time in UTC, up to seconds precision, with characters that
152- are suitable for use in filenames. For example, on 31 Dec 2019 at 11:59:59pm UTC, the result would be
153- 2019-12-31T235959Z.
154- """
155- unique_id = datetime .utcnow ().strftime ("%Y-%m-%dT%H%M%SZ" )
156- return unique_id
0 commit comments