Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Allows user to provide the start/stop epoch in SaveModelCallback #3868

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Allows user to provide the start/stop epoch in SaveModelCallback #3868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4311783
nb changes
kunaltyagi Feb 1, 2023
806d1f5
update tracker.py
kunaltyagi Feb 1, 2023
eb1d3fd
add stop_epoch
kunaltyagi Feb 1, 2023
c5665d1
Update test
kunaltyagi Feb 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
10 changes: 7 additions & 3 deletions fastai/callback/tracker.py
View file
Open in desktop
Expand Up @@ -81,6 +81,8 @@ def __init__(self,
min_delta=0., # minimum delta between the last monitor value and the best monitor value.
fname='model', # model name to be used when saving model.
every_epoch=False, # if true, save model after every epoch; else save only when model is better than existing best.
start_epoch=0, # save model after every epoch once start_epoch have elapsed
stop_epoch=None, # don't save model after every epoch once stop_epoch have elapsed
at_end=False, # if true, save model when training ends; else load best model if there is only one saved model.
with_opt=False, # if true, save optimizer state (if any available) when saving model.
reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).
Expand All @@ -89,14 +91,16 @@ def __init__(self,
assert not (every_epoch and at_end), "every_epoch and at_end cannot both be set to True"
# keep track of file path for loggers
self.last_saved_path = None
store_attr('fname,every_epoch,at_end,with_opt')
store_attr('fname,every_epoch,start_epoch,stop_epoch,at_end,with_opt')

def _save(self, name): self.last_saved_path = self.learn.save(name, with_opt=self.with_opt)

def after_epoch(self):
"Compare the value monitored to its best score and save if best."
if self.every_epoch:
if (self.epoch%self.every_epoch) == 0: self._save(f'{self.fname}_{self.epoch}')
if self.every_epoch and self.epoch >= self.start_epoch:
if self.stop_epoch is not None and self.epoch > self.stop_epoch: return
epoch_offset = self.epoch - self.start_epoch
if (epoch_offset%self.every_epoch) == 0: self._save(f'{self.fname}_{self.epoch}')
else: #every improvement
super().after_epoch()
if self.new_best:
Expand Down
21 changes: 18 additions & 3 deletions nbs/17_callback.tracker.ipynb
View file
Open in desktop
Expand Up @@ -581,6 +581,8 @@
" min_delta=0., # minimum delta between the last monitor value and the best monitor value.\n",
" fname='model', # model name to be used when saving model.\n",
" every_epoch=False, # if true, save model after every epoch; else save only when model is better than existing best.\n",
" start_epoch=0, # save model after every epoch once start_epoch have elapsed\n",
" stop_epoch=None, # don't save model after every epoch once stop_epoch have elapsed\n",
" at_end=False, # if true, save model when training ends; else load best model if there is only one saved model.\n",
" with_opt=False, # if true, save optimizer state (if any available) when saving model. \n",
" reset_on_fit=True # before model fitting, reset value being monitored to -infinity (if monitor is metric) or +infinity (if monitor is loss).\n",
Expand All @@ -589,14 +591,16 @@
" assert not (every_epoch and at_end), \"every_epoch and at_end cannot both be set to True\"\n",
" # keep track of file path for loggers\n",
" self.last_saved_path = None\n",
" store_attr('fname,every_epoch,at_end,with_opt')\n",
" store_attr('fname,every_epoch,start_epoch,stop_epoch,at_end,with_opt')\n",
"\n",
" def _save(self, name): self.last_saved_path = self.learn.save(name, with_opt=self.with_opt)\n",
"\n",
" def after_epoch(self):\n",
" \"Compare the value monitored to its best score and save if best.\"\n",
" if self.every_epoch:\n",
" if (self.epoch%self.every_epoch) == 0: self._save(f'{self.fname}_{self.epoch}')\n",
" if self.every_epoch and self.epoch >= self.start_epoch:\n",
" if self.stop_epoch is not None and self.epoch > self.stop_epoch: return\n",
" epoch_offset = self.epoch - self.start_epoch\n",
" if (epoch_offset%self.every_epoch) == 0: self._save(f'{self.fname}_{self.epoch}')\n",
" else: #every improvement\n",
" super().after_epoch()\n",
" if self.new_best:\n",
Expand Down Expand Up @@ -804,6 +808,17 @@
"for i in range(4): \n",
" if not i%2: assert (Path.cwd()/f'tmp/models/model_{i}.pth').exists()\n",
" else: assert not (Path.cwd()/f'tmp/models/model_{i}.pth').exists()\n",
"shutil.rmtree(Path.cwd()/'tmp')\n",
"learn.fit(n_epoch=4, cbs=SaveModelCallback(every_epoch=2, start_epoch=1))\n",
"for i in range(4):\n",
" if i < 1: continue\n",
" if not (i-1) %2: assert (Path.cwd()/f'tmp/models/model_{i}.pth').exists()\n",
" else: assert not (Path.cwd()/f'tmp/models/model_{i}.pth').exists()\n",
"shutil.rmtree(Path.cwd()/'tmp')\n",
"learn.fit(n_epoch=4, cbs=SaveModelCallback(stop_epoch=2))\n",
"for i in range(4):\n",
" if i <= 2: assert (Path.cwd()/f'tmp/models/model_{i}.pth').exists()\n",
" else: assert not (Path.cwd()/f'tmp/models/model_{i}.pth').exists()\n",
"shutil.rmtree(Path.cwd()/'tmp')"
]
},
Expand Down