Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Very Minor updates #43

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Very Minor updates #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9a10c56
added train checkpoint interval as a tf Flag, default changed from 10…
looibot May 5, 2017
a8a997c
added train checkpoint interval as a tf Flag, default changed from 10…
looibot May 5, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions libs/configs/config_v1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@
'train_dir', './output/mask_rcnn/',
'Directory where checkpoints and event logs are written to.')

tf.app.flags.DEFINE_integer(
'train_checkpoint_interval', 3000,
'The number of steps per saved checkpoint.')

tf.app.flags.DEFINE_string(
'pretrained_model', './data/pretrained_models/resnet_v1_50.ckpt',
'Path to pretrained model')
Expand Down Expand Up @@ -42,7 +46,7 @@
'The name of the train/test/val split.')

tf.app.flags.DEFINE_string(
'dataset_dir', 'data/coco/',
'dataset_dir', './data/coco/',
'The directory where the dataset files are stored.')

tf.app.flags.DEFINE_integer(
Expand Down Expand Up @@ -130,7 +134,7 @@
'Specifies how the learning rate is decayed. One of "fixed", "exponential",'
' or "polynomial"')

tf.app.flags.DEFINE_float('learning_rate', 0.002,
tf.app.flags.DEFINE_float('learning_rate', 2e-3
'Initial learning rate.')

tf.app.flags.DEFINE_float(
Expand All @@ -143,8 +147,8 @@
tf.app.flags.DEFINE_float(
'learning_rate_decay_factor', 0.94, 'Learning rate decay factor.')

tf.app.flags.DEFINE_float(
'num_epochs_per_decay', 2.0,
tf.app.flags.DEFINE_integer(
'num_epochs_per_decay', 2,
'Number of epochs after which learning rate decays.')

tf.app.flags.DEFINE_bool(
Expand Down
13 changes: 9 additions & 4 deletions train/train.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,21 +63,23 @@ def solve(global_step):

def restore(sess):
"""choose which param to restore"""
is_restored = False
if FLAGS.restore_previous_if_exists:
try:
checkpoint_path = tf.train.latest_checkpoint(FLAGS.train_dir)
restorer = tf.train.Saver()
restorer.restore(sess, checkpoint_path)
is_restored = True
print ('restored previous model %s from %s'\
%(checkpoint_path, FLAGS.train_dir))
time.sleep(2)
return
except:
print ('--restore_previous_if_exists is set, but failed to restore in %s %s'\
print ('--restore_previous_if_exists is set, but FAILED TO RESTORE in %s %s'\
% (FLAGS.train_dir, checkpoint_path))
time.sleep(2)

if FLAGS.pretrained_model:
if FLAGS.pretrained_model and not is_restored:
if tf.gfile.IsDirectory(FLAGS.pretrained_model):
checkpoint_path = tf.train.latest_checkpoint(FLAGS.pretrained_model)
else:
Expand Down Expand Up @@ -105,6 +107,8 @@ def restore(sess):
def train():
"""The main function that runs training"""

print("Starting learning rate %.7f"%(FLAGS.learning_rate))

## data
image, ih, iw, gt_boxes, gt_masks, num_instances, img_id = \
datasets.get_dataset(FLAGS.dataset_name,
Expand Down Expand Up @@ -143,6 +147,7 @@ def train():

## solvers
global_step = slim.create_global_step()
# global_step = tf.Variable(0, name='global_step', trainable=False)
update_op = solve(global_step)

cropped_rois = tf.get_collection('__CROPPED__')[0]
Expand Down Expand Up @@ -208,10 +213,10 @@ def train():
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, step)

if (step % 10000 == 0 or step + 1 == FLAGS.max_iters) and step != 0:
if (step % FLAGS.train_checkpoint_interval == 0 or step + 1 == FLAGS.max_iters) and step != 0:
checkpoint_path = os.path.join(FLAGS.train_dir,
FLAGS.dataset_name + '_' + FLAGS.network + '_model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
saver.save(sess, checkpoint_path, global_step=global_step)

if coord.should_stop():
coord.request_stop()
Expand Down