# Basic imports import os import time import sys import argparse import tensorflow as tf import numpy as np import multiprocessing as mp # Tensorflow ops imports from tensorflow.python.ops import clip_ops from tensorflow.python.ops import init_ops from tensorflow.python.ops import control_flow_ops from tensorflow.python.ops import variable_scope as vs from tensorflow.python.ops import variables as vars_ from tensorflow.python.training import queue_runner_impl # Custom imports from models import * from utils import initialize_from_dict, save_checkpoint, load_checkpoint, make_dir, Metrics from Queue import Queue from utils.logger import Logger from random import shuffle from utils.load_dataset_tfrecords import load_dataset from utils.argument_utils import read_json, assign_args parser = argparse.ArgumentParser() # Argument loading parser.add_argument('--argsFile', action= 'store', type=str, default='none', help= 'The name of the file that contains a model\'s arguments. Also requires --model.') parser.add_argument('--model', action= 'store', required=True, help= 'Model architecture (c3d, tsn, i3d, resnet)') args_init = parser.parse_known_args()[0] model_name = args_init.model args_file = args_init.argsFile args_json = read_json(model_name, args_file) json_keys = args_json.keys() # Model parameters parser.add_argument('--inputDims', action='store', required='inputDims' not in json_keys, type=int, help = 'Input Dimensions (Number of frames to pass as input to the model)') parser.add_argument('--outputDims', action='store', required='outputDims' not in json_keys, type=int, help = 'Output Dimensions (Number of classes in dataset)') parser.add_argument('--seqLength', action='store', required='seqLength' not in json_keys, type=int, help = 'Number of output frames expected from model') parser.add_argument('--modelAlpha', action='store', type=float, default=1., help = 'Resampling factor for constant value resampling and alpha initialization') parser.add_argument('--inputAlpha', action='store', type=float, default=1., help = 'Resampling factor for constant value resampling of input video, used mainly for testing models.') parser.add_argument('--dropoutRate', action='store', type=float, default=0.5, help = 'Value indicating proability of keeping inputs of the model\'s dropout layers.') parser.add_argument('--freeze', action='store', type=int, default=0, help = 'Freeze weights during training of any layers within the model that have the option set. (default 0)') parser.add_argument('--loadWeights', action='store', type=str, default='default', help = 'String which can be used to specify the default weights to load.') # Optimization parameters parser.add_argument('--lr', action='store', type=float, default=0.001, help = 'Learning Rate') parser.add_argument('--wd', action='store', type=float, default=0.0, help = 'Weight Decay') parser.add_argument('--lossType', action='store', default='full_loss', help = 'String defining loss type associated with chosen model.') parser.add_argument('--returnLayer', nargs='+',type=str, default=['logits'], help = 'Which model layers to be returned by the model\'s inference during testing.') parser.add_argument('--optChoice', action='store', default='default', help = 'String indicating optimizer choice') parser.add_argument('--gradClipValue', action='store', type=float, default=5.0, help = 'Value of normalized gradient at which to clip.') # Experiment parameters parser.add_argument('--dataset', action= 'store', required='dataset' not in json_keys, help= 'Dataset (UCF101, HMDB51)') parser.add_argument('--numGpus', action= 'store', type=int, default=1, help = 'Number of Gpus used for calculation') parser.add_argument('--gpuList', nargs='+',type=str, default=[], help = 'List of GPU IDs to be used') parser.add_argument('--train', action= 'store', type=int, default=1, help = 'Binary value to indicate training or evaluation instance') parser.add_argument('--load', action='store', type=int, default=0, help = 'Whether you want to load a saved model to train from scratch.') parser.add_argument('--loadedCheckpoint', action='store', type=int, default=-1, help = 'Specify the step of the saved model checkpoint that will be loaded for testing. Defaults to most recent checkpoint.') parser.add_argument('--size', action='store', required='size' not in json_keys, type=int, help = 'Input frame size') parser.add_argument('--expName', action='store', required='expName' not in json_keys, help = 'Unique name of experiment being run') parser.add_argument('--numVids', action='store', required='numVids' not in json_keys, type=int, help = 'Number of videos to be used for training') parser.add_argument('--nEpochs', action='store', type=int, default=1, help = 'Number of Epochs') parser.add_argument('--split', action='store', type=int, default=1, help = 'Dataset split to use') parser.add_argument('--baseDataPath', action='store', default='/z/dat', help = 'Path to datasets') parser.add_argument('--fName', action='store', required='fName' not in json_keys, help = 'Which dataset list to use (trainlist, testlist, vallist)') parser.add_argument('--saveFreq', action='store', type=int, default=1, help = 'Frequency in epochs to save model checkpoints') parser.add_argument('--clipLength', action='store', type=int, default=-1, help = 'Length of clips to cut video into, -1 indicates using the entire video as one clip') parser.add_argument('--videoOffset', action='store', default='none', help = '(none or random) indicating where to begin selecting video clips assuming clipOffset is none') parser.add_argument('--clipOffset', action='store', default='none', help = '(none or random) indicating if clips are selected sequentially or randomly') parser.add_argument('--clipStride', action='store', type=int, default=0, help = 'Number of frames that overlap between clips, 0 indicates no overlap and negative values indicate a gap of frames between clips') parser.add_argument('--numClips', action='store', type=int, default=-1, help = 'Number of clips to break video into, -1 indicates breaking the video into the maximum number of clips based on clipLength, clipStride, and clipOffset') parser.add_argument('--batchSize', action='store', type=int, default=1, help = 'Number of clips to load into the model each step.') parser.add_argument('--metricsDir', action='store', type=str, default='default', help = 'Name of sub directory within experiment to store metrics. Unique directory names allow for parallel testing.') parser.add_argument('--metricsMethod', action='store', default='avg_pooling', help = 'Which method to use to calculate accuracy metrics. (avg_pooling, last_frame, svm, svm_train or extract_features)') parser.add_argument('--preprocMethod', action='store', default='default', help = 'Which preprocessing method to use (default, cvr, rr, sr are options for existing models)') parser.add_argument('--randomInit', action='store', type=int, default=0, help = 'Randomly initialize model weights, not loading from any files (deafult False)') parser.add_argument('--shuffleSeed', action='store', type=int, default=0, help = 'Seed integer for random shuffle of files in load_dataset function') parser.add_argument('--preprocDebugging', action='store', type=int, default=0, help = 'Boolean indicating whether to load videos and clips in a queue or to load them directly for debugging (Default 0)') parser.add_argument('--verbose', action='store', type=int, default=1, help = 'Boolean switch to display all print statements or not') parser.add_argument('--save', action='store', type=int, default=1, help = 'Boolean indicating whether to save results, metrics, and logs. Used to test code.') parser.add_argument('--reverse', action='store', type=int, default=0, help = 'Boolean indicating whether reverse videos and classify them as a new action class. 0 all videos are forward, 1 randomly reversed videos, 2 all videos are reversed') args = parser.parse_args() args = assign_args(args, args_json, sys.argv) if args.verbose: print "Setup of current experiments" print "\n############################" print args print "############################ \n" # END IF save_bool = args.save model = models_import.create_model_object(modelName = model_name, inputAlpha = args.inputAlpha, modelAlpha = args.modelAlpha, clipLength = args.clipLength, numVids = args.numVids, numEpochs = args.nEpochs, batchSize = args.batchSize, numClips = args.numClips, numGpus = args.numGpus, train = args.train, expName = args.expName, outputDims = args.outputDims, inputDims = args.inputDims, preprocMethod = args.preprocMethod, dropoutRate = args.dropoutRate, freeze = args.freeze, loadWeights = args.loadWeights, verbose = args.verbose) def _average_gradients(tower_grads): """ Calculate the average gradient for each shared variable across all towers. Note that this function provides a synchronization point across all towers. Args: tower_grads: List of lists of (gradient, variable) tuples. The outer list is over individual gradients. The inner list is over the gradient calculation for each tower. Returns: List of pairs of (gradient, variable) where the gradient has been averaged across all towers. """ average_grads = [] for grad_and_vars in zip(*tower_grads): # Note that each grad_and_vars looks like the following: # ((grad0_gpu0, var0_gpu0), ... , (grad0_gpuN, var0_gpuN)) grads = [] for g, _ in grad_and_vars: # Add 0 dimension to the gradients to represent the tower. expanded_g = tf.expand_dims(g, 0) # Append on a 'tower' dimension which we will average over below. grads.append(expanded_g) # END FOR # Average over the 'tower' dimension. grad = tf.concat(axis=0, values=grads) grad = tf.reduce_mean(grad, 0) # Keep in mind that the Variables are redundant because they are shared # across towers. So .. we will just return the first tower's pointer to # the Variable. v = grad_and_vars[0][1] grad_and_var = (grad, v) average_grads.append(grad_and_var) # END FOR return average_grads def train(model, input_dims, output_dims, seq_length, size, num_gpus, dataset, experiment_name, load_model, num_vids, n_epochs, split, base_data_path, f_name, learning_rate_init, wd, save_freq, clip_length, video_offset, clip_offset, num_clips, clip_stride, batch_size, loss_type, metrics_dir, loaded_checkpoint, verbose, opt_choice, gpu_list, grad_clip_value, preproc_method, random_init, shuffle_seed, preproc_debugging, reverse): """ Training function used to train or fine-tune a chosen model Args: :model: tf-activity-recognition framework model object :input_dims: Number of frames used in input :output_dims: Integer number of classes in current dataset :seq_length: Length of output sequence expected from LSTM :size: List detailing height and width of frame :num_gpus: Number of gpus to use when training :dataset: Name of dataset being processed :experiment_name: Name of current experiment :load_model: Boolean variable indicating whether to load from a checkpoint or not :num_vids: Number of videos to be used for training :n_epochs: Total number of epochs to train :split: Split of dataset being used :base_data_path: Full path to root directory containing datasets :f_name: Specific video directory within a chosen split of a dataset :learning_rate_init: Initializer for learning rate :wd: Weight decay :save_freq: Frequency, in epochs, with which to save :clip_length: Length of clips to cut video into, -1 indicates using the entire video as one clip') :video_offset: String indicating where to begin selecting video clips (provided clipOffset is None) :clip_offset: "none" or "random" indicating where to begin selecting video clips :num_clips: Number of clips to break video into :clip_stride: Number of frames that overlap between clips, 0 indicates no overlap and negative values indicate a gap of frames between clips :batch_size: Number of clips to load into the model each step. :loss_type: String declaring loss type associated with a chosen model :metrics_dir: Name of subdirectory within the experiment to store metrics. Unique directory names allow for parallel testing :loaded_checkpoint: Specify the exact checkpoint of saved model to be loaded for further training/testing :verbose: Boolean to indicate if all print statement should be procesed or not :opt_choice: String indicating optimizer selected :gpu_list: List of GPU IDs to be used :grad_clip_value: Float value at which to clip normalized gradients :lr_boundaries: List of epoch boundaries at which lr will be updated :lr_values: List of lr multipliers to learning_rate_init at boundaries mentioned in lr_boundaries :preproc_method: The preprocessing method to use, default, cvr, rr, sr, or any other custom preprocessing :random_init: Randomly initialize model weights, not loading from any files (deafult False) :preproc_debugging: Boolean indicating whether to load videos and clips in a queue or to load them directly for debugging (Default 0) :reverse: Boolean indicating whether reverse videos and classify them as a new action class. Returns: Does not return anything """ with tf.name_scope("my_scope") as scope: # Initializers for checkpoint and global step variable ckpt = None gs_init = 0 ################################### Checkpoint loading block ####################################################### # Load pre-trained/saved model to continue training (or fine-tune) if load_model: try: ckpt, gs_init, learning_rate_init = load_checkpoint(model.name, dataset, experiment_name, loaded_checkpoint, preproc_method) if verbose: print 'A better checkpoint is found. The global_step value is: ' + str(gs_init) except: if verbose: print "Failed loading checkpoint requested. Please check." exit() # END TRY else: ckpt = model.load_default_weights() # END IF ###################################################################################################################### # Initialize model variables global_step = tf.Variable(gs_init, name='global_step', trainable=False) number_of_videos = tf.Variable(num_vids, name='number_of_videos', trainable=False) number_of_epochs = tf.Variable(n_epochs, name='number_of_epochs', trainable=False) video_step = tf.Variable(1.0, name='video_step', trainable=False) istraining = True reuse_variables = None # TF session setup config = tf.ConfigProto(allow_soft_placement=True) #, gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.8)) sess = tf.Session(config=config) init = tf.global_variables_initializer() # Variables get randomly initialized into tf graph sess.run(init) tower_losses = [] tower_grads = [] tower_slogits = [] data_path = os.path.join(base_data_path, 'tfrecords_'+dataset, 'Split'+str(split), f_name) # Setup tensors for models # input_data_tensor - [batchSize, inputDims, height, width, channels] input_data_tensor, labels_tensor, names_tensor = load_dataset(model, num_gpus, batch_size, output_dims, input_dims, seq_length, size, data_path, dataset, istraining, clip_length, video_offset, clip_offset, num_clips, clip_stride, video_step, preproc_debugging, shuffle_seed, verbose, reverse=reverse) ############### TO DO: FIX THIS ASAP ######################## if ((batch_size == 1) and (num_clips==1)): sess.run(tf.assign_add(video_step, -2)) else: sess.run(tf.assign_add(video_step, -1)) # END IF ############################################################ learning_rate = tf.Variable(learning_rate_init, name='learning_rate', trainable=False) # Define optimizer (Current selection is only momentum optimizer) if opt_choice == 'gd': optimizer = lambda lr: tf.train.GradientDescentOptimizer(lr) elif opt_choice == 'adam': optimizer = lambda lr: tf.train.AdamOptimizer(lr) else: optimizer = lambda lr: tf.train.MomentumOptimizer(learning_rate=lr, momentum=0.9) # END IF """ Multi-GPU setup: 1) Associate gpu device to specific model replica 2) Setup tower name scope for variables """ ################# GPU list check block #################### assert((len(gpu_list) == num_gpus) or (len(gpu_list) == 0)) if len(gpu_list) == 0: gpu_list = [str(x) for x in range(num_gpus)] # END IF ########################################################### ################################################## Setup TF graph block ###################################################### for gpu_idx in range(num_gpus): with tf.device('/gpu:'+str(gpu_list[gpu_idx])): with tf.name_scope('%s_%d' % ('tower', int(gpu_list[gpu_idx]))) as scope: with tf.variable_scope(tf.get_variable_scope(), reuse = reuse_variables): returned_layers = model.inference(input_data_tensor[gpu_idx*batch_size:gpu_idx*batch_size+batch_size,:,:,:,:], istraining, input_dims, output_dims, seq_length, scope, return_layer = ['logits'], weight_decay = wd) logits = tf.cast(returned_layers[0], tf.float32) # Calculating Softmax for probability outcomes : Can be modified, make function internal to model slogits = tf.nn.softmax(logits) # END WITH reuse_variables = True """ Within GPU mini-batch: 1) Calculate loss, 2) Initialize optimizer with required learning rate and 3) Compute gradients 4) Aggregate losses, gradients and logits """ total_loss = model.loss(logits, labels_tensor[gpu_idx*batch_size:gpu_idx*batch_size+batch_size, :], loss_type) opt = optimizer(learning_rate) gradients = opt.compute_gradients(total_loss, vars_.trainable_variables()) tower_losses.append(total_loss) tower_grads.append(gradients) tower_slogits.append(slogits) # END WITH # END WITH # END FOR """ After: 1) Computing gradients and losses need to be stored and averaged 2) Clip gradients by norm to required value 3) Apply mean gradient updates """ gradients = _average_gradients(tower_grads) gradients, variables = zip(*gradients) clipped_gradients, _ = clip_ops.clip_by_global_norm(gradients, grad_clip_value) gradients = list(zip(clipped_gradients, variables)) grad_updates = opt.apply_gradients(gradients, global_step=global_step, name="train") train_op = grad_updates ############################################################################################################################################ if save_bool: ######################### Logger Setup block ###################################### # Logging setup initialization (Naming format: Date, month, hour, minute, second) log_name = ("exp_train_%s_%s_%s" % ( time.strftime("%d_%m_%H_%M_%S"), dataset, experiment_name)) make_dir('results') make_dir(os.path.join('results',model.name)) make_dir(os.path.join('results',model.name, dataset)) make_dir(os.path.join('results',model.name, dataset, preproc_method)) make_dir(os.path.join('results',model.name, dataset, preproc_method, experiment_name)) make_dir(os.path.join('results',model.name, dataset, preproc_method, experiment_name, 'checkpoints')) curr_logger = Logger(os.path.join('logs', model.name, dataset, preproc_method, metrics_dir, log_name)) #################################################################################### # END IF init = tf.global_variables_initializer() coord = tf.train.Coordinator() threads = queue_runner_impl.start_queue_runners(sess=sess, coord=coord) # Variables get randomly initialized into tf graph sess.run(init) # Check that weights were loaded or random initializations are requested if ((ckpt == None) or (random_init)): print "Caution: Model weights are not being loaded, using random initialization." else: # Model variables initialized from previous saved models initialize_from_dict(sess, ckpt, model.name) # END IF del ckpt # Initialize tracking variables previous_vid_name = "" videos_loaded = 0 tot_count = 0 acc = 0 epoch_count = 0 tot_load_time = 0.0 tot_train_time = 0.0 last_loss = None losses = [] total_pred = [] save_data = [] total_params = [] losses_tracker = [] # Timing test setup time_init = time.time() batch_count = 0 epoch_acc = 0 l_r = learning_rate_init ########################################## Training loop block ################################################################ # Loop epoch number of time over the training set while videos_loaded < n_epochs*num_vids: # Variable to update during epoch intervals if (epoch_count+1)*num_vids <= videos_loaded < (epoch_count+1)*num_vids + num_gpus*batch_size: batch_count = 0 epoch_acc = 0 if epoch_count % save_freq == 0 and tot_count > 0: if save_bool: if verbose: print "Saving..." save_checkpoint(sess, model.name, dataset, experiment_name, preproc_method, l_r, global_step.eval(session=sess)) # END IF epoch_count += 1 # END IF time_pre_train = time.time() ######################################### Running TF training session block ################################## _, loss_train, predictions, gs, labels, vid_names, l_r, track_vars = sess.run([train_op, tower_losses, tower_slogits, global_step, labels_tensor, names_tensor, learning_rate, model.get_track_variables()]) ################################################################################################################ if verbose: print vid_names for name in vid_names: if name != previous_vid_name: videos_loaded += 1 previous_vid_name = name tot_count += 1 ######## Adaptive Learning Rate Control Block ############################ losses_tracker.append(np.mean(loss_train)) if videos_loaded % 10 == 0 and videos_loaded > 0: if last_loss is None: last_loss = sum(losses_tracker)/10 else: difference_loss = last_loss - sum(losses_tracker)/10 last_loss = sum(losses_tracker)/10 if abs(difference_loss) < 0.001: learning_rate/=10 # END IF # END IF if len(losses_tracker) == 10: losses_tracker = [] # END IF # END IF ########################################################################### # Transpose the extracted layers such that the mean is taken across the gpus and over any matrix with more than 1 dimension params_array = [] for key in track_vars.keys(): curr_params = np.array(track_vars[key]) if len(curr_params.shape) > 1: indices = np.arange(len(curr_params.shape)) + 1 indices[-1] = 0 curr_params = curr_params.transpose(indices) params_array.append(np.mean(curr_params, axis=tuple(range(len(curr_params.shape))[1:]))) else: params_array.append([np.mean(curr_params)]) # END IF # END FOR #################### Training accuracy computation block ############### # Compute training epoch accuracy for gpu_pred_idx in range(len(predictions)): for batch_idx in range(predictions[gpu_pred_idx].shape[0]): pred = np.mean(predictions[gpu_pred_idx][batch_idx], 0).argmax() if pred == labels[gpu_pred_idx*batch_size + batch_idx][0]: epoch_acc +=1 # END IF batch_count+=1 # END FOR # END FOR ###################### Add variables to be tracked to logger ############# time_post_train = time.time() tot_train_time += time_post_train - time_pre_train if verbose: print 'train_time: ', time_post_train-time_pre_train print 'step, loss: ', gs, loss_train print 'labels: ', labels # END IF if save_bool: curr_logger.add_scalar_value('train/train_time',time_post_train - time_pre_train, step=gs) curr_logger.add_scalar_value('train/loss', float(np.mean(loss_train)), step=gs) curr_logger.add_scalar_value('train/epoch_acc', epoch_acc/float(batch_count), step=gs) for layer in range(len(params_array)): for p in range(len(params_array[layer])): curr_logger.add_scalar_value('tracked_training_variables/'+str(track_vars.keys()[layer]+'_'+str(p)), float(params_array[layer][p]), step=gs) # END FOR # END FOR total_params.append(params_array) curr_logger.add_scalar_value('tracked_training_variables/learning_rate', float(l_r), step=gs) # END IF # END WHILE ######################################################################################################################################################### if save_bool: if verbose: print "Saving..." # END IF save_checkpoint(sess, model.name, dataset, experiment_name, preproc_method, l_r, gs) coord.request_stop() coord.join(threads) # END IF if verbose: print "Tot train time: ", tot_train_time print "Tot time: ", time.time()-time_init # END WITH if save_bool: # Save tracked parameterization variables as a numpy file if len(total_params) != 0: total_params = np.array(total_params).flatten() make_dir(os.path.join('results',model.name, dataset, preproc_method, experiment_name, metrics_dir)) if os.path.isfile(os.path.join('results', model.name, dataset, preproc_method, experiment_name, metrics_dir, 'train_params_'+dataset+'.npy')): loaded_params = np.load(os.path.join('results', model.name, dataset, preproc_method, experiment_name, metrics_dir, 'train_params_'+dataset+'.npy')) total_params = np.concatenate([loaded_params, total_params]) # END IF np.save(os.path.join('results', model.name, dataset, preproc_method, experiment_name, metrics_dir, 'train_params_'+dataset+'.npy'), total_params) # END IF # END IF if __name__=="__main__": if args.train: train( model = model, input_dims = args.inputDims, output_dims = args.outputDims, seq_length = args.seqLength, size = [args.size, args.size], num_gpus = args.numGpus, dataset = args.dataset, experiment_name = args.expName, load_model = args.load, num_vids = args.numVids, n_epochs = args.nEpochs, split = args.split, base_data_path = args.baseDataPath, f_name = args.fName, learning_rate_init = args.lr, wd = args.wd, save_freq = args.saveFreq, clip_length = args.clipLength, video_offset = args.videoOffset, clip_offset = args.clipOffset, num_clips = args.numClips, clip_stride = args.clipStride, batch_size = args.batchSize, loss_type = args.lossType, metrics_dir = args.metricsDir, loaded_checkpoint = args.loadedCheckpoint, verbose = args.verbose, opt_choice = args.optChoice, gpu_list = args.gpuList, grad_clip_value = args.gradClipValue, preproc_method = args.preprocMethod, random_init = args.randomInit, shuffle_seed = args.shuffleSeed, preproc_debugging = args.preprocDebugging, reverse = args.reverse) # END IF