upload kaggle dog vs cat

Author: L1aoXingyu

chapter8_Application/kaggle_dog_vs_cat/README.md

@@ -0,0 +1,4 @@
+# kaggle competition
+## dog vs cat
+
+This is my first competition in Kaggle.

chapter8_Application/kaggle_dog_vs_cat/model/dataset.py

@@ -0,0 +1,29 @@
+__author__ = 'SherlockLiao'
+
+import torch
+from torch.utils.data import Dataset
+import h5py
+
+
+class h5Dataset(Dataset):
+
+    def __init__(self, h5py_list):
+        label_file = h5py.File(h5py_list[0], 'r')
+        self.label = torch.from_numpy(label_file['label'].value)
+        self.nSamples = self.label.size(0)
+        temp_dataset = torch.FloatTensor()
+        for file in h5py_list:
+            h5_file = h5py.File(file, 'r')
+            dataset = torch.from_numpy(h5_file['data'].value)
+            temp_dataset = torch.cat((temp_dataset, dataset), 1)
+
+        self.dataset = temp_dataset
+
+    def __len__(self):
+        return self.nSamples
+
+    def __getitem__(self, index):
+        assert index < len(self), 'index range error'
+        data = self.dataset[index]
+        label = self.label[index]
+        return (data, label)

chapter8_Application/kaggle_dog_vs_cat/model/feature_extraction.py

@@ -0,0 +1,100 @@
+__author__ = 'SherlockLiao'
+
+import os
+from tqdm import tqdm
+import h5py
+import numpy as np
+import argparse
+
+import torch
+from torchvision import models, transforms
+from torch import optim, nn
+from torch.autograd import Variable
+from torchvision.datasets import ImageFolder
+from torch.utils.data import DataLoader
+from net import feature_net, classifier
+
+parse = argparse.ArgumentParser()
+parse.add_argument(
+    '--model', required=True, help='vgg, inceptionv3, resnet152')
+parse.add_argument('--bs', type=int, default=32)
+parse.add_argument('--phase', required=True, help='train, val')
+opt = parse.parse_args()
+print(opt)
+
+img_transform = transforms.Compose([
+    transforms.Scale(320),
+    transforms.CenterCrop(299),
+    transforms.ToTensor(),
+    transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
+])
+
+root = '/media/sherlock/Files/kaggle_dog_vs_cat/data'
+data_folder = {
+    'train': ImageFolder(os.path.join(root, 'train'), transform=img_transform),
+    'val': ImageFolder(os.path.join(root, 'val'), transform=img_transform)
+}
+
+# define dataloader to load images
+batch_size = opt.bs
+dataloader = {
+    'train':
+    DataLoader(
+        data_folder['train'],
+        batch_size=batch_size,
+        shuffle=False,
+        num_workers=4),
+    'val':
+    DataLoader(
+        data_folder['val'],
+        batch_size=batch_size,
+        shuffle=False,
+        num_workers=4)
+}
+
+# get train data size and validation data size
+data_size = {
+    'train': len(dataloader['train'].dataset),
+    'val': len(dataloader['val'].dataset)
+}
+
+# get numbers of classes
+img_classes = len(dataloader['train'].dataset.classes)
+
+# test if using GPU
+use_gpu = torch.cuda.is_available()
+
+
+def CreateFeature(model, phase, outputPath='.'):
+    """
+    Create h5py dataset for feature extraction.
+
+    ARGS:
+        outputPath    : h5py output path
+        model         : used model
+        labelList     : list of corresponding groundtruth texts
+    """
+    featurenet = feature_net(model)
+    if use_gpu:
+        featurenet.cuda()
+    feature_map = torch.FloatTensor()
+    label_map = torch.LongTensor()
+    for data in tqdm(dataloader[phase]):
+        img, label = data
+        if use_gpu:
+            img = Variable(img, volatile=True).cuda()
+        else:
+            img = Variable(img, volatile=True)
+        out = featurenet(img)
+        feature_map = torch.cat((feature_map, out.cpu().data), 0)
+        label_map = torch.cat((label_map, label), 0)
+    feature_map = feature_map.numpy()
+    label_map = label_map.numpy()
+    file_name = '_feature_{}.hd5f'.format(model)
+    h5_path = os.path.join(outputPath, phase) + file_name
+    with h5py.File(h5_path, 'w') as h:
+        h.create_dataset('data', data=feature_map)
+        h.create_dataset('label', data=label_map)
+
+
+CreateFeature(opt.model, opt.phase)

chapter8_Application/kaggle_dog_vs_cat/model/feature_train.py

@@ -0,0 +1,126 @@
+__author__ = 'SherlockLiao'
+
+import argparse
+import time
+import os
+
+import torch
+from torch import nn
+from torch.autograd import Variable
+from torch import optim
+from torch.utils.data import DataLoader
+
+from dataset import h5Dataset
+from net import classifier
+
+parse = argparse.ArgumentParser()
+parse.add_argument(
+    '--model',
+    nargs='+',
+    help='inceptionv3, vgg, resnet152',
+    default=['vgg', 'inceptionv3', 'resnet152'])
+parse.add_argument('--batch_size', type=int, default=64)
+parse.add_argument('--epoch', type=int, default=20)
+parse.add_argument('--n_classes', default=2, type=int)
+parse.add_argument('--num_workers', type=int, default=8)
+opt = parse.parse_args()
+print(opt)
+
+root = '/media/sherlock/Files/kaggle_dog_vs_cat/'
+train_list = ['train_feature_{}.hd5f'.format(i) for i in opt.model]
+val_list = ['val_feature_{}.hd5f'.format(i) for i in opt.model]
+
+dataset = {'train': h5Dataset(train_list), 'val': h5Dataset(val_list)}
+
+datasize = {
+    'train': dataset['train'].dataset.size(0),
+    'val': dataset['val'].dataset.size(0)
+}
+
+batch_size = opt.batch_size
+epoches = opt.epoch
+
+dataloader = {
+    'train':
+    DataLoader(
+        dataset['train'],
+        batch_size=batch_size,
+        shuffle=True,
+        num_workers=opt.num_workers),
+    'val':
+    DataLoader(
+        dataset['val'],
+        batch_size=batch_size,
+        shuffle=False,
+        num_workers=opt.num_workers)
+}
+
+dimension = dataset['train'].dataset.size(1)
+
+mynet = classifier(dimension, opt.n_classes)
+mynet.cuda()
+
+criterion = nn.CrossEntropyLoss()
+optimizer = optim.SGD(mynet.parameters(), lr=1e-3)
+# train
+for epoch in range(epoches):
+    print('{}'.format(epoch + 1))
+    print('*' * 10)
+    print('Train')
+    mynet.train()
+    since = time.time()
+
+    running_loss = 0.0
+    running_acc = 0.0
+    for i, data in enumerate(dataloader['train'], 1):
+        feature, label = data
+        feature = Variable(feature).cuda()
+        label = Variable(label).cuda()
+
+        # forward
+        out = mynet(feature)
+        loss = criterion(out, label)
+        # backward
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        running_loss += loss.data[0] * label.size(0)
+        _, pred = torch.max(out, 1)
+        num_correct = torch.sum(pred == label)
+        running_acc += num_correct.data[0]
+        if i % 50 == 0:
+            print('Loss: {:.6f}, Acc: {:.6f}'.format(running_loss / (
+                i * batch_size), running_acc / (i * batch_size)))
+
+    running_loss /= datasize['train']
+    running_acc /= datasize['train']
+    eplise_time = time.time() - since
+    print('Loss: {:.6f}, Acc: {:.6f}, Time: {:.0f}s'.format(
+        running_loss, running_acc, eplise_time))
+    print('Validation')
+    mynet.eval()
+    num_correct = 0.0
+    eval_loss = 0.0
+    for data in dataloader['val']:
+        feature, label = data
+        feature = Variable(feature, volatile=True).cuda()
+        label = Variable(label, volatile=True).cuda()
+        # forward
+        out = mynet(feature)
+        loss = criterion(out, label)
+
+        _, pred = torch.max(out, 1)
+        correct = torch.sum(pred == label)
+        num_correct += correct.data[0]
+        eval_loss += loss.data[0] * label.size(0)
+
+    print('Loss: {:.6f}, Acc: {:.6f}'.format(eval_loss / datasize['val'],
+                                             num_correct / datasize['val']))
+print('Finish Training!')
+
+save_path = os.path.join(root, 'model_save')
+if not os.path.exists(save_path):
+    os.mkdir(save_path)
+
+torch.save(mynet.state_dict(), save_path + '/feature_model.pth')