Skip to content

Commit d63ae5f

Browse files
StephanieLarocquenotoraptor
StephanieLarocque
authored and
notoraptor
committed
dataset loader for unet and fcn8
1 parent 8285a9f commit d63ae5f

2 files changed

Lines changed: 364 additions & 1 deletion

File tree

‎code/fcn_2D_segm/data_loader.py‎

Lines changed: 137 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
import os
2+
import time
3+
14
from dataset_loaders.images.polyps912 import Polyps912Dataset
25
from dataset_loaders.images.camvid import CamvidDataset
36
from dataset_loaders.images.polyps912 import Polyps912Dataset
@@ -46,7 +49,7 @@ def load_data(dataset, train_data_augm_kwargs={}, one_hot=False,
4649
shuffle_at_each_epoch=False,
4750
return_list=True,
4851
return_0_255=return_0_255)
49-
52+
5053
elif dataset == 'em':
5154
train_iter = IsbiEmStacksDataset(which_set='train',
5255
start=0,
@@ -77,6 +80,7 @@ def load_data(dataset, train_data_augm_kwargs={}, one_hot=False,
7780
return_0_255=return_0_255)
7881
test_iter = None
7982
else:
83+
print 'Dataset must be either "em" or "polyps912" '
8084
raise NotImplementedError
8185

8286
if which_set == 'train':
@@ -89,3 +93,135 @@ def load_data(dataset, train_data_augm_kwargs={}, one_hot=False,
8993
ret = [train_iter, val_iter, test_iter]
9094

9195
return ret
96+
97+
98+
99+
100+
def test_load_em():
101+
102+
train_iter = IsbiEmStacksDataset(
103+
which_set='train',
104+
batch_size=1,
105+
data_augm_kwargs={},
106+
return_one_hot=False,
107+
return_01c=False,
108+
return_list=True,
109+
use_threads=False)
110+
111+
valid_iter = IsbiEmStacksDataset(
112+
which_set='valid',
113+
batch_size=1,
114+
data_augm_kwargs={},
115+
return_one_hot=False,
116+
return_01c=False,
117+
return_list=True,
118+
use_threads=False)
119+
120+
test_iter = None
121+
122+
train_nbatches = train_iter.nbatches
123+
valid_nbatches = valid_iter.nbatches
124+
125+
126+
# Simulate training
127+
max_epochs = 2
128+
print "Simulate training for", str(max_epochs), "epochs"
129+
start_training = time.time()
130+
for epoch in range(max_epochs):
131+
print "Epoch #", str(epoch)
132+
133+
start_epoch = time.time()
134+
135+
print "Iterate on the training set", train_nbatches, "minibatches"
136+
for mb in range(train_nbatches):
137+
start_batch = time.time()
138+
batch = train_iter.next()
139+
if mb%5 ==0:
140+
print("Minibatch train {}: {} sec".format(mb, (time.time() -
141+
start_batch)))
142+
143+
print "Iterate on the validation set", valid_nbatches, "minibatches"
144+
for mb in range(valid_nbatches):
145+
start_batch = time.time()
146+
batch = valid_iter.next()
147+
if mb%5 ==0:
148+
print("Minibatch valid {}: {} sec".format(mb, (time.time() -
149+
start_batch)))
150+
151+
print("Epoch time: %s" % str(time.time() - start_epoch))
152+
print("Training simulation time: %s" % str(time.time() - start_training))
153+
154+
155+
def test_load_polyps():
156+
train_iter = Polyps912Dataset(
157+
which_set='train',
158+
batch_size=1,
159+
data_augm_kwargs={},
160+
return_one_hot=False,
161+
return_01c=False,
162+
return_list=True,
163+
use_threads=False)
164+
165+
valid_iter = Polyps912Dataset(
166+
which_set='valid',
167+
batch_size=1,
168+
data_augm_kwargs={},
169+
return_one_hot=False,
170+
return_01c=False,
171+
return_list=True,
172+
use_threads=False)
173+
174+
test_iter = Polyps912Dataset(
175+
which_set='test',
176+
batch_size=1,
177+
data_augm_kwargs={},
178+
return_one_hot=False,
179+
return_01c=False,
180+
return_list=True,
181+
use_threads=False)
182+
183+
train_nbatches = train_iter.nbatches
184+
valid_nbatches = valid_iter.nbatches
185+
test_nbatches = test_iter.nbatches
186+
187+
188+
# Simulate training
189+
max_epochs = 2
190+
print "Simulate training for", str(max_epochs), "epochs"
191+
start_training = time.time()
192+
for epoch in range(max_epochs):
193+
print "Epoch #", str(epoch)
194+
195+
start_epoch = time.time()
196+
197+
print "Iterate on the training set", train_nbatches, "minibatches"
198+
for mb in range(train_nbatches):
199+
start_batch = time.time()
200+
batch = train_iter.next()
201+
if mb%50 ==0:
202+
print("Minibatch train {}: {} sec".format(mb, (time.time() - start_batch)))
203+
204+
print "Iterate on the validation set", valid_nbatches, "minibatches"
205+
for mb in range(valid_nbatches):
206+
start_batch = time.time()
207+
batch = valid_iter.next()
208+
if mb%50 ==0:
209+
print("Minibatch valid {}: {} sec".format(mb, (time.time() -start_batch)))
210+
211+
print("Epoch time: %s" % str(time.time() - start_epoch))
212+
print "Iterate on the test set", test_nbatches, "minibatches"
213+
for mb in range(test_nbatches):
214+
start_batch = time.time()
215+
batch = test_iter.next()
216+
if mb%50 ==0:
217+
print("Minibatch test {}: {} sec".format(mb, (time.time() - start_batch)))
218+
print("Training simulation time: %s" % str(time.time() - start_training))
219+
220+
if __name__=='__main__':
221+
print "Iterating through polyps dataset"
222+
test_load_polyps()
223+
print "Success!"
224+
225+
print "Iterating through IsbiEmStacks dataset"
226+
test_load_em()
227+
print "Success!"

‎code/unet/data_loader.py‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
import os
2+
import time
3+
4+
from dataset_loaders.images.polyps912 import Polyps912Dataset
5+
from dataset_loaders.images.camvid import CamvidDataset
6+
from dataset_loaders.images.polyps912 import Polyps912Dataset
7+
from dataset_loaders.images.isbi_em_stacks import IsbiEmStacksDataset
8+
9+
10+
def load_data(dataset, train_data_augm_kwargs={}, one_hot=False,
11+
batch_size=[10, 10, 10], shuffle_train=True, return_0_255=False,
12+
which_set='all'):
13+
14+
assert which_set in ['all', 'train', 'val', 'test']
15+
16+
# Build dataset iterator
17+
if dataset == 'polyps912':
18+
train_iter = Polyps912Dataset(which_set='train',
19+
batch_size=batch_size[0],
20+
seq_per_subset=0,
21+
seq_length=0,
22+
data_augm_kwargs=train_data_augm_kwargs,
23+
return_one_hot=one_hot,
24+
return_01c=False,
25+
overlap=0,
26+
use_threads=False,
27+
shuffle_at_each_epoch=shuffle_train,
28+
return_list=True,
29+
return_0_255=return_0_255)
30+
val_iter = Polyps912Dataset(which_set='val',
31+
batch_size=batch_size[1],
32+
seq_per_subset=0,
33+
seq_length=0,
34+
return_one_hot=one_hot,
35+
return_01c=False,
36+
overlap=0,
37+
use_threads=False,
38+
shuffle_at_each_epoch=False,
39+
return_list=True,
40+
return_0_255=return_0_255)
41+
test_iter = Polyps912Dataset(which_set='test',
42+
batch_size=batch_size[2],
43+
seq_per_subset=0,
44+
seq_length=0,
45+
return_one_hot=one_hot,
46+
return_01c=False,
47+
overlap=0,
48+
use_threads=False,
49+
shuffle_at_each_epoch=False,
50+
return_list=True,
51+
return_0_255=return_0_255)
52+
53+
elif dataset == 'em':
54+
train_iter = IsbiEmStacksDataset(which_set='train',
55+
start=0,
56+
end=25,
57+
batch_size=batch_size[0],
58+
seq_per_subset=0,
59+
seq_length=0,
60+
data_augm_kwargs=train_data_augm_kwargs,
61+
return_one_hot=one_hot,
62+
return_01c=False,
63+
overlap=0,
64+
use_threads=True,
65+
shuffle_at_each_epoch=shuffle_train,
66+
return_list=True,
67+
return_0_255=return_0_255)
68+
69+
val_iter = IsbiEmStacksDataset(which_set='train',
70+
batch_size=batch_size[1],
71+
seq_per_subset=0,
72+
seq_length=0,
73+
return_one_hot=one_hot,
74+
return_01c=False,
75+
use_threads=True,
76+
shuffle_at_each_epoch=False,
77+
start=25,
78+
end=30,
79+
return_list=True,
80+
return_0_255=return_0_255)
81+
test_iter = None
82+
else:
83+
print 'Dataset must be either "em" or "polyps912" '
84+
raise NotImplementedError
85+
86+
if which_set == 'train':
87+
ret = train_iter
88+
elif which_set == 'val':
89+
ret = val_iter
90+
elif which_set == 'test':
91+
ret = test_iter
92+
else:
93+
ret = [train_iter, val_iter, test_iter]
94+
95+
return ret
96+
97+
98+
99+
100+
def test_load_em():
101+
102+
train_iter = IsbiEmStacksDataset(
103+
which_set='train',
104+
batch_size=1,
105+
data_augm_kwargs={},
106+
return_one_hot=False,
107+
return_01c=False,
108+
return_list=True,
109+
use_threads=False)
110+
111+
valid_iter = IsbiEmStacksDataset(
112+
which_set='valid',
113+
batch_size=1,
114+
data_augm_kwargs={},
115+
return_one_hot=False,
116+
return_01c=False,
117+
return_list=True,
118+
use_threads=False)
119+
120+
test_iter = None
121+
122+
train_nbatches = train_iter.nbatches
123+
valid_nbatches = valid_iter.nbatches
124+
125+
126+
# Simulate training
127+
max_epochs = 2
128+
print "Simulate training for", str(max_epochs), "epochs"
129+
start_training = time.time()
130+
for epoch in range(max_epochs):
131+
print "Epoch #", str(epoch)
132+
133+
start_epoch = time.time()
134+
135+
print "Iterate on the training set", train_nbatches, "minibatches"
136+
for mb in range(train_nbatches):
137+
start_batch = time.time()
138+
batch = train_iter.next()
139+
if mb%5 ==0:
140+
print("Minibatch train {}: {} sec".format(mb, (time.time() -
141+
start_batch)))
142+
143+
print "Iterate on the validation set", valid_nbatches, "minibatches"
144+
for mb in range(valid_nbatches):
145+
start_batch = time.time()
146+
batch = valid_iter.next()
147+
if mb%5 ==0:
148+
print("Minibatch valid {}: {} sec".format(mb, (time.time() -
149+
start_batch)))
150+
151+
print("Epoch time: %s" % str(time.time() - start_epoch))
152+
print("Training simulation time: %s" % str(time.time() - start_training))
153+
154+
155+
def test_load_polyps():
156+
train_iter = Polyps912Dataset(
157+
which_set='train',
158+
batch_size=1,
159+
data_augm_kwargs={},
160+
return_one_hot=False,
161+
return_01c=False,
162+
return_list=True,
163+
use_threads=False)
164+
165+
valid_iter = Polyps912Dataset(
166+
which_set='valid',
167+
batch_size=1,
168+
data_augm_kwargs={},
169+
return_one_hot=False,
170+
return_01c=False,
171+
return_list=True,
172+
use_threads=False)
173+
174+
test_iter = Polyps912Dataset(
175+
which_set='test',
176+
batch_size=1,
177+
data_augm_kwargs={},
178+
return_one_hot=False,
179+
return_01c=False,
180+
return_list=True,
181+
use_threads=False)
182+
183+
train_nbatches = train_iter.nbatches
184+
valid_nbatches = valid_iter.nbatches
185+
test_nbatches = test_iter.nbatches
186+
187+
188+
# Simulate training
189+
max_epochs = 2
190+
print "Simulate training for", str(max_epochs), "epochs"
191+
start_training = time.time()
192+
for epoch in range(max_epochs):
193+
print "Epoch #", str(epoch)
194+
195+
start_epoch = time.time()
196+
197+
print "Iterate on the training set", train_nbatches, "minibatches"
198+
for mb in range(train_nbatches):
199+
start_batch = time.time()
200+
batch = train_iter.next()
201+
if mb%50 ==0:
202+
print("Minibatch train {}: {} sec".format(mb, (time.time() - start_batch)))
203+
204+
print "Iterate on the validation set", valid_nbatches, "minibatches"
205+
for mb in range(valid_nbatches):
206+
start_batch = time.time()
207+
batch = valid_iter.next()
208+
if mb%50 ==0:
209+
print("Minibatch valid {}: {} sec".format(mb, (time.time() -start_batch)))
210+
211+
print("Epoch time: %s" % str(time.time() - start_epoch))
212+
print "Iterate on the test set", test_nbatches, "minibatches"
213+
for mb in range(test_nbatches):
214+
start_batch = time.time()
215+
batch = test_iter.next()
216+
if mb%50 ==0:
217+
print("Minibatch test {}: {} sec".format(mb, (time.time() - start_batch)))
218+
print("Training simulation time: %s" % str(time.time() - start_training))
219+
220+
if __name__=='__main__':
221+
print "Iterating through polyps dataset"
222+
test_load_polyps()
223+
print "Success!"
224+
225+
print "Iterating through IsbiEmStacks dataset"
226+
test_load_em()
227+
print "Success!"

0 commit comments

Comments
 (0)