cifar10.py
3.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
# coding: utf-8
try:
import urllib.request
except ImportError:
raise ImportError('You should use Python 3.x')
import os.path
import gzip
import pickle
import os
import numpy as np
key_file = {
'train':'cifar10-train.gz',
'test':'cifar10-test.gz'
}
dataset_dir = os.path.dirname(os.path.abspath('/Users/HyeonJun/Desktop/simple_convnet/dataset'))
save_file = dataset_dir + "/cifar10.pkl"
train_num = 50000
test_num = 10000
img_dim = (3, 32, 32)
img_size = 3072
def _load_label(file_name):
file_path = dataset_dir + "/" + file_name
print("Converting " + file_name + " to NumPy Array ...")
with gzip.open(file_path, 'rb') as f:
labels = np.frombuffer(f.read(), np.uint8, offset=0)
labels = labels.reshape(-1, img_size+1)
labels = labels.T
print("Done")
return labels[0]
def _load_img(file_name):
file_path = dataset_dir + "/" + file_name
print("Converting " + file_name + " to NumPy Array ...")
with gzip.open(file_path, 'rb') as f:
data = np.frombuffer(f.read(), np.uint8, offset=0)
data = data.reshape(-1, img_size+1)
data = np.delete(data, 0, 1)
print("Done")
return data
def _convert_numpy():
dataset = {}
dataset['train_img'] = _load_img(key_file['train'])
dataset['train_label'] = _load_label(key_file['train'])
dataset['test_img'] = _load_img(key_file['test'])
dataset['test_label'] = _load_label(key_file['test'])
return dataset
def init_cifar10():
dataset = _convert_numpy()
print("Creating pickle file ...")
with open(save_file, 'wb') as f:
pickle.dump(dataset, f, -1)
print("Done!")
def _change_one_hot_label(X):
T = np.zeros((X.size, 10))
for idx, row in enumerate(T):
row[X[idx]] = 1
return T
def load_cifar10(normalize=True, flatten=True, one_hot_label=False):
"""CIFAR-10データセットの読み込み
Parameters
----------
normalize : 画像のピクセル値を0.0~1.0に正規化する
one_hot_label :
one_hot_labelがTrueの場合、ラベルはone-hot配列として返す
one-hot配列とは、たとえば[0,0,1,0,0,0,0,0,0,0]のような配列
flatten : 画像を一次元配列に平にするかどうか
Returns
-------
(訓練画像, 訓練ラベル), (テスト画像, テストラベル)
"""
if not os.path.exists(save_file):
init_cifar10()
with open(save_file, 'rb') as f:
dataset = pickle.load(f)
if normalize:
for key in ('train_img', 'test_img'):
dataset[key] = dataset[key].astype(np.float32)
dataset[key] /= 255.0
if one_hot_label:
dataset['train_label'] = _change_one_hot_label(dataset['train_label'])
dataset['test_label'] = _change_one_hot_label(dataset['test_label'])
if not flatten:
for key in ('train_img', 'test_img'):
dataset[key] = dataset[key].reshape(-1, 3, 32, 32)
return (dataset['train_img'], dataset['train_label']), (dataset['test_img'], dataset['test_label'])
if __name__ == '__main__':
init_cifar10()