Graduate

Commit for merge

File mode changed
import numpy as np
import cv2
import asyncio
import websockets
from io import BytesIO
from PIL import Image, ImageDraw
from IPython import display
async def recv_image(websocket, path):
buf = await websocket.recv()
byte = BytesIO(buf)
image = Image.open(byte)
remote_ip = websocket.remote_address[0]
msg='[{ip}] receive face properly, image size={size}'.format(ip=remote_ip, size=image.size)
print(msg)
await websocket.send('100')
#for debug
#frame = np.array(image)
#frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
#cv2.imshow('recv', frame)
#cv2.waitKey(2000)
#cv2.destroyAllWindows()
print('run image server')
start_image_server = websockets.serve(recv_image, '0.0.0.0', 8766)
asyncio.get_event_loop().run_until_complete(start_image_server)
asyncio.get_event_loop().run_forever()
\ No newline at end of file
import torch
import numpy as np
import os
import asyncio
import json
import websockets
from io import BytesIO
from PIL import Image, ImageDraw
from IPython import display
from models.mtcnn import MTCNN
from models.inception_resnet_v1 import InceptionResnetV1
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print('Running on device: {}'.format(device))
model = InceptionResnetV1().eval().to(device)
async def get_embeddings(face_list):
global model
x = torch.Tensor(face_list).to(device)
yhat = model(x)
return yhat
def get_distance(someone, database):
distance = [(someone - data).norm().item() for data in database]
return distance
def get_argmin(someone, database):
distance = get_distance(someone, database)
for i in range(len(distance)):
return np.argmin(distance)
return -1
async def recv_face(websocket, path):
buf = await websocket.recv()
face = np.frombuffer(buf, dtype = np.float32)
face = face.reshape((1,3,160,160))
remote_ip = websocket.remote_address[0]
msg='[{ip}] receive face properly, numpy shape={shape}'.format(ip=remote_ip, shape=face.shape)
print(msg)
embedding = await get_embeddings(face)
await websocket.send('100')
##embedding DB서버에 넘기기##
print('run verification server')
start_server = websockets.serve(recv_face, '0.0.0.0', 8765)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
\ No newline at end of file
No preview for this file type
No preview for this file type
No preview for this file type
No preview for this file type
import torch
from torch import nn
from torch.nn import functional as F
import os
class BasicConv2d(nn.Module):
def __init__(self, in_planes, out_planes, kernel_size, stride, padding=0):
super().__init__()
self.conv = nn.Conv2d(
in_planes, out_planes,
kernel_size=kernel_size, stride=stride,
padding=padding, bias=False
) # verify bias false
self.bn = nn.BatchNorm2d(
out_planes,
eps=0.001, # value found in tensorflow
momentum=0.1, # default pytorch value
affine=True
)
self.relu = nn.ReLU(inplace=False)
def forward(self, x):
x = self.conv(x)
x = self.bn(x)
x = self.relu(x)
return x
class Block35(nn.Module):
def __init__(self, scale=1.0):
super().__init__()
self.scale = scale
self.branch0 = BasicConv2d(256, 32, kernel_size=1, stride=1)
self.branch1 = nn.Sequential(
BasicConv2d(256, 32, kernel_size=1, stride=1),
BasicConv2d(32, 32, kernel_size=3, stride=1, padding=1)
)
self.branch2 = nn.Sequential(
BasicConv2d(256, 32, kernel_size=1, stride=1),
BasicConv2d(32, 32, kernel_size=3, stride=1, padding=1),
BasicConv2d(32, 32, kernel_size=3, stride=1, padding=1)
)
self.conv2d = nn.Conv2d(96, 256, kernel_size=1, stride=1)
self.relu = nn.ReLU(inplace=False)
def forward(self, x):
x0 = self.branch0(x)
x1 = self.branch1(x)
x2 = self.branch2(x)
out = torch.cat((x0, x1, x2), 1)
out = self.conv2d(out)
out = out * self.scale + x
out = self.relu(out)
return out
class Block17(nn.Module):
def __init__(self, scale=1.0):
super().__init__()
self.scale = scale
self.branch0 = BasicConv2d(896, 128, kernel_size=1, stride=1)
self.branch1 = nn.Sequential(
BasicConv2d(896, 128, kernel_size=1, stride=1),
BasicConv2d(128, 128, kernel_size=(1,7), stride=1, padding=(0,3)),
BasicConv2d(128, 128, kernel_size=(7,1), stride=1, padding=(3,0))
)
self.conv2d = nn.Conv2d(256, 896, kernel_size=1, stride=1)
self.relu = nn.ReLU(inplace=False)
def forward(self, x):
x0 = self.branch0(x)
x1 = self.branch1(x)
out = torch.cat((x0, x1), 1)
out = self.conv2d(out)
out = out * self.scale + x
out = self.relu(out)
return out
class Block8(nn.Module):
def __init__(self, scale=1.0, noReLU=False):
super().__init__()
self.scale = scale
self.noReLU = noReLU
self.branch0 = BasicConv2d(1792, 192, kernel_size=1, stride=1)
self.branch1 = nn.Sequential(
BasicConv2d(1792, 192, kernel_size=1, stride=1),
BasicConv2d(192, 192, kernel_size=(1,3), stride=1, padding=(0,1)),
BasicConv2d(192, 192, kernel_size=(3,1), stride=1, padding=(1,0))
)
self.conv2d = nn.Conv2d(384, 1792, kernel_size=1, stride=1)
if not self.noReLU:
self.relu = nn.ReLU(inplace=False)
def forward(self, x):
x0 = self.branch0(x)
x1 = self.branch1(x)
out = torch.cat((x0, x1), 1)
out = self.conv2d(out)
out = out * self.scale + x
if not self.noReLU:
out = self.relu(out)
return out
class Mixed_6a(nn.Module):
def __init__(self):
super().__init__()
self.branch0 = BasicConv2d(256, 384, kernel_size=3, stride=2)
self.branch1 = nn.Sequential(
BasicConv2d(256, 192, kernel_size=1, stride=1),
BasicConv2d(192, 192, kernel_size=3, stride=1, padding=1),
BasicConv2d(192, 256, kernel_size=3, stride=2)
)
self.branch2 = nn.MaxPool2d(3, stride=2)
def forward(self, x):
x0 = self.branch0(x)
x1 = self.branch1(x)
x2 = self.branch2(x)
out = torch.cat((x0, x1, x2), 1)
return out
class Mixed_7a(nn.Module):
def __init__(self):
super().__init__()
self.branch0 = nn.Sequential(
BasicConv2d(896, 256, kernel_size=1, stride=1),
BasicConv2d(256, 384, kernel_size=3, stride=2)
)
self.branch1 = nn.Sequential(
BasicConv2d(896, 256, kernel_size=1, stride=1),
BasicConv2d(256, 256, kernel_size=3, stride=2)
)
self.branch2 = nn.Sequential(
BasicConv2d(896, 256, kernel_size=1, stride=1),
BasicConv2d(256, 256, kernel_size=3, stride=1, padding=1),
BasicConv2d(256, 256, kernel_size=3, stride=2)
)
self.branch3 = nn.MaxPool2d(3, stride=2)
def forward(self, x):
x0 = self.branch0(x)
x1 = self.branch1(x)
x2 = self.branch2(x)
x3 = self.branch3(x)
out = torch.cat((x0, x1, x2, x3), 1)
return out
class InceptionResnetV1(nn.Module):
"""Inception Resnet V1 model with optional loading of pretrained weights.
Model parameters can be loaded based on pretraining on the VGGFace2 or CASIA-Webface
datasets. Pretrained state_dicts are automatically downloaded on model instantiation if
requested and cached in the torch cache. Subsequent instantiations use the cache rather than
redownloading.
Keyword Arguments:
pretrained {str} -- Optional pretraining dataset. Either 'vggface2' or 'casia-webface'.
(default: {None})
classify {bool} -- Whether the model should output classification probabilities or feature
embeddings. (default: {False})
num_classes {int} -- Number of output classes. If 'pretrained' is set and num_classes not
equal to that used for the pretrained model, the final linear layer will be randomly
initialized. (default: {None})
dropout_prob {float} -- Dropout probability. (default: {0.6})
"""
def __init__(self, classify=False, dropout_prob=0.6, device=None):
super().__init__()
# Set simple attributes
self.classify = classify
self.num_classes = 8631
# Define layers
self.conv2d_1a = BasicConv2d(3, 32, kernel_size=3, stride=2)
self.conv2d_2a = BasicConv2d(32, 32, kernel_size=3, stride=1)
self.conv2d_2b = BasicConv2d(32, 64, kernel_size=3, stride=1, padding=1)
self.maxpool_3a = nn.MaxPool2d(3, stride=2)
self.conv2d_3b = BasicConv2d(64, 80, kernel_size=1, stride=1)
self.conv2d_4a = BasicConv2d(80, 192, kernel_size=3, stride=1)
self.conv2d_4b = BasicConv2d(192, 256, kernel_size=3, stride=2)
self.repeat_1 = nn.Sequential(
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
Block35(scale=0.17),
)
self.mixed_6a = Mixed_6a()
self.repeat_2 = nn.Sequential(
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
Block17(scale=0.10),
)
self.mixed_7a = Mixed_7a()
self.repeat_3 = nn.Sequential(
Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20),
Block8(scale=0.20),
)
self.block8 = Block8(noReLU=True)
self.avgpool_1a = nn.AdaptiveAvgPool2d(1)
self.dropout = nn.Dropout(dropout_prob)
self.last_linear = nn.Linear(1792, 512, bias=False)
self.last_bn = nn.BatchNorm1d(512, eps=0.001, momentum=0.1, affine=True)
self.logits = nn.Linear(512, self.num_classes)
load_weights(self)
self.device = torch.device('cpu')
if device is not None:
self.device = device
self.to(device)
def forward(self, x):
"""Calculate embeddings or logits given a batch of input image tensors.
Arguments:
x {torch.tensor} -- Batch of image tensors representing faces.
Returns:
torch.tensor -- Batch of embedding vectors or multinomial logits.
"""
x = self.conv2d_1a(x)
x = self.conv2d_2a(x)
x = self.conv2d_2b(x)
x = self.maxpool_3a(x)
x = self.conv2d_3b(x)
x = self.conv2d_4a(x)
x = self.conv2d_4b(x)
x = self.repeat_1(x)
x = self.mixed_6a(x)
x = self.repeat_2(x)
x = self.mixed_7a(x)
x = self.repeat_3(x)
x = self.block8(x)
x = self.avgpool_1a(x)
x = self.dropout(x)
x = self.last_linear(x.view(x.shape[0], -1))
x = self.last_bn(x)
if self.classify:
x = self.logits(x)
else:
x = F.normalize(x, p=2, dim=1)
return x
def load_weights(mdl):
features_path = state_dict_path = os.path.join(os.path.dirname(__file__), 'vggface2-dict/20180402-114759-vggface2-features.pt')
logits_path = state_dict_path = os.path.join(os.path.dirname(__file__), 'vggface2-dict/20180402-114759-vggface2-logits.pt')
state_dict = {}
for i, path in enumerate([features_path, logits_path]):
state_dict.update(torch.load(path))
mdl.load_state_dict(state_dict)
def get_torch_home():
torch_home = os.path.expanduser(
os.getenv(
'TORCH_HOME',
os.path.join(os.getenv('XDG_CACHE_HOME', '~/.cache'), 'torch')
)
)
return torch_home
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
import torch
import numpy as np
import time
class Logger(object):
def __init__(self, mode, length, calculate_mean=False):
self.mode = mode
self.length = length
self.calculate_mean = calculate_mean
if self.calculate_mean:
self.fn = lambda x, i: x / (i + 1)
else:
self.fn = lambda x, i: x
def __call__(self, loss, metrics, i):
track_str = '\r{} | {:5d}/{:<5d}| '.format(self.mode, i + 1, self.length)
loss_str = 'loss: {:9.4f} | '.format(self.fn(loss, i))
metric_str = ' | '.join('{}: {:9.4f}'.format(k, self.fn(v, i)) for k, v in metrics.items())
print(track_str + loss_str + metric_str + ' ', end='')
if i + 1 == self.length:
print('')
class BatchTimer(object):
"""Batch timing class.
Use this class for tracking training and testing time/rate per batch or per sample.
Keyword Arguments:
rate {bool} -- Whether to report a rate (batches or samples per second) or a time (seconds
per batch or sample). (default: {True})
per_sample {bool} -- Whether to report times or rates per sample or per batch.
(default: {True})
"""
def __init__(self, rate=True, per_sample=True):
self.start = time.time()
self.end = None
self.rate = rate
self.per_sample = per_sample
def __call__(self, y_pred, y):
self.end = time.time()
elapsed = self.end - self.start
self.start = self.end
self.end = None
if self.per_sample:
elapsed /= len(y_pred)
if self.rate:
elapsed = 1 / elapsed
return torch.tensor(elapsed)
def accuracy(logits, y):
_, preds = torch.max(logits, 1)
return (preds == y).float().mean()
def pass_epoch(
model, loss_fn, loader, optimizer=None, scheduler=None,
batch_metrics={'time': BatchTimer()}, show_running=True,
device='cpu', writer=None
):
"""Train or evaluate over a data epoch.
Arguments:
model {torch.nn.Module} -- Pytorch model.
loss_fn {callable} -- A function to compute (scalar) loss.
loader {torch.utils.data.DataLoader} -- A pytorch data loader.
Keyword Arguments:
optimizer {torch.optim.Optimizer} -- A pytorch optimizer.
scheduler {torch.optim.lr_scheduler._LRScheduler} -- LR scheduler (default: {None})
batch_metrics {dict} -- Dictionary of metric functions to call on each batch. The default
is a simple timer. A progressive average of these metrics, along with the average
loss, is printed every batch. (default: {{'time': iter_timer()}})
show_running {bool} -- Whether or not to print losses and metrics for the current batch
or rolling averages. (default: {False})
device {str or torch.device} -- Device for pytorch to use. (default: {'cpu'})
writer {torch.utils.tensorboard.SummaryWriter} -- Tensorboard SummaryWriter. (default: {None})
Returns:
tuple(torch.Tensor, dict) -- A tuple of the average loss and a dictionary of average
metric values across the epoch.
"""
mode = 'Train' if model.training else 'Valid'
logger = Logger(mode, length=len(loader), calculate_mean=show_running)
loss = 0
metrics = {}
for i_batch, (x, y) in enumerate(loader):
x = x.to(device)
y = y.to(device)
y_pred = model(x)
loss_batch = loss_fn(y_pred, y)
if model.training:
loss_batch.backward()
optimizer.step()
optimizer.zero_grad()
metrics_batch = {}
for metric_name, metric_fn in batch_metrics.items():
metrics_batch[metric_name] = metric_fn(y_pred, y).detach().cpu()
metrics[metric_name] = metrics.get(metric_name, 0) + metrics_batch[metric_name]
if writer is not None and model.training:
if writer.iteration % writer.interval == 0:
writer.add_scalars('loss', {mode: loss_batch.detach().cpu()}, writer.iteration)
for metric_name, metric_batch in metrics_batch.items():
writer.add_scalars(metric_name, {mode: metric_batch}, writer.iteration)
writer.iteration += 1
loss_batch = loss_batch.detach().cpu()
loss += loss_batch
if show_running:
logger(loss, metrics, i_batch)
else:
logger(loss_batch, metrics_batch, i_batch)
if model.training and scheduler is not None:
scheduler.step()
loss = loss / (i_batch + 1)
metrics = {k: v / (i_batch + 1) for k, v in metrics.items()}
if writer is not None and not model.training:
writer.add_scalars('loss', {mode: loss.detach()}, writer.iteration)
for metric_name, metric in metrics.items():
writer.add_scalars(metric_name, {mode: metric})
return loss, metrics
def collate_pil(x):
out_x, out_y = [], []
for xx, yy in x:
out_x.append(xx)
out_y.append(yy)
return out_x, out_y
import os
import torch
import numpy as np
import asyncio
import json
import base64
import websockets
from io import BytesIO
import pymysql
from datetime import datetime
from PIL import Image, ImageDraw
from IPython import display
from models.mtcnn import MTCNN
from models.inception_resnet_v1 import InceptionResnetV1
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
print('Running on device: {}'.format(device))
model = InceptionResnetV1().eval().to(device)
attendance_db = pymysql.connect(
user='root',
passwd='1234',
host='localhost',
db='attendance',
charset='utf8'
)
lock = asyncio.Lock()
clients = set()
#processes = []
async def get_embeddings(face_list):
global model
x = torch.Tensor(face_list).to(device)
yhat = model(x)
return yhat
async def get_distance(arr1, arr2):
distance = np.linalg.norm(arr1 - arr2)
return distance
async def get_cosine_similarity(arr1, arr2):
similarity = np.inner(arr1, arr2) / (np.linalg.norm(arr1) * np.linalg.norm(arr2))
return similarity
async def register(websocket):
global lock
global clients
async with lock:
clients.add(websocket)
remote_ip = websocket.remote_address[0]
msg='[{ip}] connected'.format(ip=remote_ip)
print(msg)
async def unregister(websocket):
global lock
global clients
async with lock:
clients.remove(websocket)
remote_ip = websocket.remote_address[0]
msg='[{ip}] disconnected'.format(ip=remote_ip)
print(msg)
async def thread(websocket, path):
await register(websocket)
try:
async for message in websocket:
data = json.loads(message)
remote_ip = websocket.remote_address[0]
if data['action'] == 'register':
# log
msg='[{ip}] register face'.format(ip=remote_ip)
print(msg)
# load json
student_id = data['student_id']
student_name = data['student_name']
face = np.asarray(data['MTCNN'], dtype = np.float32)
face = face.reshape((1,3,160,160))
# DB에 연결
cursor = attendance_db.cursor(pymysql.cursors.DictCursor)
# 학생을 찾음
sql = "SELECT student_id FROM student WHERE student_id = %s;"
cursor.execute(sql, (student_id))
# DB에 학생이 없으면 등록
if not cursor.fetchone():
sql = "INSERT INTO student(student_id, student_name) VALUES (%s, %s)"
cursor.execute(sql, (student_id, student_name))
sql = "INSERT INTO lecture_students(lecture_id, student_id) VALUES (%s, %s)"
cursor.execute(sql, ('0', student_id))
msg='[{ip}] {id} is registered'.format(ip=remote_ip, id=student_id)
print(msg)
# student_embedding Table에 등록
embedding = await get_embeddings(face)
embedding = embedding.detach().numpy().tobytes()
embedding_date = datetime.now().strftime('%Y-%m-%d')
sql = "insert into student_embedding(student_id, embedding_date, embedding) values (%s, %s, _binary %s)"
cursor.execute(sql, (student_id, embedding_date, embedding))
attendance_db.commit()
send = json.dumps({'status': 'success', 'student_id': student_id})
await websocket.send(send)
elif data['action'] == 'verify':
# log
msg='[{ip}] verify face'.format(ip=remote_ip)
print(msg)
# load json
face = np.asarray(data['MTCNN'], dtype = np.float32)
face = face.reshape((1,3,160,160))
embedding = await get_embeddings(face)
embedding = embedding.detach().numpy()
# 가장 비슷한 Embedding을 찾는 SQL
cursor = attendance_db.cursor(pymysql.cursors.DictCursor)
sql = "SELECT student_id, embedding FROM student_embedding;"
cursor.execute(sql)
result = cursor.fetchall()
verified_id = '0'
distance_min = 99
for row_data in result:
db_embedding = np.frombuffer(row_data['embedding'], dtype=np.float32)
db_embedding = db_embedding.reshape((1,512))
distance = await get_distance(embedding, db_embedding)
if (distance < distance_min):
verified_id = row_data['student_id']
distance_min = distance
# 출석 데이터 전송
print('[debug] distance:', distance_min)
send = ''
if distance_min < 0.4:
# 인증 성공
# 오늘 이미 출석 됐는지 확인
sql = "SELECT DATE(timestamp) FROM student_attendance WHERE (lecture_id=%s) AND (student_id=%s) AND (DATE(timestamp) = CURDATE());"
cursor.execute(sql, ('0', verified_id))
# 출석 기록이 없는 경우에만
if not cursor.fetchone():
# 테이블 맨 뒤에 datetime attribute가 있음. 서버 시간 가져오게 default로 설정해둠.
sql = "INSERT INTO student_attendance(lecture_id, student_id, status) VALUES (%s, %s, %s)"
# TODO: attend / late 처리
cursor.execute(sql, ('0', verified_id, 'attend'))
attendance_db.commit()
# log 작성
msg='[{ip}] verification success {id}'.format(ip=remote_ip, id=verified_id)
print(msg)
send = json.dumps({'status': 'success', 'student_id': verified_id})
else:
msg='[{ip}] verification failed: {id} is already verified'.format(ip=remote_ip, id=verified_id)
print(msg)
send = json.dumps({'status': 'already', 'student_id': verified_id})
else:
# 인증 실패
msg='[{ip}] verification failed'.format(ip=remote_ip)
print(msg)
send = json.dumps({'status': 'fail'})
await websocket.send(send)
elif data['action'] == "save_image":
# 출석이 제대로 이뤄지지 않으면 이미지를 저장하여
# 나중에 교강사가 출석을 확인할 수 있도록 한다
msg='[{ip}] save image'.format(ip=remote_ip)
print(msg)
arr = np.asarray(data['image'], dtype = np.uint8)
blob = arr.tobytes()
# TODO: lecture DB에 tuple 삽입해야 아래 코드가 돌아감
# 테이블 맨 뒤에 datetime attribute가 있음. 서버 시간 가져오게 default로 설정해둠.
cursor = attendance_db.cursor(pymysql.cursors.DictCursor)
sql = "INSERT INTO undefined_image(lecture_id, image, width, height) VALUES (%s, _binary %s, %s, %s)"
cursor.execute(sql, ('0', blob, arr.shape[0], arr.shape[1]))
attendance_db.commit()
else:
print("unsupported event: {}", data)
finally:
await unregister(websocket)
print('run verification server')
start_server = websockets.serve(thread, '0.0.0.0', 8765)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()