完整代码
liujf69/Data-Processing/Event_Camera
1--代码和数据参考
什么是事件相机，计算机视觉中事件相机这一方向需要学哪些相关知识？
THU-EACT-50
事件相机的数据格式一般是一个四元组 (t, x, y, p)。
t表示事件发生的事件戳。
x, y表示事件发生的像素位置
p表示极性，-1表示亮度降低，+1表示亮度升高。
2--处理代码
2-1--Channel-by-channel accumulation
将events按时间戳累积到不同通道内。
import cv2
import numpy as np# Channel-by-channel accumulation
# 将events按时间戳累积到不同通道内
def gen_events_array(events_ori, C_event, duration, event_h, event_w):events = np.zeros((event_h, event_w, C_event))C_inter = duration / C_eventfor i in range(events_ori.shape[0]):W, H, t, p = events_ori[i]p = -1 if p == 0 else pevents[int(H), int(W), min(int(t // C_inter), C_event - 1)] += preturn eventsdef vis_event(gray):h,w = gray.shapeout = 255*np.ones([h, w, 3])pos_weight = gray.copy()neg_weight = gray.copy()pos_weight[pos_weight < 0] = 0pos_weight = pos_weight * 2 * 255neg_weight[neg_weight > 0] = 0neg_weight = abs(neg_weight) * 2 * 255out[..., 1] = out[...,1] - pos_weight - neg_weightout[..., 0] -= pos_weightout[..., 2] -= neg_weightout = out.clip(0, 255)return out.astype(np.uint8)if __name__ == "__main__":file_name = "./A0P10C0-2021_11_04_12_30_08.npy"events_ori = np.load(file_name, allow_pickle = True) # N W H T Pevent_w = int(np.max(events_ori[:, 0]) + 1) # max_W event_h = int(np.max(events_ori[:, 1]) + 1) # max_HC_event = 5events_ori[:, 2] = events_ori[:, 2] - events_ori[:, 2].min() # Tevents_ori[:, 3] = (events_ori[:, 3] - 0.5)*2 # Pduration = events_ori[:, 2].max() - events_ori[:, 2].min() # Tevents1 = gen_events_array(events_ori, C_event, duration, event_h, event_w) # event_h event_w C_eventevents1_vis = events1.clip(-5, 5) / 10cv2.imwrite('./events1_vis.jpg', vis_event(events1_vis[..., 2]))2-2--Voxel grid using temporal bilinear interpolation
import cv2
import torch
import numpy as np# Voxel grid using temporal bilinear interpolation
def events_to_arr(xs, ys, ps, events_h, events_w, accumulate=True):"""Accumulate events into an image."""img_size = [events_h, events_w]img = torch.zeros(img_size, dtype=ys.dtype)if xs.dtype is not torch.long:xs = xs.long()if ys.dtype is not torch.long:ys = ys.long()img.index_put_((ys, xs), ps, accumulate=accumulate)return imgdef gen_events_voxel(events_ori, num_bins, duration, events_h, events_w, round_ts=False):"""Generate a voxel grid from input events using temporal bilinear interpolation."""events_ori = torch.from_numpy(events_ori.copy())xs, ys, ts, ps = events_ori[:, 0], events_ori[:, 1], events_ori[:, 2], events_ori[:, 3]voxel = []ts = ts * (num_bins - 1) / durationif round_ts:ts = torch.round(ts)zeros = torch.zeros(ts.shape)for b_idx in range(num_bins):weights = torch.max(zeros, 1.0 - torch.abs(ts - b_idx))voxel_bin = events_to_arr(xs, ys, ps * weights, events_h, events_w)voxel.append(voxel_bin)return torch.stack(voxel).numpy().transpose(1, 2, 0)def vis_event(gray):h,w = gray.shapeout = 255*np.ones([h, w, 3])pos_weight = gray.copy()neg_weight = gray.copy()pos_weight[pos_weight < 0] = 0pos_weight = pos_weight * 2 * 255neg_weight[neg_weight > 0] = 0neg_weight = abs(neg_weight) * 2 * 255out[..., 1] = out[...,1] - pos_weight - neg_weightout[..., 0] -= pos_weightout[..., 2] -= neg_weightout = out.clip(0, 255)return out.astype(np.uint8)if __name__ == "__main__":file_name = "./A0P10C0-2021_11_04_12_30_08.npy"events_ori = np.load(file_name, allow_pickle = True) # N W H T Pevent_w = int(np.max(events_ori[:, 0]) + 1) # max_W event_h = int(np.max(events_ori[:, 1]) + 1) # max_HC_event = 5events_ori[:, 2] = events_ori[:, 2] - events_ori[:, 2].min() # Tevents_ori[:, 3] = (events_ori[:, 3] - 0.5)*2 # Pduration = events_ori[:, 2].max() - events_ori[:, 2].min() # Tevents2 = gen_events_voxel(events_ori.astype(np.float32), C_event, duration, event_h, event_w, round_ts=False)events2_vis = events2.clip(-5,5) / 10cv2.imwrite('events2_vis.jpg',vis_event(events2_vis[...,2]))print("All Done!")2-3--Symmetric Cumulative Event Representation
import cv2
import torch
import numpy as np# Symmetric Cumulative Event Representation
def events_to_image_torch(xs, ys, ps,device=None, sensor_size=(180, 240), clip_out_of_range=True,interpolation=None, padding=True, default=0):if device is None:device = xs.deviceif interpolation == 'bilinear' and padding:img_size = (sensor_size[0]+1, sensor_size[1]+1)else:img_size = list(sensor_size)mask = torch.ones(xs.size(), device=device)if clip_out_of_range:zero_v = torch.tensor([0.], device=device)ones_v = torch.tensor([1.], device=device)clipx = img_size[1] if interpolation is None and padding==False else img_size[1]-1clipy = img_size[0] if interpolation is None and padding==False else img_size[0]-1mask = torch.where(xs>=clipx, zero_v, ones_v)*torch.where(ys>=clipy, zero_v, ones_v)img = (torch.ones(img_size)*default).to(device)img = img.float()if xs.dtype is not torch.long:xs = xs.long().to(device)if ys.dtype is not torch.long:ys = ys.long().to(device)try:mask = mask.long().to(device)xs, ys = xs*mask, ys*maskimg.index_put_((ys, xs), ps, accumulate=True)# print("able to put tensor {} positions ({}, {}) into {}. Range = {},{}".format(# ps.shape, ys.shape, xs.shape, img.shape, torch.max(ys), torch.max(xs)))except Exception as e:print("Unable to put tensor {} positions ({}, {}) into {}. Range = {},{}".format(ps.shape, ys.shape, xs.shape, img.shape, torch.max(ys), torch.max(xs)))raise ereturn imgdef gen_events_voxel_v2(events_ori, B, device=None, sensor_size=(180, 240), keep_middle=True):events_ori = torch.from_numpy(events_ori)events_ori = events_ori.float()# xs, ys, ts, ps = events_ori[:,1], events_ori[:,2], events_ori[:,0], events_ori[:,3]xs, ys, ts, ps = events_ori[:, 0], events_ori[:, 1], events_ori[:, 2], events_ori[:, 3]if device is None:device = xs.deviceassert(len(xs)==len(ys) and len(ys)==len(ts) and len(ts)==len(ps))bins = []dt = ts[-1]-ts[0]t_mid = ts[0] + (dt/2)# left of the mid -tend = t_midend = binary_search_torch_tensor(ts, 0, len(ts)-1, tend)for bi in range(int(B/2)):tstart = ts[0] + (dt/B)*bibeg = binary_search_torch_tensor(ts, 0, len(ts)-1, tstart)vb = events_to_image_torch(xs[beg:end], ys[beg:end],ps[beg:end], device, sensor_size=sensor_size,clip_out_of_range=False)bins.append(-vb) # !# selfif keep_middle:bins.append(torch.zeros_like(vb)) # TODO!!!# right of the mid +tstart = t_midbeg = binary_search_torch_tensor(ts, 0, len(ts)-1, tstart)for bi in range(int(B/2), B):tend = ts[0] + (dt/B)*(bi+1)end = binary_search_torch_tensor(ts, 0, len(ts)-1, tend)vb = events_to_image_torch(xs[beg:end], ys[beg:end],ps[beg:end], device, sensor_size=sensor_size,clip_out_of_range=False)bins.append(vb)bins = torch.stack(bins)bins = bins.numpy().transpose(1,2,0)return binsdef binary_search_torch_tensor(t, l, r, x, side='left'):if r is None:r = len(t)-1while l <= r:mid = l + (r - l)//2midval = t[mid]if midval == x:return midelif midval < x:l = mid + 1else:r = mid - 1if side == 'left':return lreturn rdef vis_event(gray):h,w = gray.shapeout = 255*np.ones([h, w, 3])pos_weight = gray.copy()neg_weight = gray.copy()pos_weight[pos_weight < 0] = 0pos_weight = pos_weight * 2 * 255neg_weight[neg_weight > 0] = 0neg_weight = abs(neg_weight) * 2 * 255out[..., 1] = out[...,1] - pos_weight - neg_weightout[..., 0] -= pos_weightout[..., 2] -= neg_weightout = out.clip(0, 255)return out.astype(np.uint8)if __name__ == "__main__":file_name = "./A0P10C0-2021_11_04_12_30_08.npy"events_ori = np.load(file_name, allow_pickle = True) # N W H T Pevent_w = int(np.max(events_ori[:, 0]) + 1) # max_W event_h = int(np.max(events_ori[:, 1]) + 1) # max_HC_event = 5events_ori[:, 2] = events_ori[:, 2] - events_ori[:, 2].min() # Tevents_ori[:, 3] = (events_ori[:, 3] - 0.5)*2 # Pduration = events_ori[:, 2].max() - events_ori[:, 2].min() # Tevents3 = gen_events_voxel_v2(events_ori, C_event, device=None, sensor_size=(event_h, event_w), keep_middle=False)events3_vis = events3.clip(-5,5) / 10cv2.imwrite('events3_vis.jpg',vis_event(events3_vis[...,1]))print("All Done!")未完待续！