askill
/
Video-Summary
mirror of https://github.com/Askill/Video-Summary.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed layer merging

This commit is contained in:
Askill 2020-12-18 21:27:19 +01:00
parent 34ebcf5f8c
commit 7baa75dc19
7 changed files with 145 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Application/Config.py
View File

@ -1,20 +1,19 @@
class Config: class Config:
c = { c = {
"min_area" : 100, "min_area" : 5000,
"max_area" : 40000, "max_area" : 40000,
"threashold" : 8, "threashold" : 7,
"resizeWidth" : 512, "resizeWidth" : 512,
"inputPath" : None, "inputPath" : None,
"outputPath": None, "outputPath": None,
"maxLayerLength": 900, "maxLayerLength": 1500,
"minLayerLength": 20, "minLayerLength": 40,
"tolerance": 5, "tolerance": 20,
"maxLength": None, "maxLength": None,
"ttolerance": 60, "ttolerance": 20,
"videoBufferLength": 500, "videoBufferLength": 500,
"noiseThreashold": 0.4, "LayersPerContour": 220,
"LayersPerContour": 20,
"avgNum":10 "avgNum":10
} }

14
Application/Exporter.py
View File

@ -45,20 +45,14 @@ class Exporter:
if len(layer.bounds[0]) == 0: if len(layer.bounds[0]) == 0:
continue continue
if layer.stats["dev"] < 5: videoReader = VideoReader(self.config)
continue
listOfFrames = self.makeListOfFrames([layer]) listOfFrames = self.makeListOfFrames([layer])
videoReader = VideoReader(self.config, listOfFrames) videoReader.fillBuffer(listOfFrames)
videoReader.fillBuffer()
while not videoReader.videoEnded(): while not videoReader.videoEnded():
frameCount, frame = videoReader.pop() frameCount, frame = videoReader.pop()
if frameCount % (60*self.fps) == 0:
print("Minutes processed: ", frameCount/(60*self.fps))
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
frame2 = np.copy(underlay) frame2 = np.copy(underlay)
@ -77,8 +71,10 @@ class Exporter:
cv2.putText(frame2, str(layer.stats["avg"]) + " " + str(layer.stats["var"]) + " " + str(layer.stats["dev"]), (int(500), int(500)), cv2.FONT_HERSHEY_SIMPLEX, 1,(255,0,255), 2) cv2.putText(frame2, str(layer.stats["avg"]) + " " + str(layer.stats["var"]) + " " + str(layer.stats["dev"]), (int(500), int(500)), cv2.FONT_HERSHEY_SIMPLEX, 1,(255,0,255), 2)
writer.append_data(frame2) writer.append_data(frame2)
videoReader.vc.release()
videoReader.thread.join() videoReader.thread.join()
videoReader.vc.release() videoReader.vc.release()
videoReader.thread.join()
writer.close() writer.close()
@ -159,6 +155,6 @@ class Exporter:
frameNumbers = set() frameNumbers = set()
for layer in layers: for layer in layers:
frameNumbers.update( frameNumbers.update(
list(range(layer.startFrame, layer.startFrame + len(layer.bounds)))) list(range(layer.startFrame, layer.startFrame + len(layer))))
return sorted(list(frameNumbers)) return sorted(list(frameNumbers))

16
Application/Layer.py
View File

@ -34,16 +34,20 @@ class Layer:
def add(self, frameNumber, bound, mask): def add(self, frameNumber, bound, mask):
'''Adds a bound to the Layer at the layer index which corresponds to the given framenumber''' '''Adds a bound to the Layer at the layer index which corresponds to the given framenumber'''
if self.startFrame + len(self.bounds) - 1 > frameNumber: index = frameNumber - self.startFrame
if len(self.bounds[frameNumber - self.startFrame]) >= 1:
self.bounds[frameNumber - self.startFrame].append(bound) if frameNumber > self.lastFrame:
self.masks[frameNumber - self.startFrame].append(mask) for i in range(frameNumber - self.lastFrame):
else:
while len(self.bounds) + self.startFrame < frameNumber:
self.bounds.append([bound]) self.bounds.append([bound])
self.masks.append([mask]) self.masks.append([mask])
self.lastFrame = frameNumber self.lastFrame = frameNumber
if bound not in self.bounds[index]:
self.bounds[index].append(bound)
self.masks[index].append(mask)
def calcStats(self): def calcStats(self):
'''calculates average distance, variation and deviation of layer movement''' '''calculates average distance, variation and deviation of layer movement'''
middles = [] middles = []

147
Application/LayerFactory.py
View File

@ -7,6 +7,7 @@ import cv2
import numpy as np import numpy as np
import copy import copy
class LayerFactory: class LayerFactory:
def __init__(self, config, data=None): def __init__(self, config, data=None):
self.data = {} self.data = {}
@ -40,99 +41,117 @@ class LayerFactory:
for frameNumber in sorted(data.keys()): for frameNumber in sorted(data.keys()):
contours = data[frameNumber] contours = data[frameNumber]
masks = maskArr[frameNumber] masks = maskArr[frameNumber]
masks = [np.unpackbits(mask, axis=0) for mask, contours in zip(masks, contours)] masks = [np.unpackbits(mask, axis=0)
if frameNumber%100 == 0: for mask, contours in zip(masks, contours)]
print(f" {int(round(frameNumber/max(data.keys()), 2)*100)}% done with Layer extraction", end='\r') if frameNumber % 100 == 0:
print(
f" {int(round(frameNumber/max(data.keys()), 2)*100)}% done with Layer extraction {len(self.layers)} Layers", end='\r')
tmp = [[frameNumber, contour, mask] for contour, mask in zip(contours, masks)] tmp = [[frameNumber, contour, mask]
for contour, mask in zip(contours, masks)]
#pool.map(self.getLayers, tmp) #pool.map(self.getLayers, tmp)
for x in tmp: for x in tmp:
self.getLayers(x) self.getLayers(x)
#self.joinLayers()
return self.layers return self.layers
def getLayers(self, data): def getLayers(self, data):
frameNumber = data[0] frameNumber = data[0]
bounds = data[1] bounds = data[1]
mask = data[2] mask = data[2]
(x,y,w,h) = bounds (x, y, w, h) = bounds
tol = self.tolerance tol = self.tolerance
foundLayer = 0
#to merge layers
foundLayerIDs = []
for i in range(0, len(self.layers)): foundLayerIDs = set()
if foundLayer >= self.config["LayersPerContour"]: for i, layer in enumerate(self.layers):
break if frameNumber - layer.lastFrame > self.ttolerance:
if i in self.oldLayerIDs:
continue
if frameNumber - self.layers[i].lastFrame > self.ttolerance:
self.oldLayerIDs.append(i)
continue continue
lastXframes = 5 lastXframes = min(40, len(layer))
if len(self.layers[i].bounds) < lastXframes: lastBounds = [bound for bounds in layer.bounds[-lastXframes:]
lastXframes = len(self.layers[i].bounds) for bound in bounds]
lastBounds = [bound for bounds in self.layers[i].bounds[-lastXframes:] for bound in bounds]
for j, bounds in enumerate(lastBounds): for j, bounds in enumerate(lastBounds[::-1]):
if bounds is None: if bounds is None:
break break
(x2,y2,w2,h2) = bounds (x2, y2, w2, h2) = bounds
if self.contoursOverlay((x-tol,y+h+tol), (x+w+tol,y-tol), (x2,y2+h2), (x2+w2,y2)): if self.contoursOverlay((x-tol, y+h+tol), (x+w+tol, y-tol), (x2, y2+h2), (x2+w2, y2)):
self.layers[i].add(frameNumber, (x,y,w,h), mask) layer.add(frameNumber, (x, y, w, h), mask)
foundLayer += 1 foundLayerIDs.add(i)
foundLayerIDs.append(i)
break break
if foundLayer == 0: foundLayerIDs = sorted(list(foundLayerIDs))
self.layers.append(Layer(frameNumber, (x,y,w,h), mask, self.config)) if len(foundLayerIDs) == 0:
self.layers.append(
Layer(frameNumber, (x, y, w, h), mask, self.config))
if len(foundLayerIDs) > 1: if len(foundLayerIDs) > 1:
self.mergeLayers(foundLayerIDs) self.mergeLayers(foundLayerIDs)
def mergeLayers(self, foundLayerIDs): def mergeLayers(self, foundLayerIDs):
layers = self.getLayersByID(foundLayerIDs)
layers = self.sortLayers(foundLayerIDs)
layer1 = layers[0] layer1 = layers[0]
for layerID in range(0, len(layers)): for layer in layers[1:]:
layer2 = layers[layerID] for i, (contours, masks) in enumerate(zip(layer.bounds, layer.masks)):
layer1 = self.merge2Layers(layer1, layer2) for contour, mask in zip(contours, masks):
layer1.add(layer.startFrame, contour, mask)
self.layers[foundLayerIDs[0]] = layer1 for i, id in enumerate(foundLayerIDs):
del self.layers[id - i]
layers = [] self.layers.append(layer1)
def joinLayers(self):
self.layers.sort(key=lambda c: c.startFrame)
minFrame = self.getMinStart(self.layers)
maxFrame = self.getMaxEnd(self.layers)
for i in range(minFrame, maxFrame):
pL, indexes = self.getPossibleLayers(i)
if len(pL) <= 1:
continue
merge = set()
innerMax = self.getMaxEnd(pL)
for x in range(self.getMinStart(pL), innerMax):
for lc, l in enumerate(pL):
if l.startFrame < x or l.lastFrame > x:
continue
for lc2, l2 in enumerate(pL):
if lc2 == lc:
continue
for cnt in l.bounds[x-l.startFrame]:
for cnt2 in l2.bounds[x-l2.startFrame]:
if self.contoursOverlay(cnt, cnt2):
merge.add(indexes[lc])
merge.add(indexes[lc2])
merge = list(merge)
if len(merge) > 1:
self.mergeLayers(megre)
i = innerMax
def getPossibleLayers(self, t):
ret = []
ii = []
for i, layer in enumerate(self.layers): for i, layer in enumerate(self.layers):
if i not in foundLayerIDs[1:]: if layer.startFrame <= t and layer.lastFrame <= t:
layers.append(layer) ret.append(layer)
self.layers = layers ii.append(i)
return (ret, ii)
def getMinStart(self, layers):
minFrame = layers[0].startFrame
for l in layers:
if l.startFrame < minFrame:
minFrame = l.startFrame
return minFrame
def merge2Layers(self, layer1, layer2): def getMaxEnd(self, layers):
"""Merges 2 Layers, Layer1 must start before Layer2""" maxFrame = layers[0].lastFrame
for l in layers:
dSF = layer2.startFrame - layer1.startFrame if l.lastFrame < maxFrame:
l1bounds = copy.deepcopy(layer1.bounds) maxFrame = l.lastFrame
l1masks = copy.deepcopy(layer1.masks)
for i in range(len(layer2.bounds)):
bounds = layer2.bounds[i]
masks = layer2.masks[i]
while dSF + i >= len(l1bounds):
l1bounds.append([])
while dSF + i >= len(l1masks):
l1masks.append([])
for bound, mask in zip(bounds, masks):
if bound not in l1bounds[dSF + i]:
l1bounds[dSF + i].append(bound)
l1masks[dSF + i].append(mask)
layer1.bounds = l1bounds
layer1.masks = l1masks
return layer1
return maxFrame
def contoursOverlay(self, l1, r1, l2, r2): def contoursOverlay(self, l1, r1, l2, r2):
# If one rectangle is on left side of other # If one rectangle is on left side of other
@ -143,10 +162,10 @@ class LayerFactory:
return False return False
return True return True
def sortLayers(self, foundLayerIDs): def getLayersByID(self, foundLayerIDs):
layers = [] layers = []
for layerID in foundLayerIDs: for layerID in foundLayerIDs:
layers.append(self.layers[layerID]) layers.append(self.layers[layerID])
layers.sort(key = lambda c:c.startFrame) layers.sort(key=lambda c: c.startFrame)
return layers return layers

20
Application/LayerManager.py
View File

@ -19,16 +19,30 @@ class LayerManager:
self.resizeWidth = config["resizeWidth"] self.resizeWidth = config["resizeWidth"]
self.footagePath = config["inputPath"] self.footagePath = config["inputPath"]
self.config = config self.config = config
self.classifier = Classifier() #self.classifier = Classifier()
self.tags = [] self.tags = []
print("LayerManager constructed") print("LayerManager constructed")
def transformLayers(self): def transformLayers(self):
print("'Cleaning' Layers") print("'Cleaning' Layers")
print("Before deleting short layers ", len(self.layers))
self.freeMin() self.freeMin()
print("Before deleting long layers ", len(self.layers))
self.freeMax() self.freeMax()
self.sortLayers() self.sortLayers()
self.calcStats() self.calcStats()
self.deleteSparse()
print("after deleting sparse layers ", len(self.layers))
def deleteSparse(self):
toDelete = []
for i, l in enumerate(self.layers):
empty = l.bounds.count([])
if empty / len(l) > 0.2:
toDelete.append(i)
for i, id in enumerate(toDelete):
del self.layers[id - i]
def calcStats(self): def calcStats(self):
for layer in self.layers: for layer in self.layers:
@ -58,7 +72,7 @@ class LayerManager:
self.data.clear() self.data.clear()
layers = [] layers = []
for l in self.layers: for l in self.layers:
if l.getLength() > self.minLayerLength: if len(l) > self.minLayerLength:
layers.append(l) layers.append(l)
self.layers = layers self.layers = layers
@ -66,7 +80,7 @@ class LayerManager:
def freeMax(self): def freeMax(self):
layers = [] layers = []
for l in self.layers: for l in self.layers:
if l.getLength() < self.maxLayerLength: if len(l) < self.maxLayerLength:
layers.append(l) layers.append(l)
self.layers = layers self.layers = layers

4
Application/VideoReader.py
View File

@ -38,10 +38,12 @@ class VideoReader:
def pop(self): def pop(self):
return self.buffer.get(block=True) return self.buffer.get(block=True)
def fillBuffer(self): def fillBuffer(self, listOfFrames=None):
if self.buffer.full(): if self.buffer.full():
print("VideoReader::fillBuffer was called when buffer was full.") print("VideoReader::fillBuffer was called when buffer was full.")
self.endFrame = int(self.vc.get(cv2.CAP_PROP_FRAME_COUNT)) self.endFrame = int(self.vc.get(cv2.CAP_PROP_FRAME_COUNT))
if listOfFrames is not None:
self.listOfFrames = listOfFrames
if self.listOfFrames is not None: if self.listOfFrames is not None:
self.thread = threading.Thread(target=self.readFramesByList, args=()) self.thread = threading.Thread(target=self.readFramesByList, args=())

10
main.py
View File

@ -18,7 +18,7 @@ def main():
start = startTotal start = startTotal
config = Config() config = Config()
fileName = "X23-1.mp4" fileName = "3.mp4"
outputPath = os.path.join(os.path.dirname(__file__), "output") outputPath = os.path.join(os.path.dirname(__file__), "output")
dirName = os.path.join(os.path.dirname(__file__), "generate test footage") dirName = os.path.join(os.path.dirname(__file__), "generate test footage")
@ -29,9 +29,6 @@ def main():
config["w"], config["h"] = VideoReader(config).getWH() config["w"], config["h"] = VideoReader(config).getWH()
stats = dict() stats = dict()
stats["File Name"] = config["importPath"]
stats["threads"] = "16, 16"
stats["buffer"] = config["bufferLength"]
if not os.path.exists(config["importPath"]): if not os.path.exists(config["importPath"]):
contours, masks = ContourExtractor(config).extractContours() contours, masks = ContourExtractor(config).extractContours()
stats["Contour Extractor"] = time.time() - start stats["Contour Extractor"] = time.time() - start
@ -54,9 +51,12 @@ def main():
#layerManager.tagLayers() #layerManager.tagLayers()
layers = layerManager.layers layers = layerManager.layers
print([len(l) for l in sorted(layers, key = lambda c:len(c), reverse=True)[:20]])
if len(layers) == 0:
exit(1)
exporter = Exporter(config) exporter = Exporter(config)
print(f"Exporting {len(contours)} Contours and {len(layers)} Layers") print(f"Exporting {len(contours)} Contours and {len(layers)} Layers")
exporter.export(layers, contours, masks, raw=True, overlayed=True) exporter.export(layers, contours, masks, raw=True, overlayed=False)
stats["Exporter"] = time.time() - start stats["Exporter"] = time.time() - start
print("Total time: ", time.time() - startTotal) print("Total time: ", time.time() - startTotal)