From 7baa75dc193eeffd4c269f66e154df6596a7df50 Mon Sep 17 00:00:00 2001 From: Askill Date: Fri, 18 Dec 2020 21:27:19 +0100 Subject: [PATCH] fixed layer merging --- Application/Config.py | 15 ++-- Application/Exporter.py | 18 ++-- Application/Layer.py | 16 ++-- Application/LayerFactory.py | 169 ++++++++++++++++++++---------------- Application/LayerManager.py | 22 ++++- Application/VideoReader.py | 6 +- main.py | 10 +-- 7 files changed, 145 insertions(+), 111 deletions(-) diff --git a/Application/Config.py b/Application/Config.py index 3ba1602..3234fe9 100644 --- a/Application/Config.py +++ b/Application/Config.py @@ -1,20 +1,19 @@ class Config: c = { - "min_area" : 100, + "min_area" : 5000, "max_area" : 40000, - "threashold" : 8, + "threashold" : 7, "resizeWidth" : 512, "inputPath" : None, "outputPath": None, - "maxLayerLength": 900, - "minLayerLength": 20, - "tolerance": 5, + "maxLayerLength": 1500, + "minLayerLength": 40, + "tolerance": 20, "maxLength": None, - "ttolerance": 60, + "ttolerance": 20, "videoBufferLength": 500, - "noiseThreashold": 0.4, - "LayersPerContour": 20, + "LayersPerContour": 220, "avgNum":10 } diff --git a/Application/Exporter.py b/Application/Exporter.py index fb082a4..9115f01 100644 --- a/Application/Exporter.py +++ b/Application/Exporter.py @@ -45,20 +45,14 @@ class Exporter: if len(layer.bounds[0]) == 0: continue - if layer.stats["dev"] < 5: - continue - + videoReader = VideoReader(self.config) listOfFrames = self.makeListOfFrames([layer]) - - videoReader = VideoReader(self.config, listOfFrames) - videoReader.fillBuffer() + + videoReader.fillBuffer(listOfFrames) while not videoReader.videoEnded(): frameCount, frame = videoReader.pop() - if frameCount % (60*self.fps) == 0: - print("Minutes processed: ", frameCount/(60*self.fps)) - frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) frame2 = np.copy(underlay) @@ -77,8 +71,10 @@ class Exporter: cv2.putText(frame2, str(layer.stats["avg"]) + " " + str(layer.stats["var"]) + " " + str(layer.stats["dev"]), (int(500), int(500)), cv2.FONT_HERSHEY_SIMPLEX, 1,(255,0,255), 2) writer.append_data(frame2) - videoReader.thread.join() videoReader.vc.release() + videoReader.thread.join() + videoReader.vc.release() + videoReader.thread.join() writer.close() @@ -159,6 +155,6 @@ class Exporter: frameNumbers = set() for layer in layers: frameNumbers.update( - list(range(layer.startFrame, layer.startFrame + len(layer.bounds)))) + list(range(layer.startFrame, layer.startFrame + len(layer)))) return sorted(list(frameNumbers)) diff --git a/Application/Layer.py b/Application/Layer.py index 9deb9b2..b16b3f4 100644 --- a/Application/Layer.py +++ b/Application/Layer.py @@ -34,16 +34,20 @@ class Layer: def add(self, frameNumber, bound, mask): '''Adds a bound to the Layer at the layer index which corresponds to the given framenumber''' - if self.startFrame + len(self.bounds) - 1 > frameNumber: - if len(self.bounds[frameNumber - self.startFrame]) >= 1: - self.bounds[frameNumber - self.startFrame].append(bound) - self.masks[frameNumber - self.startFrame].append(mask) - else: - while len(self.bounds) + self.startFrame < frameNumber: + index = frameNumber - self.startFrame + + if frameNumber > self.lastFrame: + for i in range(frameNumber - self.lastFrame): self.bounds.append([bound]) self.masks.append([mask]) + self.lastFrame = frameNumber + if bound not in self.bounds[index]: + self.bounds[index].append(bound) + self.masks[index].append(mask) + + def calcStats(self): '''calculates average distance, variation and deviation of layer movement''' middles = [] diff --git a/Application/LayerFactory.py b/Application/LayerFactory.py index 73d1df2..b152237 100644 --- a/Application/LayerFactory.py +++ b/Application/LayerFactory.py @@ -7,6 +7,7 @@ import cv2 import numpy as np import copy + class LayerFactory: def __init__(self, config, data=None): self.data = {} @@ -33,120 +34,138 @@ class LayerFactory: for contour, mask in zip(contours, masks): mask = np.unpackbits(mask, axis=0) self.layers.append(Layer(frameNumber, contour, mask, self.config)) - + self.oldLayerIDs = [] - + with ThreadPool(16) as pool: for frameNumber in sorted(data.keys()): contours = data[frameNumber] masks = maskArr[frameNumber] - masks = [np.unpackbits(mask, axis=0) for mask, contours in zip(masks, contours)] - if frameNumber%100 == 0: - print(f" {int(round(frameNumber/max(data.keys()), 2)*100)}% done with Layer extraction", end='\r') + masks = [np.unpackbits(mask, axis=0) + for mask, contours in zip(masks, contours)] + if frameNumber % 100 == 0: + print( + f" {int(round(frameNumber/max(data.keys()), 2)*100)}% done with Layer extraction {len(self.layers)} Layers", end='\r') - tmp = [[frameNumber, contour, mask] for contour, mask in zip(contours, masks)] + tmp = [[frameNumber, contour, mask] + for contour, mask in zip(contours, masks)] #pool.map(self.getLayers, tmp) for x in tmp: self.getLayers(x) + #self.joinLayers() return self.layers def getLayers(self, data): frameNumber = data[0] bounds = data[1] mask = data[2] - (x,y,w,h) = bounds + (x, y, w, h) = bounds tol = self.tolerance - foundLayer = 0 - #to merge layers - foundLayerIDs = [] - for i in range(0, len(self.layers)): - if foundLayer >= self.config["LayersPerContour"]: - break - - if i in self.oldLayerIDs: + foundLayerIDs = set() + for i, layer in enumerate(self.layers): + if frameNumber - layer.lastFrame > self.ttolerance: continue - if frameNumber - self.layers[i].lastFrame > self.ttolerance: - self.oldLayerIDs.append(i) - continue - - lastXframes = 5 - if len(self.layers[i].bounds) < lastXframes: - lastXframes = len(self.layers[i].bounds) - lastBounds = [bound for bounds in self.layers[i].bounds[-lastXframes:] for bound in bounds] - for j, bounds in enumerate(lastBounds): + lastXframes = min(40, len(layer)) + lastBounds = [bound for bounds in layer.bounds[-lastXframes:] + for bound in bounds] + + for j, bounds in enumerate(lastBounds[::-1]): if bounds is None: break - (x2,y2,w2,h2) = bounds - if self.contoursOverlay((x-tol,y+h+tol), (x+w+tol,y-tol), (x2,y2+h2), (x2+w2,y2)): - self.layers[i].add(frameNumber, (x,y,w,h), mask) - foundLayer += 1 - foundLayerIDs.append(i) + (x2, y2, w2, h2) = bounds + if self.contoursOverlay((x-tol, y+h+tol), (x+w+tol, y-tol), (x2, y2+h2), (x2+w2, y2)): + layer.add(frameNumber, (x, y, w, h), mask) + foundLayerIDs.add(i) break - if foundLayer == 0: - self.layers.append(Layer(frameNumber, (x,y,w,h), mask, self.config)) - + foundLayerIDs = sorted(list(foundLayerIDs)) + if len(foundLayerIDs) == 0: + self.layers.append( + Layer(frameNumber, (x, y, w, h), mask, self.config)) if len(foundLayerIDs) > 1: self.mergeLayers(foundLayerIDs) def mergeLayers(self, foundLayerIDs): - - layers = self.sortLayers(foundLayerIDs) + layers = self.getLayersByID(foundLayerIDs) layer1 = layers[0] - for layerID in range(0, len(layers)): - layer2 = layers[layerID] - layer1 = self.merge2Layers(layer1, layer2) - - self.layers[foundLayerIDs[0]] = layer1 + for layer in layers[1:]: + for i, (contours, masks) in enumerate(zip(layer.bounds, layer.masks)): + for contour, mask in zip(contours, masks): + layer1.add(layer.startFrame, contour, mask) - layers = [] + for i, id in enumerate(foundLayerIDs): + del self.layers[id - i] + + self.layers.append(layer1) + + def joinLayers(self): + self.layers.sort(key=lambda c: c.startFrame) + minFrame = self.getMinStart(self.layers) + maxFrame = self.getMaxEnd(self.layers) + + for i in range(minFrame, maxFrame): + pL, indexes = self.getPossibleLayers(i) + if len(pL) <= 1: + continue + merge = set() + innerMax = self.getMaxEnd(pL) + for x in range(self.getMinStart(pL), innerMax): + for lc, l in enumerate(pL): + if l.startFrame < x or l.lastFrame > x: + continue + for lc2, l2 in enumerate(pL): + if lc2 == lc: + continue + for cnt in l.bounds[x-l.startFrame]: + for cnt2 in l2.bounds[x-l2.startFrame]: + if self.contoursOverlay(cnt, cnt2): + merge.add(indexes[lc]) + merge.add(indexes[lc2]) + merge = list(merge) + if len(merge) > 1: + self.mergeLayers(megre) + i = innerMax + + def getPossibleLayers(self, t): + ret = [] + ii = [] for i, layer in enumerate(self.layers): - if i not in foundLayerIDs[1:]: - layers.append(layer) - self.layers = layers - - - def merge2Layers(self, layer1, layer2): - """Merges 2 Layers, Layer1 must start before Layer2""" - - dSF = layer2.startFrame - layer1.startFrame - l1bounds = copy.deepcopy(layer1.bounds) - l1masks = copy.deepcopy(layer1.masks) + if layer.startFrame <= t and layer.lastFrame <= t: + ret.append(layer) + ii.append(i) + return (ret, ii) - for i in range(len(layer2.bounds)): - bounds = layer2.bounds[i] - masks = layer2.masks[i] - while dSF + i >= len(l1bounds): - l1bounds.append([]) - while dSF + i >= len(l1masks): - l1masks.append([]) + def getMinStart(self, layers): + minFrame = layers[0].startFrame + for l in layers: + if l.startFrame < minFrame: + minFrame = l.startFrame + return minFrame - for bound, mask in zip(bounds, masks): - if bound not in l1bounds[dSF + i]: - l1bounds[dSF + i].append(bound) - l1masks[dSF + i].append(mask) + def getMaxEnd(self, layers): + maxFrame = layers[0].lastFrame + for l in layers: + if l.lastFrame < maxFrame: + maxFrame = l.lastFrame - layer1.bounds = l1bounds - layer1.masks = l1masks - return layer1 + return maxFrame - - def contoursOverlay(self, l1, r1, l2, r2): - # If one rectangle is on left side of other - if(l1[0] >= r2[0] or l2[0] >= r1[0]): + def contoursOverlay(self, l1, r1, l2, r2): + # If one rectangle is on left side of other + if(l1[0] >= r2[0] or l2[0] >= r1[0]): return False - # If one rectangle is above other - if(l1[1] <= r2[1] or l2[1] <= r1[1]): + # If one rectangle is above other + if(l1[1] <= r2[1] or l2[1] <= r1[1]): return False return True - def sortLayers(self, foundLayerIDs): + def getLayersByID(self, foundLayerIDs): layers = [] for layerID in foundLayerIDs: layers.append(self.layers[layerID]) - - layers.sort(key = lambda c:c.startFrame) - return layers \ No newline at end of file + + layers.sort(key=lambda c: c.startFrame) + return layers diff --git a/Application/LayerManager.py b/Application/LayerManager.py index 1ba0c29..96e138d 100644 --- a/Application/LayerManager.py +++ b/Application/LayerManager.py @@ -19,16 +19,30 @@ class LayerManager: self.resizeWidth = config["resizeWidth"] self.footagePath = config["inputPath"] self.config = config - self.classifier = Classifier() + #self.classifier = Classifier() self.tags = [] print("LayerManager constructed") def transformLayers(self): print("'Cleaning' Layers") + print("Before deleting short layers ", len(self.layers)) self.freeMin() + print("Before deleting long layers ", len(self.layers)) self.freeMax() - self.sortLayers() + self.sortLayers() self.calcStats() + self.deleteSparse() + print("after deleting sparse layers ", len(self.layers)) + + def deleteSparse(self): + toDelete = [] + for i, l in enumerate(self.layers): + empty = l.bounds.count([]) + if empty / len(l) > 0.2: + toDelete.append(i) + + for i, id in enumerate(toDelete): + del self.layers[id - i] def calcStats(self): for layer in self.layers: @@ -58,7 +72,7 @@ class LayerManager: self.data.clear() layers = [] for l in self.layers: - if l.getLength() > self.minLayerLength: + if len(l) > self.minLayerLength: layers.append(l) self.layers = layers @@ -66,7 +80,7 @@ class LayerManager: def freeMax(self): layers = [] for l in self.layers: - if l.getLength() < self.maxLayerLength: + if len(l) < self.maxLayerLength: layers.append(l) self.layers = layers diff --git a/Application/VideoReader.py b/Application/VideoReader.py index 7781bfb..83424f6 100644 --- a/Application/VideoReader.py +++ b/Application/VideoReader.py @@ -38,11 +38,13 @@ class VideoReader: def pop(self): return self.buffer.get(block=True) - def fillBuffer(self): + def fillBuffer(self, listOfFrames=None): if self.buffer.full(): print("VideoReader::fillBuffer was called when buffer was full.") self.endFrame = int(self.vc.get(cv2.CAP_PROP_FRAME_COUNT)) - + if listOfFrames is not None: + self.listOfFrames = listOfFrames + if self.listOfFrames is not None: self.thread = threading.Thread(target=self.readFramesByList, args=()) else: diff --git a/main.py b/main.py index 1ecfa17..1d4eb6c 100644 --- a/main.py +++ b/main.py @@ -18,7 +18,7 @@ def main(): start = startTotal config = Config() - fileName = "X23-1.mp4" + fileName = "3.mp4" outputPath = os.path.join(os.path.dirname(__file__), "output") dirName = os.path.join(os.path.dirname(__file__), "generate test footage") @@ -29,9 +29,6 @@ def main(): config["w"], config["h"] = VideoReader(config).getWH() stats = dict() - stats["File Name"] = config["importPath"] - stats["threads"] = "16, 16" - stats["buffer"] = config["bufferLength"] if not os.path.exists(config["importPath"]): contours, masks = ContourExtractor(config).extractContours() stats["Contour Extractor"] = time.time() - start @@ -54,9 +51,12 @@ def main(): #layerManager.tagLayers() layers = layerManager.layers + print([len(l) for l in sorted(layers, key = lambda c:len(c), reverse=True)[:20]]) + if len(layers) == 0: + exit(1) exporter = Exporter(config) print(f"Exporting {len(contours)} Contours and {len(layers)} Layers") - exporter.export(layers, contours, masks, raw=True, overlayed=True) + exporter.export(layers, contours, masks, raw=True, overlayed=False) stats["Exporter"] = time.time() - start print("Total time: ", time.time() - startTotal)