diff --git a/Application/ContourExctractor.py b/Application/ContourExctractor.py index 5e57eb5..3e0aa69 100644 --- a/Application/ContourExctractor.py +++ b/Application/ContourExctractor.py @@ -24,12 +24,17 @@ class ContourExtractor: #extracedContours = {frame_number: [(contour, (x,y,w,h)), ...], } # dict with frame numbers as keys and the contour bounds of every contour for that frame - def getextractedContours(self): + def getExtractedContours(self): return self.extractedContours + def getExtractedMasks(self): + return self.extractedMasks + + def __init__(self, config): self.frameBuffer = Queue(16) self.extractedContours = dict() + self.extractedMasks = dict() self.min_area = config["min_area"] self.max_area = config["max_area"] self.threashold = config["threashold"] @@ -64,7 +69,7 @@ class ContourExtractor: frameCount = tmpData[-1][0] videoReader.thread.join() - return self.extractedContours + return self.extractedContours, self.extractedMasks def getContours(self, data): frameCount, frame = data @@ -89,17 +94,21 @@ class ContourExtractor: cnts = imutils.grab_contours(cnts) contours = [] + masks = [] for c in cnts: ca = cv2.contourArea(c) + (x, y, w, h) = cv2.boundingRect(c) + #ca = (x+w)*(y+h) if ca < self.min_area or ca > self.max_area: continue - (x, y, w, h) = cv2.boundingRect(c) contours.append((x, y, w, h)) - + masks.append(np.packbits(np.copy(thresh[y:y+h,x:x+w]), axis=0)) + if len(contours) != 0 and contours is not None: # this should be thread safe self.extractedContours[frameCount] = contours + self.extractedMasks[frameCount] = masks def prepareFrame(self, frame): frame = imutils.resize(frame, width=self.resizeWidth) @@ -141,3 +150,4 @@ class ContourExtractor: for jj in range(0,averageFrames-1): avg += self.prepareFrame(frames[j-jj][1])/averageFrames self.averages[frameNumber] = np.array(np.round(avg), dtype=np.uint8) + #self.averages[frameNumber] = self.prepareFrame(frames[j-averageFrames - 1][1]) diff --git a/Application/Exporter.py b/Application/Exporter.py index ee83032..df2ae7a 100644 --- a/Application/Exporter.py +++ b/Application/Exporter.py @@ -104,7 +104,10 @@ class Exporter: frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) for layer in layers: if layer.startFrame <= frameCount and layer.startFrame + len(layer.bounds) > frameCount: - for (x, y, w, h) in layer.bounds[frameCount - layer.startFrame]: + for i in range(0, len(layer.bounds[frameCount - layer.startFrame])): + (x, y, w, h) = layer.bounds[frameCount - layer.startFrame][i] + mask = layer.masks[frameCount - layer.startFrame][i] + if x is None: break factor = videoReader.w / self.resizeWidth @@ -112,11 +115,13 @@ class Exporter: y = int(y * factor) w = int(w * factor) h = int(h * factor) + mask = imutils.resize(mask, width=w, height=h+1)*255 + mask = np.resize(mask, (h,w)) # if exportFrame as index instead of frameCount - layer.startFrame then we have layer after layer frame2 = frames[frameCount - layer.startFrame] - frame2[y:y+h, x:x+w] = frame2[y:y+h, x:x+w]/2 + frame[y:y+h, x:x+w]/2 + frame2[y:y+h, x:x+w] = cv2.bitwise_and(frame2[y:y+h, x:x+w],frame2[y:y+h, x:x+w], mask)# + frame2[y:y+h, x:x+w]/2 - frames[frameCount - layer.startFrame] = np.copy(frame2) + frames[frameCount - layer.startFrame] = np.copy(frame2) cv2.putText(frames[frameCount - layer.startFrame], str(int(frameCount/self.fps)), (int(x+w/2), int(y+h/2)), cv2.FONT_HERSHEY_SIMPLEX, 1,(255,255,255), 2) videoReader.thread.join() @@ -132,7 +137,7 @@ class Exporter: def exportRawData(self, layers, contours): with open(self.config["importPath"], "wb+") as file: - pickle.dump((layers, contours), file) + pickle.dump((layers, contours, masks), file) diff --git a/Application/Importer.py b/Application/Importer.py index bc2fe11..eed9301 100644 --- a/Application/Importer.py +++ b/Application/Importer.py @@ -7,5 +7,5 @@ class Importer: def importRawData(self): print("Loading previous results") with open(self.path, "rb") as file: - layers, contours = pickle.load(file) - return (layers, contours) \ No newline at end of file + layers, contours, masks = pickle.load(file) + return (layers, contours, masks) \ No newline at end of file diff --git a/Application/Layer.py b/Application/Layer.py index c2c3882..ff16ad0 100644 --- a/Application/Layer.py +++ b/Application/Layer.py @@ -16,7 +16,7 @@ class Layer: lastFrame = None length = None - def __init__(self, startFrame, data, config): + def __init__(self, startFrame, data, mask, config): '''returns a Layer object Layers are collections of contours with a StartFrame, @@ -32,21 +32,26 @@ class Layer: self.config = config self.data = [] self.bounds = [] + self.masks = [] self.stats = dict() self.bounds.append([data]) + self.masks.append([mask]) #print("Layer constructed") - def add(self, frameNumber, bound): + def add(self, frameNumber, bound, mask): '''Adds a bound to the Layer at the layer index which corresponds to the given framenumber''' if self.startFrame + len(self.bounds) - 1 > frameNumber: if len(self.bounds[frameNumber - self.startFrame]) >= 1: self.bounds[frameNumber - self.startFrame].append(bound) + self.masks[frameNumber - self.startFrame].append(mask) else: self.lastFrame = frameNumber self.bounds.append([bound]) + self.masks.append([mask]) def calcStats(self): + '''calculates average distance, variation and deviation of layer movement''' middles = [] for i, bounds in enumerate(self.bounds): for j, bound in enumerate(bounds): @@ -78,93 +83,4 @@ class Layer: def __len__(self): self.length = len(self.bounds) return self.length - - def clusterDelete(self): - '''Uses a cluster analysis to remove contours which are not the result of movement''' - org = self.bounds - if len(org) == 1: - return - mapped = [] - mapping = [] - clusterCount = 1 - noiseThreashold = self.config["noiseThreashold"] - - # calculates the middle of each contour in the 2d bounds[] and saves it in 1d list - # and saves the 2d indexes in a mapping array - for i, bounds in enumerate(org): - for j, bound in enumerate(bounds): - x = (bound[0] + bound[2]/2) / self.config["w"] - y = (bound[1] + bound[3]/2) / self.config["w"] - - mapped.append(list((x,y))) - mapping.append([i,j]) - - mapped = np.array(mapped).astype(np.float16) - labels = [] - centers = [] - kmeans = None - - # the loop isn't nessecary (?) if the number of clusters is known, since it isn't the loop tries to optimize - - kmeans = KMeans(init="random", n_clusters=clusterCount, n_init=10, max_iter=300, random_state=42) - kmeans.fit(mapped) - labels = list(kmeans.labels_) - - maxm = 0 - for x in set(labels): - y = labels.count(x) - if y > maxm: - maxm = y - - - # transformes the labels array - # new array: - # the index is the cluster id, the array is the id of the contour - # [ - # [1,2,3] - # [3,4,5] - # [6,7,8,9] - # ] - classed = [[]] - for i, x in enumerate(list(labels)): - while len(classed) <= x: - classed.append([]) - classed[x].append(i) - - # calculates the euclidean distance (without the sqrt) of each point in a cluster to the cluster center - dists = [] - for num, cen in enumerate(centers): - dist = 0 - for i in classed[num]: - dist += (mapped[i][0]-cen[0])**2 + (mapped[i][1]-cen[1])**2 - dist/=len(classed[num]) - dists.append(dist*1000) - - # copy all contours of the clusters with more movement than the threshold - newContours = [[]] - for i, dis in enumerate(dists): - # copy contours which are spread out, delete rest by not copying them - - for j in classed[i]: - x, y = mapping[j] - while x >= len(newContours): - newContours.append([]) - while y > len(newContours[x]): - newContours[x].append((None, None, None, None)) - if dis > noiseThreashold: - newContours[x].append(org[x][y]) - - self.bounds = newContours - #print(f"{clusterCount} clusters identified {dists}") - - #fig, ax = plt.subplots() - #x=mapped[:,0] - #y=mapped[:,1] - - #ax.scatter(x, y, labels, s=10, cmap="rainbow") - #ax.grid(True) - #plt.show() - - #print("done") - diff --git a/Application/LayerFactory.py b/Application/LayerFactory.py index d02161c..8e62c89 100644 --- a/Application/LayerFactory.py +++ b/Application/LayerFactory.py @@ -23,29 +23,28 @@ class LayerFactory: if data is not None: self.extractLayers(data) - def extractLayers(self, data = None): + def extractLayers(self, data, maskArr): '''Bundle given contours together into Layer Objects''' - if self.data is None: - if data is None or len(data) == 0 : - print("LayerFactory data was none") - return None - else: - self.data = data frameNumber = min(data) contours = data[frameNumber] - for contour in contours: - self.layers.append(Layer(frameNumber, contour, self.config)) + masks = maskArr[frameNumber] + + for contour, mask in zip(contours, masks): + mask = np.unpackbits(mask, axis=0) + self.layers.append(Layer(frameNumber, contour, mask, self.config)) self.oldLayerIDs = [] with ThreadPool(16) as pool: for frameNumber in sorted(data.keys()): contours = data[frameNumber] + masks = maskArr[frameNumber] + masks = [np.unpackbits(mask, axis=0) for mask, contours in zip(masks, contours)] if frameNumber%5000 == 0: print(f" {int(round(frameNumber/max(data.keys()), 2)*100)}% done with Layer extraction", end='\r') - tmp = [[frameNumber, contour] for contour in contours] + tmp = [[frameNumber, contour, mask] for contour, mask in zip(contours, masks)] #pool.map(self.getLayers, tmp) for x in tmp: self.getLayers(x) @@ -55,6 +54,7 @@ class LayerFactory: def getLayers(self, data): frameNumber = data[0] bounds = data[1] + mask = data[2] (x,y,w,h) = bounds tol = self.tolerance foundLayer = 0 @@ -81,13 +81,13 @@ class LayerFactory: break (x2,y2,w2,h2) = bounds if self.contoursOverlay((x-tol,y+h+tol), (x+w+tol,y-tol), (x2,y2+h2), (x2+w2,y2)): - self.layers[i].add(frameNumber, (x,y,w,h)) + self.layers[i].add(frameNumber, (x,y,w,h), mask) foundLayer += 1 foundLayerIDs.append(i) break if foundLayer == 0: - self.layers.append(Layer(frameNumber, (x,y,w,h), self.config)) + self.layers.append(Layer(frameNumber, (x,y,w,h), mask, self.config)) if len(foundLayerIDs) > 1: self.mergeLayers(foundLayerIDs) @@ -114,16 +114,23 @@ class LayerFactory: dSF = layer2.startFrame - layer1.startFrame l1bounds = copy.deepcopy(layer1.bounds) + l1masks = copy.deepcopy(layer1.masks) for i in range(len(layer2.bounds)): bounds = layer2.bounds[i] + masks = layer2.masks[i] while dSF + i >= len(l1bounds): l1bounds.append([]) - for bound in bounds: + while dSF + i >= len(l1masks): + l1masks.append([]) + + for bound, mask in zip(bounds, masks): if bound not in l1bounds[dSF + i]: l1bounds[dSF + i].append(bound) + l1masks[dSF + i].append(mask) layer1.bounds = l1bounds + layer1.masks = l1masks return layer1 diff --git a/Application/LayerManager.py b/Application/LayerManager.py index f3e440f..17c119e 100644 --- a/Application/LayerManager.py +++ b/Application/LayerManager.py @@ -27,9 +27,8 @@ class LayerManager: def transformLayers(self): print("'Cleaning' Layers") self.freeMin() - self.sortLayers() - #self.cleanLayers2() self.freeMax() + self.sortLayers() self.calcStats() def calcStats(self): @@ -110,10 +109,3 @@ class LayerManager: def sortLayers(self): self.layers.sort(key = lambda c:c.startFrame) - - def cleanLayers2(self): - #with ThreadPool(16) as pool: - # pool.map(self.getContours, tmpData) - - for layer in self.layers: - layer.clusterDelete() diff --git a/main.py b/main.py index 61fa214..150ae8a 100644 --- a/main.py +++ b/main.py @@ -14,7 +14,7 @@ def main(): start = time.time() config = Config() - fileName = "x23.mp4" + fileName = "3.mp4" outputPath = os.path.join(os.path.dirname(__file__), "output") dirName = os.path.join(os.path.dirname(__file__), "generate test footage") @@ -26,14 +26,14 @@ def main(): config["w"], config["h"] = VideoReader(config).getWH() if not os.path.exists(config["importPath"]): - contours = ContourExtractor(config).extractContours() + contours, masks = ContourExtractor(config).extractContours() print("Time consumed extracting: ", time.time() - start) layerFactory = LayerFactory(config) - layers = layerFactory.extractLayers(contours) + layers = layerFactory.extractLayers(contours, masks) else: - layers, contours = Importer(config).importRawData() + layers, contours, masks = Importer(config).importRawData() layerFactory = LayerFactory(config) - layers = layerFactory.extractLayers(contours) + layers = layerFactory.extractLayers(contours, masks) layerManager = LayerManager(config, layers) layerManager.transformLayers() diff --git a/ueberblick.drawio.png b/ueberblick.drawio.png index 60ace41..74f565a 100644 Binary files a/ueberblick.drawio.png and b/ueberblick.drawio.png differ