From edeb73cca3f7137c4567f83906e5d47a2ce5dfbf Mon Sep 17 00:00:00 2001 From: Ethan Wellenreiter Date: Tue, 14 Nov 2023 21:48:39 -0500 Subject: [PATCH 1/8] Fixing whiteout background again. Changed the technique to find the edges because it proved inconsistent. Signed-off-by: Ethan Wellenreiter --- code/autocropper/houghlinedevspace.ipynb | 209 +++++++++++++++++------ code/autocropper/myfunctions.py | 73 ++++---- 2 files changed, 194 insertions(+), 88 deletions(-) diff --git a/code/autocropper/houghlinedevspace.ipynb b/code/autocropper/houghlinedevspace.ipynb index eb860d9..26ee11a 100644 --- a/code/autocropper/houghlinedevspace.ipynb +++ b/code/autocropper/houghlinedevspace.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2312, + "execution_count": 143, "metadata": {}, "outputs": [], "source": [ @@ -15,17 +15,78 @@ }, { "cell_type": "code", - "execution_count": 2313, + "execution_count": 144, "metadata": {}, "outputs": [], "source": [ - "# img = cv2.imread('/mnt/dataset/baseimages/1.jpg')\n", - "img = cv2.imread('/mnt/code/autocropper/test_images/IMG_7594.jpg')" + "import os\n", + "import pathlib\n", + "\n", + "def testondataset(pathtodataset, function):\n", + " imagefileextensions = [\".jpg\", \".png\"]\n", + " filenames = next(os.walk(pathtodataset), (None, None, []))[2]\n", + " \n", + " outs = []\n", + " for filename in filenames:\n", + " suffix = pathlib.Path(filename).suffix\n", + " if (suffix not in imagefileextensions):\n", + " print(\"Not a valid image \"+filename)\n", + " continue\n", + " img = cv2.imread(pathtodataset+filename)\n", + " outs.append(function(img))\n", + " return outs\n", + " " ] }, { "cell_type": "code", - "execution_count": 2314, + "execution_count": 145, + "metadata": {}, + "outputs": [], + "source": [ + "def showimgs(imgs):\n", + " if (isinstance(imgs, np.ndarray)):\n", + " if (imgs.shape[0] > imgs.shape[1]):\n", + " cv2.imshow(\"test\", mf.ResizeWithAspectRatio(imgs, height=1350))\n", + " else:\n", + " cv2.imshow(\"test\", mf.ResizeWithAspectRatio(imgs, width=1000))\n", + " else:\n", + " for i, out in enumerate(imgs):\n", + " if (out.shape[0] > out.shape[1]):\n", + " cv2.imshow(\"test\"+str(i), mf.ResizeWithAspectRatio(out, height=1350))\n", + " else:\n", + " cv2.imshow(\"test\"+str(i), mf.ResizeWithAspectRatio(out, width=1000))\n", + " cv2.waitKey(0)\n", + " cv2.destroyAllWindows()" + ] + }, + { + "cell_type": "code", + "execution_count": 146, + "metadata": {}, + "outputs": [], + "source": [ + "def writeimgs(directorypath, imgs):\n", + " if (isinstance(imgs, np.ndarray)):\n", + " cv2.imwrite(directorypath+\"test.jpg\", imgs)\n", + " else:\n", + " for i, out in enumerate(imgs):\n", + " cv2.imwrite(directorypath+\"test\"+str(i)+\".jpg\", out)" + ] + }, + { + "cell_type": "code", + "execution_count": 147, + "metadata": {}, + "outputs": [], + "source": [ + "img = cv2.imread('/mnt/dataset/baseimages/3.jpg')\n", + "# img = cv2.imread('/mnt/code/autocropper/test_images/IMG_7594.jpg')" + ] + }, + { + "cell_type": "code", + "execution_count": 148, "metadata": {}, "outputs": [], "source": [ @@ -36,35 +97,49 @@ " # blur = cv2.blur(gray, (7,7))\n", " \n", " # window = 51\n", - " window = min(gray.shape[1], gray.shape[0])//20\n", + " window = gray.shape[1]//8\n", " if window % 2 == 0:\n", " window += 1\n", - " thresh1 = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, window, 5)\n", - " thresh2 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU)[1]\n", - " thresh = cv2.bitwise_and(thresh1, thresh2)\n", + " thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 2)\n", + " # thresh2 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU)[1]\n", + " # thresh = cv2.bitwise_and(thresh1, thresh2)\n", " # return thresh\n", "\n", " # dim = int(min(thresh.shape[0], thresh.shape[1])/400)\n", - " dim = 3\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " morphedthresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel)\n", + " # dim = 3\n", + " # kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", + " # morphedthresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel)\n", " # return morphedthresh\n", " \n", " \n", " \n", - " # contours1, heirarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " contours2, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " contours1, heirarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " # contours2, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " \n", + "\n", + "\n", + " biggestcontour1 = max(contours1, key=cv2.contourArea)\n", + " # biggestcontour2 = max(contours2, key=cv2.contourArea)\n", + " \n", + " epsilon = 0.0005*cv2.arcLength(biggestcontour1,True)\n", + " approx = cv2.approxPolyDP(biggestcontour1,epsilon,True)\n", + " # approx = cv2.convexHull(approx)\n", + " epsilon = 0.001*cv2.arcLength(approx,True)\n", + " approx = cv2.approxPolyDP(approx,epsilon,True)\n", + " # approx = cv2.convexHull(biggestcontour1)\n", + " # print(approx)\n", " \n", - " # biggestcontour1 = max(contours1, key=cv2.contourArea)\n", - " biggestcontour2 = max(contours2, key=cv2.contourArea)\n", " # imagecpy = cv2.drawContours(imagecpy, [biggestcontour1], -1, (0,255,0), thickness=3)\n", " # imagecpy = cv2.drawContours(imagecpy, [biggestcontour2], -1, (0,0,255), thickness=3)\n", + " \n", + " # imagecpy = cv2.drawContours(imagecpy, [approx], -1, (0,255,0), thickness=3)\n", " # return imagecpy\n", " \n", " blank = np.full(thresh.shape, 255, dtype=np.uint8)\n", " mask = blank.copy()\n", - " # mask = cv2.drawContours(mask, [biggestcontour1], -1, (0,0,0), thickness=cv2.FILLED)\n", - " mask = cv2.drawContours(mask, [biggestcontour2], -1, (0,0,0), thickness=cv2.FILLED)\n", + " mask = cv2.drawContours(mask, [biggestcontour1], -1, (0,0,0), thickness=cv2.FILLED)\n", + " # mask = cv2.drawContours(mask, [approx], -1, (0,0,0), thickness=cv2.FILLED)\n", + " # mask = cv2.drawContours(mask, [biggestcontour2], -1, (0,0,0), thickness=cv2.FILLED)\n", "\n", " # return mask\n", "\n", @@ -79,10 +154,33 @@ " mask = 255 - cv2.morphologyEx(invertmask, cv2.MORPH_ERODE, kernel, iterations=1)\n", " # return mask\n", " \n", - " mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)\n", - " whitedbackground = cv2.bitwise_or(image, mask)\n", + " mask1 = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)\n", + " whitedbackground = cv2.bitwise_or(image, mask1)\n", " # return whitedbackground\n", " \n", + " mask2 = blank.copy()\n", + " mask2 = 255-cv2.drawContours(mask2, [approx], -1, (0,0,0), thickness=cv2.FILLED)\n", + " \n", + " dim = int(min(mask2.shape[0], mask2.shape[1])/50)\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", + " morphedmask = 255-cv2.morphologyEx(mask2, cv2.MORPH_OPEN, kernel, iterations=3)\n", + " # return morphedmask\n", + " \n", + " finalmask = cv2.bitwise_or(mask, morphedmask)\n", + " \n", + " \n", + " finalmaskbgr = cv2.cvtColor(finalmask, cv2.COLOR_GRAY2BGR)\n", + " # return finalmaskbgr\n", + "\n", + " whitedbackground = cv2.bitwise_or(whitedbackground, finalmaskbgr)\n", + " # return whitedbackground\n", + " \n", + " test = cv2.inpaint(whitedbackground, finalmask, 3, cv2.INPAINT_TELEA)\n", + " return test\n", + " \n", + " \n", + " \n", + " \n", " gray2 = cv2.cvtColor(whitedbackground, cv2.COLOR_BGR2GRAY)\n", "\n", " canny = cv2.Canny(gray2, 0, 500, None, 3)\n", @@ -96,7 +194,7 @@ " # print(dim)\n", " dim = int(min(maskgray.shape[0], maskgray.shape[1])/200)\n", " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " morphedmask = cv2.morphologyEx(invert, cv2.MORPH_ERODE, kernel)\n", + " morphedmask = cv2.morphologyEx(invert, cv2.MORPH_ERODE, kernel, iterations=2)\n", " dim = int(min(maskgray.shape[0], maskgray.shape[1])/50)\n", " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", " morphedmask = cv2.morphologyEx(morphedmask, cv2.MORPH_OPEN, kernel, iterations=7)\n", @@ -119,7 +217,7 @@ }, { "cell_type": "code", - "execution_count": 2315, + "execution_count": 149, "metadata": {}, "outputs": [], "source": [ @@ -139,12 +237,14 @@ }, { "cell_type": "code", - "execution_count": 2316, + "execution_count": 150, "metadata": {}, "outputs": [], "source": [ "def houghlineprocessing(image):\n", " croppedanddeskewed, _ = mf.houghlinedeskewandcrop(image)\n", + " return croppedanddeskewed\n", + " \n", " \n", " postprocessed = cropclarifying(croppedanddeskewed)\n", " # return postprocessed\n", @@ -168,12 +268,12 @@ }, { "cell_type": "code", - "execution_count": 2317, + "execution_count": 151, "metadata": {}, "outputs": [], "source": [ "# prepped, scaler, hp, vp = mf.squareandthenresize(img, fill=255, width=1000, returnscalerinfo=True)\n", - "outs = mf.houghlineprocessing(img)\n", + "outs = houghlineprocessing(img)\n", "# outs = houghlinedeskewandcrop(img)\n", "# outs = outs[0]\n", "# print(croprect)\n", @@ -182,7 +282,7 @@ }, { "cell_type": "code", - "execution_count": 2318, + "execution_count": 152, "metadata": {}, "outputs": [], "source": [ @@ -196,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": 2319, + "execution_count": 153, "metadata": {}, "outputs": [], "source": [ @@ -208,38 +308,16 @@ }, { "cell_type": "code", - "execution_count": 2320, + "execution_count": 154, "metadata": {}, "outputs": [], "source": [ - "def showimgs(imgs):\n", - " if (isinstance(imgs, np.ndarray)):\n", - " if (imgs.shape[0] > imgs.shape[1]):\n", - " cv2.imshow(\"test\", mf.ResizeWithAspectRatio(imgs, height=1350))\n", - " else:\n", - " cv2.imshow(\"test\", mf.ResizeWithAspectRatio(imgs, width=1000))\n", - " else:\n", - " for i, out in enumerate(imgs):\n", - " if (out.shape[0] > out.shape[1]):\n", - " cv2.imshow(\"test\"+str(i), mf.ResizeWithAspectRatio(out, height=1350))\n", - " else:\n", - " cv2.imshow(\"test\"+str(i), mf.ResizeWithAspectRatio(out, width=1000))\n", - " cv2.waitKey(0)\n", - " cv2.destroyAllWindows()" + "# showimgs(outs)" ] }, { "cell_type": "code", - "execution_count": 2321, - "metadata": {}, - "outputs": [], - "source": [ - "showimgs(outs)" - ] - }, - { - "cell_type": "code", - "execution_count": 2322, + "execution_count": 155, "metadata": {}, "outputs": [], "source": [ @@ -266,6 +344,33 @@ "# cv2.waitKey(0)\n", "# cv2.destroyAllWindows()" ] + }, + { + "cell_type": "code", + "execution_count": 156, + "metadata": {}, + "outputs": [], + "source": [ + "results = testondataset(\"/mnt/dataset/baseimages/\", houghlineprocessing)" + ] + }, + { + "cell_type": "code", + "execution_count": 157, + "metadata": {}, + "outputs": [], + "source": [ + "# showimgs(results)" + ] + }, + { + "cell_type": "code", + "execution_count": 158, + "metadata": {}, + "outputs": [], + "source": [ + "writeimgs(\"/mnt/code/autocropper/result_images/\", results)" + ] } ], "metadata": { diff --git a/code/autocropper/myfunctions.py b/code/autocropper/myfunctions.py index f937da8..05213d5 100644 --- a/code/autocropper/myfunctions.py +++ b/code/autocropper/myfunctions.py @@ -504,35 +504,49 @@ def whiteoutbackground(image): # blur = cv2.blur(gray, (7,7)) # window = 51 - window = min(gray.shape[1], gray.shape[0])//20 + window = gray.shape[1]//8 if window % 2 == 0: window += 1 - thresh1 = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, window, 5) - thresh2 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU)[1] - thresh = cv2.bitwise_and(thresh1, thresh2) + thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 2) + # thresh2 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU)[1] + # thresh = cv2.bitwise_and(thresh1, thresh2) # return thresh # dim = int(min(thresh.shape[0], thresh.shape[1])/400) - dim = 3 - kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim)) - morphedthresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel) + # dim = 3 + # kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim)) + # morphedthresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel) # return morphedthresh - # contours1, heirarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) - contours2, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + contours1, heirarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + # contours2, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) + + + + biggestcontour1 = max(contours1, key=cv2.contourArea) + # biggestcontour2 = max(contours2, key=cv2.contourArea) + + epsilon = 0.0005*cv2.arcLength(biggestcontour1,True) + approx = cv2.approxPolyDP(biggestcontour1,epsilon,True) + # approx = cv2.convexHull(approx) + epsilon = 0.001*cv2.arcLength(approx,True) + approx = cv2.approxPolyDP(approx,epsilon,True) + # approx = cv2.convexHull(biggestcontour1) + # print(approx) - # biggestcontour1 = max(contours1, key=cv2.contourArea) - biggestcontour2 = max(contours2, key=cv2.contourArea) # imagecpy = cv2.drawContours(imagecpy, [biggestcontour1], -1, (0,255,0), thickness=3) # imagecpy = cv2.drawContours(imagecpy, [biggestcontour2], -1, (0,0,255), thickness=3) + + # imagecpy = cv2.drawContours(imagecpy, [approx], -1, (0,255,0), thickness=3) # return imagecpy blank = np.full(thresh.shape, 255, dtype=np.uint8) mask = blank.copy() - # mask = cv2.drawContours(mask, [biggestcontour1], -1, (0,0,0), thickness=cv2.FILLED) - mask = cv2.drawContours(mask, [biggestcontour2], -1, (0,0,0), thickness=cv2.FILLED) + mask = cv2.drawContours(mask, [biggestcontour1], -1, (0,0,0), thickness=cv2.FILLED) + # mask = cv2.drawContours(mask, [approx], -1, (0,0,0), thickness=cv2.FILLED) + # mask = cv2.drawContours(mask, [biggestcontour2], -1, (0,0,0), thickness=cv2.FILLED) # return mask @@ -547,34 +561,21 @@ def whiteoutbackground(image): mask = 255 - cv2.morphologyEx(invertmask, cv2.MORPH_ERODE, kernel, iterations=1) # return mask - mask = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) - whitedbackground = cv2.bitwise_or(image, mask) + mask1 = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR) + whitedbackground = cv2.bitwise_or(image, mask1) # return whitedbackground - gray2 = cv2.cvtColor(whitedbackground, cv2.COLOR_BGR2GRAY) - - canny = cv2.Canny(gray2, 0, 500, None, 3) + mask2 = blank.copy() + mask2 = 255-cv2.drawContours(mask2, [approx], -1, (0,0,0), thickness=cv2.FILLED) - maskgray = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY) - - - - invert = 255-maskgray - # dim = 21 - # print(dim) - dim = int(min(maskgray.shape[0], maskgray.shape[1])/200) + dim = int(min(mask2.shape[0], mask2.shape[1])/50) kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim)) - morphedmask = cv2.morphologyEx(invert, cv2.MORPH_ERODE, kernel) - dim = int(min(maskgray.shape[0], maskgray.shape[1])/50) - kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim)) - morphedmask = cv2.morphologyEx(morphedmask, cv2.MORPH_OPEN, kernel, iterations=7) - # return 255 - morphedmask - morphedmask = 255 - morphedmask - - finalmask = cv2.bitwise_or(morphedmask, maskgray) - # return finalmask + morphedmask = 255-cv2.morphologyEx(mask2, cv2.MORPH_OPEN, kernel, iterations=3) + # return morphedmask + + finalmask = cv2.bitwise_or(mask, morphedmask) + - # edgecontours, _ = cv2.findContours(255-morphedmask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) finalmaskbgr = cv2.cvtColor(finalmask, cv2.COLOR_GRAY2BGR) # return finalmaskbgr From fc2de6ed4f4fa21bb2697dc09e876ab4a96fe2b4 Mon Sep 17 00:00:00 2001 From: Ethan Wellenreiter Date: Wed, 15 Nov 2023 17:23:22 -0500 Subject: [PATCH 2/8] Updating cropping checkpoint Signed-off-by: Ethan Wellenreiter --- code/autocropper/houghlinedevspace.ipynb | 219 +++++++++++++++-------- code/autocropper/myfunctions.py | 118 ++++++++---- 2 files changed, 228 insertions(+), 109 deletions(-) diff --git a/code/autocropper/houghlinedevspace.ipynb b/code/autocropper/houghlinedevspace.ipynb index 26ee11a..5b34d29 100644 --- a/code/autocropper/houghlinedevspace.ipynb +++ b/code/autocropper/houghlinedevspace.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 143, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -15,17 +15,26 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ "import os\n", "import pathlib\n", "\n", + "def removeextensionandnumeric(filename):\n", + " suffix = pathlib.Path(filename).suffix\n", + " num = filename[:-len(suffix)]\n", + " numint = int(num)\n", + " return numint\n", + " \n", + "\n", "def testondataset(pathtodataset, function):\n", " imagefileextensions = [\".jpg\", \".png\"]\n", " filenames = next(os.walk(pathtodataset), (None, None, []))[2]\n", " \n", + " filenames.sort(key=removeextensionandnumeric)\n", + " # print(filenames)\n", " outs = []\n", " for filename in filenames:\n", " suffix = pathlib.Path(filename).suffix\n", @@ -40,7 +49,7 @@ }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -62,31 +71,31 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ "def writeimgs(directorypath, imgs):\n", " if (isinstance(imgs, np.ndarray)):\n", - " cv2.imwrite(directorypath+\"test.jpg\", imgs)\n", + " cv2.imwrite(directorypath+\"test.png\", imgs)\n", " else:\n", " for i, out in enumerate(imgs):\n", - " cv2.imwrite(directorypath+\"test\"+str(i)+\".jpg\", out)" + " cv2.imwrite(directorypath+\"test\"+str(i)+\".png\", out)" ] }, { "cell_type": "code", - "execution_count": 147, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ - "img = cv2.imread('/mnt/dataset/baseimages/3.jpg')\n", - "# img = cv2.imread('/mnt/code/autocropper/test_images/IMG_7594.jpg')" + "img = cv2.imread('/mnt/dataset/baseimages/11.jpg')\n", + "# img = cv2.imread('/mnt/code/autocropper/test_images/IMG_7605.jpg')" ] }, { "cell_type": "code", - "execution_count": 148, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -106,33 +115,33 @@ " # return thresh\n", "\n", " # dim = int(min(thresh.shape[0], thresh.shape[1])/400)\n", - " # dim = 3\n", - " # kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " # morphedthresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel)\n", + " dim = 2\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", + " morphedthresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel)\n", " # return morphedthresh\n", " \n", " \n", " \n", " contours1, heirarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " # contours2, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " contours2, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", " \n", "\n", "\n", - " biggestcontour1 = max(contours1, key=cv2.contourArea)\n", - " # biggestcontour2 = max(contours2, key=cv2.contourArea)\n", + " biggestcontour2 = max(contours1, key=cv2.contourArea)\n", + " biggestcontour1 = max(contours2, key=cv2.contourArea)\n", " \n", " epsilon = 0.0005*cv2.arcLength(biggestcontour1,True)\n", " approx = cv2.approxPolyDP(biggestcontour1,epsilon,True)\n", " # approx = cv2.convexHull(approx)\n", - " epsilon = 0.001*cv2.arcLength(approx,True)\n", + " epsilon = 0.005*cv2.arcLength(approx,True)\n", " approx = cv2.approxPolyDP(approx,epsilon,True)\n", " # approx = cv2.convexHull(biggestcontour1)\n", " # print(approx)\n", " \n", - " # imagecpy = cv2.drawContours(imagecpy, [biggestcontour1], -1, (0,255,0), thickness=3)\n", - " # imagecpy = cv2.drawContours(imagecpy, [biggestcontour2], -1, (0,0,255), thickness=3)\n", + " imagecpy = cv2.drawContours(imagecpy, [biggestcontour1], -1, (0,255,0), thickness=3)\n", + " imagecpy = cv2.drawContours(imagecpy, [biggestcontour2], -1, (0,0,255), thickness=3)\n", " \n", - " # imagecpy = cv2.drawContours(imagecpy, [approx], -1, (0,255,0), thickness=3)\n", + " imagecpy = cv2.drawContours(imagecpy, [approx], -1, (255,0,0), thickness=3)\n", " # return imagecpy\n", " \n", " blank = np.full(thresh.shape, 255, dtype=np.uint8)\n", @@ -152,10 +161,19 @@ " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", " # invertmask = cv2.morphologyEx(invertmask, cv2.MORPH_DILATE, kernel)\n", " mask = 255 - cv2.morphologyEx(invertmask, cv2.MORPH_ERODE, kernel, iterations=1)\n", - " # return mask\n", + " return mask\n", " \n", - " mask1 = cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)\n", - " whitedbackground = cv2.bitwise_or(image, mask1)\n", + " maskcontours, heirarchy = cv2.findContours(255-mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " \n", + " mainmaskcontour = max(maskcontours, key=cv2.contourArea)\n", + " mask1 = blank.copy()\n", + " mask1 = cv2.drawContours(mask1, [mainmaskcontour], -1, (0,0,0), thickness=cv2.FILLED)\n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_ERODE, kernel, iterations=1)\n", + " # temp = cv2.drawContours(image, [mainmaskcontour], -1, (0,255,0), thickness=3)\n", + " # return temp\n", + " \n", + " mask1c = cv2.cvtColor(mask1, cv2.COLOR_GRAY2BGR)\n", + " whitedbackground = cv2.bitwise_or(image, mask1c)\n", " # return whitedbackground\n", " \n", " mask2 = blank.copy()\n", @@ -163,10 +181,10 @@ " \n", " dim = int(min(mask2.shape[0], mask2.shape[1])/50)\n", " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " morphedmask = 255-cv2.morphologyEx(mask2, cv2.MORPH_OPEN, kernel, iterations=3)\n", + " morphedmask = 255-cv2.morphologyEx(mask2, cv2.MORPH_OPEN, kernel, iterations=7)\n", " # return morphedmask\n", " \n", - " finalmask = cv2.bitwise_or(mask, morphedmask)\n", + " finalmask = cv2.bitwise_or(mask1, morphedmask)\n", " \n", " \n", " finalmaskbgr = cv2.cvtColor(finalmask, cv2.COLOR_GRAY2BGR)\n", @@ -176,54 +194,18 @@ " # return whitedbackground\n", " \n", " test = cv2.inpaint(whitedbackground, finalmask, 3, cv2.INPAINT_TELEA)\n", - " return test\n", - " \n", - " \n", - " \n", - " \n", - " gray2 = cv2.cvtColor(whitedbackground, cv2.COLOR_BGR2GRAY)\n", - "\n", - " canny = cv2.Canny(gray2, 0, 500, None, 3)\n", - " \n", - " maskgray = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)\n", - " \n", - " \n", - "\n", - " invert = 255-maskgray\n", - " # dim = 21\n", - " # print(dim)\n", - " dim = int(min(maskgray.shape[0], maskgray.shape[1])/200)\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " morphedmask = cv2.morphologyEx(invert, cv2.MORPH_ERODE, kernel, iterations=2)\n", - " dim = int(min(maskgray.shape[0], maskgray.shape[1])/50)\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " morphedmask = cv2.morphologyEx(morphedmask, cv2.MORPH_OPEN, kernel, iterations=7)\n", - " # return 255 - morphedmask\n", - " morphedmask = 255 - morphedmask\n", - "\n", - " finalmask = cv2.bitwise_or(morphedmask, maskgray)\n", - " # return finalmask\n", - " \n", - " # edgecontours, _ = cv2.findContours(255-morphedmask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " finalmaskbgr = cv2.cvtColor(finalmask, cv2.COLOR_GRAY2BGR)\n", - " # return finalmaskbgr\n", - "\n", - " whitedbackground = cv2.bitwise_or(whitedbackground, finalmaskbgr)\n", - " # return whitedbackground\n", - " \n", - " test = cv2.inpaint(whitedbackground, finalmask, 3, cv2.INPAINT_TELEA)\n", " return test" ] }, { "cell_type": "code", - "execution_count": 149, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ "def cropclarifying(image):\n", " whitedbackground = whiteoutbackground(image)\n", - " # return whitedbackground\n", + " return whitedbackground\n", "\n", " textrefined = mf.textClarifying(whitedbackground)\n", " # return textrefined\n", @@ -237,12 +219,102 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "def contourcrop(baseimage):\n", + " shrunkencbi, sizemultiplier = mf.ResizeWithAspectRatio(baseimage, width=1000, retscale=True)\n", + " gray = cv2.cvtColor(shrunkencbi, cv2.COLOR_BGR2GRAY)\n", + " # thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)[1]\n", + " thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_TRIANGLE)[1]\n", + " # window = gray.shape[1]//7\n", + " # if window % 2 == 0:\n", + " # window += 1\n", + " # thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 10)\n", + "\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))\n", + " # thresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel, iterations=2)\n", + " thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)\n", + " # return thresh\n", + " \n", + " contours, heirarchy = cv2.findContours(thresh,cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)\n", + " \n", + " # temp = cv2.drawContours(shrunkencbi, contours, -1, (0,255,0), thickness=3)\n", + " # biggestcontour = max(contours, key=cv2.contourArea)\n", + " # temp = cv2.drawContours(shrunkencbi, [biggestcontour], -1, (0,255,0), thickness=3)\n", + " \n", + " # return temp\n", + " \n", + " mx = (0,0,0,0)\n", + " mx_area = 0\n", + "\n", + " for i, cont in enumerate(contours):\n", + " rect = cv2.boundingRect(cont)\n", + " area = mf.rectArea(rect)\n", + " if (area > mx_area):\n", + " mx = rect\n", + " mx_area = area\n", + "\n", + " \n", + " scaledmx = (int(mx[0]*sizemultiplier), int(mx[1]*sizemultiplier), int(mx[2]*sizemultiplier), int(mx[3]*sizemultiplier))\n", + " finalbaseimage = baseimage[scaledmx[1]:scaledmx[1]+scaledmx[3], scaledmx[0]:scaledmx[0]+scaledmx[2], :]\n", + " return finalbaseimage" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [], + "source": [ + "def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle):\n", + " rotateddst1 = mf.rotatewithexactpadding(preppedimage, rotationangle, fill=0)\n", + " rotatedbaseimage = mf.rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n", + " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", + " # print(sizemultiplier)\n", + " # return rotatedbaseimage, rotationangle\n", + "\n", + " croppedbaseimage = mf.houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)\n", + " # return croppedbaseimage, rotationangle\n", + "\n", + " finalbaseimage = contourcrop(croppedbaseimage)\n", + "\n", + "\n", + " return finalbaseimage, rotationangle" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "def houghlinedeskewandcrop(image):\n", + " canny, croppedogimage = mf.prepimageforhoughline(image) ## scaling and cropping occurs. need to also return the changes done\n", + " # return croppedogimage, canny\n", + " # print(canny.shape)\n", + " # print(croppedogimage.shape)\n", + "\n", + " ## -----------------finding angle to deskew-----------------\n", + " rotationangle = mf.houghlinedeskewangle(canny)\n", + " # print(rotationangle)\n", + "\n", + " # -----------------end of finding angle to deskew-----------------\n", + "\n", + " ## -----------------deskewing and then cropping-----------------\n", + " outimg, angle = houghlinedeskewthencrop(croppedogimage, canny, rotationangle)\n", + " return outimg, angle" + ] + }, + { + "cell_type": "code", + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ "def houghlineprocessing(image):\n", - " croppedanddeskewed, _ = mf.houghlinedeskewandcrop(image)\n", + " croppedanddeskewed, angle = houghlinedeskewandcrop(image)\n", " return croppedanddeskewed\n", " \n", " \n", @@ -268,12 +340,13 @@ }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "# prepped, scaler, hp, vp = mf.squareandthenresize(img, fill=255, width=1000, returnscalerinfo=True)\n", "outs = houghlineprocessing(img)\n", + "# print(img.shape)\n", "# outs = houghlinedeskewandcrop(img)\n", "# outs = outs[0]\n", "# print(croprect)\n", @@ -282,7 +355,7 @@ }, { "cell_type": "code", - "execution_count": 152, + "execution_count": 32, "metadata": {}, "outputs": [], "source": [ @@ -296,7 +369,7 @@ }, { "cell_type": "code", - "execution_count": 153, + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -308,7 +381,7 @@ }, { "cell_type": "code", - "execution_count": 154, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -317,7 +390,7 @@ }, { "cell_type": "code", - "execution_count": 155, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -347,7 +420,7 @@ }, { "cell_type": "code", - "execution_count": 156, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -356,7 +429,7 @@ }, { "cell_type": "code", - "execution_count": 157, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -365,7 +438,7 @@ }, { "cell_type": "code", - "execution_count": 158, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ diff --git a/code/autocropper/myfunctions.py b/code/autocropper/myfunctions.py index 05213d5..9c8ce8d 100644 --- a/code/autocropper/myfunctions.py +++ b/code/autocropper/myfunctions.py @@ -91,6 +91,11 @@ def mergecontours(contours): finalcontour = cv2.convexHull(cont) return finalcontour +def padWithColour(img, hpadding=0, vpadding=0, fill=(0,0,0)): + borderType = cv2.BORDER_CONSTANT + out = cv2.copyMakeBorder(img, vpadding, vpadding, hpadding, hpadding, borderType, None, fill) + return out + # funtion to correct the median-angle to give it to the cv2.warpaffine() function # specifically, when getting the angle from a minAreaRect rectangle @@ -344,7 +349,10 @@ def rotateLine(img, line, angle, returnint=True): def prepimageforhoughline(image): prepped, scaler, hp, vp = squareandthenresize(image, fill=255, width=1000, returnscalerinfo=True) prepped, croprect = premorphCrop(prepped) - prepped = squareandthenresize(prepped, fill=255, width=1000) + if (prepped.shape[1] > prepped.shape[0]): + prepped = ResizeWithAspectRatio(prepped, width=1000) + else: + prepped = ResizeWithAspectRatio(prepped, height=1000) finalcroprect = (int(croprect[0]*scaler - hp), int(croprect[1]*scaler - vp), int(croprect[2]*scaler), int(croprect[3]*scaler)) gray1 = cv2.cvtColor(prepped, cv2.COLOR_BGR2GRAY) @@ -361,7 +369,7 @@ def prepimageforhoughline(image): dst1 = cv2.Canny(dst1, 0, 500, None, 3) # return dst1 accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :] - accompaniedimage = squarepad(accompaniedimage, fill=255) + # accompaniedimage = squarepad(accompaniedimage, fill=255) return dst1, accompaniedimage def houghlinedeskewangle(image): @@ -391,56 +399,94 @@ def houghlinedeskewangle(image): rotationangle = np.rad2deg(mode) return rotationangle -def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle): - rotateddst1 = rotate(preppedimage, rotationangle) - rotatedbaseimage = rotate(baseimage, rotationangle) - sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0] - # print(sizemultiplier) - - - linesP = cv2.HoughLinesP(preppedimage, 1, np.pi / 180, 30, None, 90, 30) - - - rotatedlines = [rotateLine(rotateddst1, line[0], rotationangle) for line in linesP] - rotatedlines = np.reshape(rotatedlines, (len(rotatedlines),1,4)) +def determineextrapadding(img, angle): + h, w = img.shape[0], img.shape[1] + radangle = abs(np.deg2rad(angle)) + # print(radangle) + totalheightrot = w*np.sin(radangle) + h*np.cos(radangle) + # print(h, totalheightrot) + totalwidthrot = h*np.sin(radangle) + w*np.cos(radangle) + # print(w, totalwidthrot) + vpad = int(max(0,math.ceil((totalheightrot - h)/2))) + hpad = int(max(0,math.ceil((totalwidthrot-w)/2))) + # print(vpad, hpad) + return hpad, vpad +def rotatewithexactpadding(img, angle, fill=(0,0,0)): + hpad, vpad = determineextrapadding(img, angle) + baseimage = padWithColour(img, hpad, vpad, fill=fill) + rotatedimg = rotate(baseimage, angle) + return rotatedimg +def houghlinepcrop(baseimage, preppedimage, scalingmultiplier): + rotatedlines = cv2.HoughLinesP(preppedimage, 1, np.pi / 180, 30, None, 90, 30) + vmarginlines = WithinXDegrees(rotatedlines, 7) hmarginlines = WithinXDegrees(rotatedlines, 7, baseangle=90) - vrect = lineBoundingRect(vmarginlines,asRect=False, returnint=True) - hmarginlines = lineswithinrange(hmarginlines, (vrect[0], vrect[1]), (vrect[2],vrect[3]), x=True, y=False) - - if (hmarginlines != []): - marginlines = np.append(vmarginlines, hmarginlines, axis=0) - else: - marginlines = vmarginlines - - # print(marginlines) - rect = lineBoundingRect(marginlines,asRect=False, returnint=True) - # print(rect) - scaledrect = (int(rect[0]*sizemultiplier), int(rect[1]*sizemultiplier), int(rect[2]*sizemultiplier), int(rect[3]*sizemultiplier)) - - croppedbaseimage = rotatedbaseimage[scaledrect[1]:scaledrect[3], scaledrect[0]:scaledrect[2], :] + # vrect = lineBoundingRect(vmarginlines,asRect=False, returnint=True) + # hmarginlines = lineswithinrange(hmarginlines, (vrect[0], vrect[1]), (vrect[2],vrect[3]), x=True, y=False) + marginlines = np.append(vmarginlines, hmarginlines, axis=0) + # colourdst = cv2.cvtColor(preppedimage, cv2.COLOR_GRAY2BGR) + # if marginlines is not None: + # for l in marginlines: + # cv2.line(colourdst, (int(l[0]), int(l[1])), (int(l[2]), int(l[3])), (0,0,255), 3, cv2.LINE_AA) + # return colourdst + + rect = lineBoundingRect(marginlines,asRect=False, returnint=True) + scaledrect = (int(rect[0]*scalingmultiplier), int(rect[1]*scalingmultiplier), int(rect[2]*scalingmultiplier), int(rect[3]*scalingmultiplier)) + croppedbaseimage = baseimage[scaledrect[1]:scaledrect[3], scaledrect[0]:scaledrect[2], :] + return croppedbaseimage - # print(croppedbaseimage.shape) - shrunkencbi, sizemultiplier = ResizeWithAspectRatio(croppedbaseimage, width=1000, retscale=True) +def contourcrop(baseimage): + shrunkencbi, sizemultiplier = mf.ResizeWithAspectRatio(baseimage, width=1000, retscale=True) gray = cv2.cvtColor(shrunkencbi, cv2.COLOR_BGR2GRAY) - thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)[1] + # thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)[1] + thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_TRIANGLE)[1] + # window = gray.shape[1]//7 + # if window % 2 == 0: + # window += 1 + # thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 10) + + kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5)) + # thresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel, iterations=2) + thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel) + # return thresh + contours, heirarchy = cv2.findContours(thresh,cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE) + + # temp = cv2.drawContours(shrunkencbi, contours, -1, (0,255,0), thickness=3) + # biggestcontour = max(contours, key=cv2.contourArea) + # temp = cv2.drawContours(shrunkencbi, [biggestcontour], -1, (0,255,0), thickness=3) + + # return temp + mx = (0,0,0,0) mx_area = 0 for i, cont in enumerate(contours): rect = cv2.boundingRect(cont) - area = rectArea(rect) + area = mf.rectArea(rect) if (area > mx_area): mx = rect mx_area = area scaledmx = (int(mx[0]*sizemultiplier), int(mx[1]*sizemultiplier), int(mx[2]*sizemultiplier), int(mx[3]*sizemultiplier)) - finalbaseimage = croppedbaseimage[scaledmx[1]:scaledmx[1]+scaledmx[3], scaledmx[0]:scaledmx[0]+scaledmx[2], :] + finalbaseimage = baseimage[scaledmx[1]:scaledmx[1]+scaledmx[3], scaledmx[0]:scaledmx[0]+scaledmx[2], :] + return finalbaseimage + +def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle): + rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=0) + rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0)) + sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0] + # print(sizemultiplier) + # return rotatedbaseimage, rotationangle + + croppedbaseimage = houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier) + + finalbaseimage = contourcrop(croppedbaseimage) + return finalbaseimage, rotationangle @@ -457,8 +503,8 @@ def houghlinedeskewandcrop(image): # -----------------end of finding angle to deskew----------------- ## -----------------deskewing and then cropping----------------- - - return houghlinedeskewthencrop(croppedogimage, canny, rotationangle) + outimg, angle = houghlinedeskewthencrop(croppedogimage, canny, rotationangle) + return outimg, angle def bruteforceprocessrects(greaterrects, lesserrects): # squaredgrects = np.array([mf.xywhrectto2prect(rect) for rect in greaterrects]) @@ -1066,7 +1112,7 @@ def receipttextdeskew(img, fill=(0,0,0)): ## ------------------------------Full deskewing and cropping------------------------------ def houghlineprocessing(image): - croppedanddeskewed, _ = houghlinedeskewandcrop(image) + croppedanddeskewed, angle = houghlinedeskewandcrop(image) postprocessed = cropclarifying(croppedanddeskewed) # return postprocessed From c2ca87eea5aa4b9b1916cdb03ee985d1a315f295 Mon Sep 17 00:00:00 2001 From: Ethan Wellenreiter Date: Fri, 17 Nov 2023 16:23:07 -0500 Subject: [PATCH 3/8] Whiteoutbackground update checkpoint. Using lab a and b channels (although it hasn't worked too well yet. Signed-off-by: Ethan Wellenreiter --- code/autocropper/houghlinedevspace.ipynb | 448 +++++++++++++++++------ code/autocropper/myfunctions.py | 6 +- 2 files changed, 332 insertions(+), 122 deletions(-) diff --git a/code/autocropper/houghlinedevspace.ipynb b/code/autocropper/houghlinedevspace.ipynb index 5b34d29..7d02e20 100644 --- a/code/autocropper/houghlinedevspace.ipynb +++ b/code/autocropper/houghlinedevspace.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 20, + "execution_count": 3033, "metadata": {}, "outputs": [], "source": [ @@ -15,12 +15,13 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 3034, "metadata": {}, "outputs": [], "source": [ "import os\n", "import pathlib\n", + "import time\n", "\n", "def removeextensionandnumeric(filename):\n", " suffix = pathlib.Path(filename).suffix\n", @@ -36,20 +37,25 @@ " filenames.sort(key=removeextensionandnumeric)\n", " # print(filenames)\n", " outs = []\n", + " tdiffs = []\n", " for filename in filenames:\n", " suffix = pathlib.Path(filename).suffix\n", " if (suffix not in imagefileextensions):\n", " print(\"Not a valid image \"+filename)\n", " continue\n", " img = cv2.imread(pathtodataset+filename)\n", + " t1 = time.time()\n", " outs.append(function(img))\n", + " tdiffs.append(time.time() - t1)\n", + " tdiffs = np.array(tdiffs)\n", + " print(\"average time: \" + str(np.mean(tdiffs))+\"(s)\")\n", " return outs\n", " " ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 3035, "metadata": {}, "outputs": [], "source": [ @@ -71,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 3036, "metadata": {}, "outputs": [], "source": [ @@ -85,42 +91,306 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 3037, "metadata": {}, "outputs": [], "source": [ - "img = cv2.imread('/mnt/dataset/baseimages/11.jpg')\n", - "# img = cv2.imread('/mnt/code/autocropper/test_images/IMG_7605.jpg')" + "img = cv2.imread('/mnt/dataset/baseimages/12.jpg')\n", + "# img = cv2.imread('/mnt/code/autocropper/test_images/IMG_7605.jpg')\n", + "testall = False" ] }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 3038, "metadata": {}, "outputs": [], "source": [ "def whiteoutbackground(image):\n", + " sdim = int(min(image.shape[0], image.shape[1])/10)\n", + " skernel = cv2.getStructuringElement(cv2.MORPH_RECT, (sdim, sdim))\n", + " srkernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (sdim, sdim))\n", + " \n", + " \n", + " lab = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)\n", + " \n", + " imglist = []\n", + " imglist.append(image)\n", + " \n", + " # currentimgofatype = lab[:,:,0]\n", + " # imglist.append(currentimgofatype)\n", + " # imglist.append(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY))\n", + " laba = lab[:,:,1]\n", + " # imglist.append(currentimgofatype)\n", + " labb = lab[:,:,2]\n", + " # imglist.append(currentimgofatype)\n", + "\n", + "\n", + " blurimage1 = cv2.blur(laba, (5,5))\n", + " blurimage2 = cv2.blur(labb, (5,5))\n", + " \n", + " window = lab.shape[1]//5\n", + " if window % 2 == 0:\n", + " window += 1\n", + " thresh1 = cv2.adaptiveThreshold(blurimage1, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", + " window = lab.shape[1]//10\n", + " if window % 2 == 0:\n", + " window += 1\n", + " thresh2 = cv2.adaptiveThreshold(blurimage2, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", + " imglist.append(thresh1)\n", + " imglist.append(thresh2)\n", + " threshmerge = cv2.bitwise_or(thresh1, thresh2)\n", + " imglist.append(threshmerge)\n", + " \n", + " dim = int(min(threshmerge.shape[0], threshmerge.shape[1])/100)\n", + " # dim = 2\n", + " # dim = dotsize\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", + " \n", + " padthresh = mf.padWithColour(threshmerge, sdim, sdim, fill=255)\n", + " mask1t = cv2.morphologyEx(padthresh, cv2.MORPH_OPEN, kernel, iterations=1)\n", + " # mask1t = cv2.morphologyEx(mask1t, cv2.MORPH_OPEN, skernel)\n", + " imglist.append(mask1t)\n", + " \n", + " contours, heirarchy = cv2.findContours(255-mask1t, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " biggestcontour = max(contours, key=cv2.contourArea)\n", + " \n", + " blank = np.full(threshmerge.shape, 255, dtype=np.uint8)\n", + " mask1 = blank.copy()\n", + " mask1 = mf.padWithColour(mask1, sdim, sdim, fill=255)\n", + " mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", + " # temp = cv2.drawContours(image, [biggestcontour], -1, 255, thickness=3)\n", + " # imglist.append(temp)\n", + " imglist.append(mask1)\n", + " \n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, srkernel, iterations=2)\n", + " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, srkernel)\n", + " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", + " mask1 = mask1[sdim:-sdim, sdim:-sdim]\n", + " imglist.append(mask1)\n", + " \n", + " maskc = cv2.cvtColor(mask1, cv2.COLOR_GRAY2BGR)\n", + " whitedbackground = cv2.bitwise_or(image, maskc)\n", + " imglist.append(whitedbackground)\n", + " return whitedbackground\n", + " \n", + " \n", + " return imglist\n", + " \n", + " \n", + " \n", + " # contours, heirarchy = cv2.findContours(255-threshmerge, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " # biggestcontour = max(contours, key=cv2.contourArea)\n", + " \n", + " # blank = np.full(threshmerge.shape, 255, dtype=np.uint8)\n", + " # mask1 = blank.copy()\n", + " # mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", + " # # temp = cv2.drawContours(image, [biggestcontour], -1, 255, thickness=3)\n", + " # # imglist.append(temp)\n", + " \n", + " # mask1 = mf.padWithColour(mask1, sdim, sdim, fill=255)\n", + " \n", + " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=1)\n", + " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel)\n", + " # mask1 = mask1[sdim:-sdim, sdim:-sdim]\n", + " # imglist.append(mask1)\n", + " \n", + " # mask1merge = cv2.bitwise_and(mask1, mask1t)\n", + " # imglist.append(mask1merge)\n", + " \n", + " \n", + " \n", + " # maskc = cv2.cvtColor(mask1merge, cv2.COLOR_GRAY2BGR)\n", + " # # return maskc\n", + " # # return blurimage\n", + " # # print(blurimage)\n", + " # # print(maskc)\n", + " # whitedbackground = cv2.bitwise_or(image, maskc)\n", + " # imglist.append(whitedbackground)\n", + "\n", + " # return imglist\n", + " \n", + " \n", " imagecpy = image.copy()\n", - " gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)\n", + " blurimage = cv2.blur(imagecpy, (13,13))\n", + " \n", + "\n", + " \n", + " \n", + " # ogcover = np.full(imagecpy.shape, 0, dtype=np.uint8)\n", + " # ogcover = mf.padWithColour(ogcover, sdim, sdim, fill=(255,255,255))\n", + " # ogcover = cv2.cvtColor(ogcover, cv2.COLOR_BGR2GRAY)\n", + " \n", + " # blurimage = mf.padWithColour(blurimage, sdim, sdim, fill=(255,255,255))\n", + " # imagecpy = cv2.inpaint(blurimage, ogcover, 3, cv2.INPAINT_TELEA)\n", + " # imagecpy = cv2.copyMakeBorder(blurimage, sdim, sdim, sdim, sdim, cv2.BORDER_REPLICATE)\n", + " # return imagecpy\n", + " # imagecpy = mf.padWithColour(image, hpadding=sdim, vpadding=sdim)\n", + " gray = cv2.cvtColor(blurimage, cv2.COLOR_BGR2GRAY)\n", " \n", " # blur = cv2.blur(gray, (7,7))\n", " \n", " # window = 51\n", - " window = gray.shape[1]//8\n", + " window = gray.shape[1]//15\n", " if window % 2 == 0:\n", " window += 1\n", - " thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 2)\n", + " thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 5)\n", " # thresh2 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU)[1]\n", " # thresh = cv2.bitwise_and(thresh1, thresh2)\n", " # return thresh\n", + " \n", + " # initialcontours, heirarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " \n", + " # contourrects = np.array([list(cv2.boundingRect(contour)) for contour in initialcontours])\n", + " # rectdims = contourrects[:,2:]\n", + " # flatrectdims = rectdims.flatten()\n", + " # dotsize = int(np.median(flatrectdims)//2)\n", + " # # print(contourrects)\n", + " # # print(rectdims)\n", + " # # print(dotsize)\n", + " \n", + " \n", "\n", " # dim = int(min(thresh.shape[0], thresh.shape[1])/400)\n", " dim = 2\n", + " # dim = dotsize\n", " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", " morphedthresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel)\n", " # return morphedthresh\n", " \n", " \n", + " contours, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " biggestcontour = max(contours, key=cv2.contourArea)\n", + " \n", + " \n", + " # epsilon = 0.0005*cv2.arcLength(biggestcontour,True)\n", + " # approx = cv2.approxPolyDP(biggestcontour,epsilon,True)\n", + " \n", + " blank = np.full(thresh.shape, 255, dtype=np.uint8)\n", + " mask1 = 255 - blank.copy()\n", + " mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 255, thickness=cv2.FILLED)\n", + " \n", + " mask1 = mf.padWithColour(mask1, sdim, sdim, fill=0)\n", + " \n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel)\n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel)\n", + " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel)\n", + " # return mask1\n", + " \n", + " \n", + " mask1contours, heirarchy = cv2.findContours(mask1, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " \n", + " biggestm1contour = max(mask1contours, key=cv2.contourArea)\n", + " \n", + " mask1_1 = 255-blank.copy()\n", + " mask1_1 = mf.padWithColour(mask1_1, sdim, sdim, fill=0)\n", + " mask1_1 = cv2.drawContours(mask1_1, [biggestm1contour], -1, 255, thickness=cv2.FILLED)\n", + " mask1_1 = 255-mask1_1\n", + " \n", + " dim = max(2,int(min(image.shape[0], image.shape[1])/100))\n", + " # print(dim)\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", + " mask1_1 = cv2.morphologyEx(mask1_1, cv2.MORPH_DILATE, kernel, iterations=1)\n", + " mask1_1 = mask1_1[sdim:-sdim, sdim:-sdim]\n", + " \n", + " # epsilon = 0.01*cv2.arcLength(biggestm1contour,True)\n", + " # approxm1 = cv2.approxPolyDP(biggestm1contour,epsilon,True)\n", + " \n", + " # hull = cv2.convexHull(biggestcontour)\n", + " # hull = cv2.convexHull(approx)\n", + " # mask2 = 255 - blank.copy()\n", + " # mask2 = cv2.drawContours(mask2, [hull], -1, 255, thickness=cv2.FILLED)\n", + " \n", + " # imagecpy = cv2.drawContours(imagecpy, [biggestcontour], -1, (255,0,0), thickness=3)\n", + " # # imagecpy = cv2.drawContours(imagecpy, [approx], -1, (0,255,0), thickness=3)\n", + " # # imagecpy = cv2.drawContours(imagecpy, [hull], -1, (0,255,255), thickness=3)\n", + " # imagecpy = cv2.drawContours(imagecpy, [biggestm1contour], -1, (0,0,255), thickness=3)\n", + " # # imagecpy = cv2.drawContours(imagecpy, [approxm1], -1, (255,0,255), thickness=3)\n", + " # return imagecpy\n", + " \n", + " # mask = 255 - cv2.bitwise_and(mask1, mask2)\n", + " # return mask1_1\n", + " \n", + " maskc = cv2.cvtColor(mask1_1, cv2.COLOR_GRAY2BGR)\n", + " # return maskc\n", + " # return blurimage\n", + " # print(blurimage)\n", + " # print(maskc)\n", + " whitedbackground = cv2.bitwise_or(image, maskc)\n", + " return whitedbackground\n", + "\n", + "\n", + "\n", + "\n", + "##### NEED TO FIX THE CURVE OF CONNECTING TO THE EDGES DUE TO THE ELLIPSE KERNEL\n", + " \n", + " graywhitedbackground = cv2.cvtColor(whitedbackground, cv2.COLOR_BGR2GRAY)\n", + " # thresh2 = cv2.threshold(graywhitedbackground, 0, 255, cv2.THRESH_OTSU)[1]\n", + " blurredgraywhitedbackground = cv2.blur(graywhitedbackground, (11,11))\n", + " # blurredgraywhitedbackground = graywhitedbackground\n", + " window = blurredgraywhitedbackground.shape[1]//15\n", + " if window % 2 == 0:\n", + " window += 1\n", + " thresh2 = cv2.adaptiveThreshold(blurredgraywhitedbackground, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 2)\n", + " # return thresh2\n", + " \n", + " paddedthresh2 = mf.padWithColour(thresh2, sdim, sdim, fill=255)\n", + " \n", + " \n", + " dim = max(2,int(min(image.shape[0], image.shape[1])/50))\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", + " t2morph = cv2.morphologyEx(paddedthresh2, cv2.MORPH_OPEN, kernel, iterations=2)\n", + " dim = int(window*1.5)\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", + " t2morph = cv2.morphologyEx(t2morph, cv2.MORPH_CLOSE, kernel, iterations=1)\n", + " \n", + " t2morph = t2morph[sdim:-sdim, sdim:-sdim]\n", + " # return t2morph\n", + "\n", + " # print(mask1_1.shape)\n", + " # print(t2morph.shape)\n", + " mask2 = 255-cv2.bitwise_or(mask1_1, 255-t2morph)\n", + " # return mask2\n", + " \n", + " \n", + " \n", + " mask2contours, heirarchy = cv2.findContours(mask2, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " \n", + " biggestm2contour = max(mask2contours, key=cv2.contourArea)\n", + " \n", + " mask2_1 = 255 - blank.copy()\n", + " mask2_1 = cv2.drawContours(mask2_1, [biggestm2contour], -1, 255, thickness=cv2.FILLED)\n", + " \n", + " # return mask2_1\n", + " \n", + " mask2_1 = mf.padWithColour(mask2_1, sdim, sdim, fill=0)\n", + " \n", + " mask2_2 = cv2.morphologyEx(mask2_1, cv2.MORPH_OPEN, kernel)\n", + " \n", + " mask2_2 = 255-mask2_2[sdim:-sdim, sdim:-sdim]\n", + " \n", + " # return mask2_2\n", + "\n", + " mask2c = cv2.cvtColor(mask2_2, cv2.COLOR_GRAY2BGR)\n", + " # return maskc\n", + " # return blurimage\n", + " # print(blurimage)\n", + " # print(maskc)\n", + " whitedbackground2 = cv2.bitwise_or(image, mask2c)\n", + " return whitedbackground2\n", + " \n", + " \n", + " \n", + " \n", + " canny = cv2.Canny(graywhitedbackground, 0, 500)\n", + " return canny\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", " \n", " contours1, heirarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", " contours2, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", @@ -199,7 +469,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 3039, "metadata": {}, "outputs": [], "source": [ @@ -219,107 +489,17 @@ }, { "cell_type": "code", - "execution_count": 27, - "metadata": {}, - "outputs": [], - "source": [ - "def contourcrop(baseimage):\n", - " shrunkencbi, sizemultiplier = mf.ResizeWithAspectRatio(baseimage, width=1000, retscale=True)\n", - " gray = cv2.cvtColor(shrunkencbi, cv2.COLOR_BGR2GRAY)\n", - " # thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)[1]\n", - " thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_TRIANGLE)[1]\n", - " # window = gray.shape[1]//7\n", - " # if window % 2 == 0:\n", - " # window += 1\n", - " # thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 10)\n", - "\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5,5))\n", - " # thresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel, iterations=2)\n", - " thresh = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, kernel)\n", - " # return thresh\n", - " \n", - " contours, heirarchy = cv2.findContours(thresh,cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)\n", - " \n", - " # temp = cv2.drawContours(shrunkencbi, contours, -1, (0,255,0), thickness=3)\n", - " # biggestcontour = max(contours, key=cv2.contourArea)\n", - " # temp = cv2.drawContours(shrunkencbi, [biggestcontour], -1, (0,255,0), thickness=3)\n", - " \n", - " # return temp\n", - " \n", - " mx = (0,0,0,0)\n", - " mx_area = 0\n", - "\n", - " for i, cont in enumerate(contours):\n", - " rect = cv2.boundingRect(cont)\n", - " area = mf.rectArea(rect)\n", - " if (area > mx_area):\n", - " mx = rect\n", - " mx_area = area\n", - "\n", - " \n", - " scaledmx = (int(mx[0]*sizemultiplier), int(mx[1]*sizemultiplier), int(mx[2]*sizemultiplier), int(mx[3]*sizemultiplier))\n", - " finalbaseimage = baseimage[scaledmx[1]:scaledmx[1]+scaledmx[3], scaledmx[0]:scaledmx[0]+scaledmx[2], :]\n", - " return finalbaseimage" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [], - "source": [ - "def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle):\n", - " rotateddst1 = mf.rotatewithexactpadding(preppedimage, rotationangle, fill=0)\n", - " rotatedbaseimage = mf.rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n", - " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", - " # print(sizemultiplier)\n", - " # return rotatedbaseimage, rotationangle\n", - "\n", - " croppedbaseimage = mf.houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)\n", - " # return croppedbaseimage, rotationangle\n", - "\n", - " finalbaseimage = contourcrop(croppedbaseimage)\n", - "\n", - "\n", - " return finalbaseimage, rotationangle" - ] - }, - { - "cell_type": "code", - "execution_count": 29, - "metadata": {}, - "outputs": [], - "source": [ - "def houghlinedeskewandcrop(image):\n", - " canny, croppedogimage = mf.prepimageforhoughline(image) ## scaling and cropping occurs. need to also return the changes done\n", - " # return croppedogimage, canny\n", - " # print(canny.shape)\n", - " # print(croppedogimage.shape)\n", - "\n", - " ## -----------------finding angle to deskew-----------------\n", - " rotationangle = mf.houghlinedeskewangle(canny)\n", - " # print(rotationangle)\n", - "\n", - " # -----------------end of finding angle to deskew-----------------\n", - "\n", - " ## -----------------deskewing and then cropping-----------------\n", - " outimg, angle = houghlinedeskewthencrop(croppedogimage, canny, rotationangle)\n", - " return outimg, angle" - ] - }, - { - "cell_type": "code", - "execution_count": 30, + "execution_count": 3040, "metadata": {}, "outputs": [], "source": [ "def houghlineprocessing(image):\n", - " croppedanddeskewed, angle = houghlinedeskewandcrop(image)\n", - " return croppedanddeskewed\n", + " croppedanddeskewed, angle = mf.houghlinedeskewandcrop(image)\n", + " # return croppedanddeskewed\n", " \n", " \n", " postprocessed = cropclarifying(croppedanddeskewed)\n", - " # return postprocessed\n", + " return postprocessed\n", " postprocessed = mf.croptoblack(postprocessed)\n", " \n", " # postprocessed = cv2.cvtColor(postprocessed, cv2.COLOR_GRAY2BGR)\n", @@ -340,7 +520,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 3041, "metadata": {}, "outputs": [], "source": [ @@ -355,7 +535,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 3042, "metadata": {}, "outputs": [], "source": [ @@ -369,7 +549,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 3043, "metadata": {}, "outputs": [], "source": [ @@ -381,16 +561,26 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 3044, "metadata": {}, "outputs": [], "source": [ - "# showimgs(outs)" + "testall = True" ] }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 3045, + "metadata": {}, + "outputs": [], + "source": [ + "if not testall:\n", + " showimgs(outs)" + ] + }, + { + "cell_type": "code", + "execution_count": 3046, "metadata": {}, "outputs": [], "source": [ @@ -420,29 +610,49 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 3047, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "average time: 8.021544203162193(s)\n" + ] + } + ], "source": [ - "results = testondataset(\"/mnt/dataset/baseimages/\", houghlineprocessing)" + "if testall:\n", + " results = testondataset(\"/mnt/dataset/baseimages/\", houghlineprocessing)" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 3048, "metadata": {}, "outputs": [], "source": [ - "# showimgs(results)" + "# if testall:\n", + "# showimgs(results)" ] }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 3049, "metadata": {}, "outputs": [], "source": [ - "writeimgs(\"/mnt/code/autocropper/result_images/\", results)" + "# print(results[0])" + ] + }, + { + "cell_type": "code", + "execution_count": 3050, + "metadata": {}, + "outputs": [], + "source": [ + "if testall:\n", + " writeimgs(\"/mnt/code/autocropper/result_images/\", results)" ] } ], diff --git a/code/autocropper/myfunctions.py b/code/autocropper/myfunctions.py index 9c8ce8d..8dd24af 100644 --- a/code/autocropper/myfunctions.py +++ b/code/autocropper/myfunctions.py @@ -439,7 +439,7 @@ def houghlinepcrop(baseimage, preppedimage, scalingmultiplier): return croppedbaseimage def contourcrop(baseimage): - shrunkencbi, sizemultiplier = mf.ResizeWithAspectRatio(baseimage, width=1000, retscale=True) + shrunkencbi, sizemultiplier = ResizeWithAspectRatio(baseimage, width=1000, retscale=True) gray = cv2.cvtColor(shrunkencbi, cv2.COLOR_BGR2GRAY) # thresh = cv2.threshold(gray, 200, 255, cv2.THRESH_BINARY)[1] thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_TRIANGLE)[1] @@ -466,7 +466,7 @@ def contourcrop(baseimage): for i, cont in enumerate(contours): rect = cv2.boundingRect(cont) - area = mf.rectArea(rect) + area = rectArea(rect) if (area > mx_area): mx = rect mx_area = area @@ -749,7 +749,7 @@ def textClarifying(image): autothreshold = np.clip(np.mean(gray)/1.2, 0, 255) lab = cv2.cvtColor(image, cv2.COLOR_BGR2LAB) - hls = cv2.cvtColor(image, cv2.COLOR_BGR2HLS) + # hls = cv2.cvtColor(image, cv2.COLOR_BGR2HLS) kernel1 = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5)) kernel2 = cv2.getStructuringElement(cv2.MORPH_RECT, (4, 4)) From 6465a49cfe32fae9748a5afeff34928c5109421a Mon Sep 17 00:00:00 2001 From: Ethan Wellenreiter Date: Thu, 23 Nov 2023 17:52:01 -0500 Subject: [PATCH 4/8] Checkpoint in making the rotation not have black fill spots For autocropping/deskewing, need the rotation to not have the black fill spots since they tend to mess up the later background whiteing-out. Signed-off-by: Ethan Wellenreiter --- code/autocropper/houghlinedevspace.ipynb | 205 ++++++++++++++++------- code/autocropper/myfunctions.py | 38 +++-- 2 files changed, 165 insertions(+), 78 deletions(-) diff --git a/code/autocropper/houghlinedevspace.ipynb b/code/autocropper/houghlinedevspace.ipynb index 7d02e20..585ffd6 100644 --- a/code/autocropper/houghlinedevspace.ipynb +++ b/code/autocropper/houghlinedevspace.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 3033, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 3034, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ @@ -55,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 3035, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -77,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 3036, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -91,25 +91,28 @@ }, { "cell_type": "code", - "execution_count": 3037, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ - "img = cv2.imread('/mnt/dataset/baseimages/12.jpg')\n", + "img = cv2.imread('/mnt/dataset/baseimages/15.jpg')\n", "# img = cv2.imread('/mnt/code/autocropper/test_images/IMG_7605.jpg')\n", "testall = False" ] }, { "cell_type": "code", - "execution_count": 3038, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ + "## NEED TO FIX THE EARLIER PARTS SO THAT IT DOESN'T HAVE THOSE BLACK SECTIONS AFTER THE ROTATION\n", + "\n", + "\n", "def whiteoutbackground(image):\n", " sdim = int(min(image.shape[0], image.shape[1])/10)\n", - " skernel = cv2.getStructuringElement(cv2.MORPH_RECT, (sdim, sdim))\n", - " srkernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (sdim, sdim))\n", + " srkernel = cv2.getStructuringElement(cv2.MORPH_RECT, (sdim, sdim))\n", + " skernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (sdim, sdim))\n", " \n", " \n", " lab = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)\n", @@ -129,7 +132,7 @@ " blurimage1 = cv2.blur(laba, (5,5))\n", " blurimage2 = cv2.blur(labb, (5,5))\n", " \n", - " window = lab.shape[1]//5\n", + " window = lab.shape[1]//3\n", " if window % 2 == 0:\n", " window += 1\n", " thresh1 = cv2.adaptiveThreshold(blurimage1, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", @@ -137,10 +140,12 @@ " if window % 2 == 0:\n", " window += 1\n", " thresh2 = cv2.adaptiveThreshold(blurimage2, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", + " # return thresh1\n", " imglist.append(thresh1)\n", " imglist.append(thresh2)\n", " threshmerge = cv2.bitwise_or(thresh1, thresh2)\n", " imglist.append(threshmerge)\n", + " # return threshmerge\n", " \n", " dim = int(min(threshmerge.shape[0], threshmerge.shape[1])/100)\n", " # dim = 2\n", @@ -151,6 +156,7 @@ " mask1t = cv2.morphologyEx(padthresh, cv2.MORPH_OPEN, kernel, iterations=1)\n", " # mask1t = cv2.morphologyEx(mask1t, cv2.MORPH_OPEN, skernel)\n", " imglist.append(mask1t)\n", + " # return mask1t\n", " \n", " contours, heirarchy = cv2.findContours(255-mask1t, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", " biggestcontour = max(contours, key=cv2.contourArea)\n", @@ -162,13 +168,15 @@ " # temp = cv2.drawContours(image, [biggestcontour], -1, 255, thickness=3)\n", " # imglist.append(temp)\n", " imglist.append(mask1)\n", + " # return mask1\n", " \n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, srkernel, iterations=2)\n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, srkernel)\n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel)\n", " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", " mask1 = mask1[sdim:-sdim, sdim:-sdim]\n", " imglist.append(mask1)\n", + " # return mask1\n", " \n", " maskc = cv2.cvtColor(mask1, cv2.COLOR_GRAY2BGR)\n", " whitedbackground = cv2.bitwise_or(image, maskc)\n", @@ -180,38 +188,6 @@ " \n", " \n", " \n", - " # contours, heirarchy = cv2.findContours(255-threshmerge, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " # biggestcontour = max(contours, key=cv2.contourArea)\n", - " \n", - " # blank = np.full(threshmerge.shape, 255, dtype=np.uint8)\n", - " # mask1 = blank.copy()\n", - " # mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", - " # # temp = cv2.drawContours(image, [biggestcontour], -1, 255, thickness=3)\n", - " # # imglist.append(temp)\n", - " \n", - " # mask1 = mf.padWithColour(mask1, sdim, sdim, fill=255)\n", - " \n", - " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=1)\n", - " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel)\n", - " # mask1 = mask1[sdim:-sdim, sdim:-sdim]\n", - " # imglist.append(mask1)\n", - " \n", - " # mask1merge = cv2.bitwise_and(mask1, mask1t)\n", - " # imglist.append(mask1merge)\n", - " \n", - " \n", - " \n", - " # maskc = cv2.cvtColor(mask1merge, cv2.COLOR_GRAY2BGR)\n", - " # # return maskc\n", - " # # return blurimage\n", - " # # print(blurimage)\n", - " # # print(maskc)\n", - " # whitedbackground = cv2.bitwise_or(image, maskc)\n", - " # imglist.append(whitedbackground)\n", - "\n", - " # return imglist\n", - " \n", - " \n", " imagecpy = image.copy()\n", " blurimage = cv2.blur(imagecpy, (13,13))\n", " \n", @@ -227,15 +203,18 @@ " # imagecpy = cv2.copyMakeBorder(blurimage, sdim, sdim, sdim, sdim, cv2.BORDER_REPLICATE)\n", " # return imagecpy\n", " # imagecpy = mf.padWithColour(image, hpadding=sdim, vpadding=sdim)\n", - " gray = cv2.cvtColor(blurimage, cv2.COLOR_BGR2GRAY)\n", + " \n", + " lab = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)\n", + " gray = lab[:,:,0]\n", + " # gray = cv2.cvtColor(blurimage, cv2.COLOR_BGR2GRAY)\n", " \n", " # blur = cv2.blur(gray, (7,7))\n", " \n", " # window = 51\n", - " window = gray.shape[1]//15\n", + " window = gray.shape[1]//7\n", " if window % 2 == 0:\n", " window += 1\n", - " thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 5)\n", + " thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 2)\n", " # thresh2 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU)[1]\n", " # thresh = cv2.bitwise_and(thresh1, thresh2)\n", " # return thresh\n", @@ -257,7 +236,7 @@ " # dim = dotsize\n", " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", " morphedthresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel)\n", - " # return morphedthresh\n", + " return morphedthresh\n", " \n", " \n", " contours, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", @@ -273,8 +252,8 @@ " \n", " mask1 = mf.padWithColour(mask1, sdim, sdim, fill=0)\n", " \n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel)\n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel)\n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=1)\n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel, iterations=1)\n", " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel)\n", " # return mask1\n", " \n", @@ -469,7 +448,7 @@ }, { "cell_type": "code", - "execution_count": 3039, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -489,13 +468,100 @@ }, { "cell_type": "code", - "execution_count": 3040, + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle, croprect):\n", + " # rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=0)\n", + " # rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n", + " # print(croprect)\n", + " hpad, vpad = mf.determineextrapadding(croprect[3], croprect[2], rotationangle)\n", + " # print(rotationangle)\n", + " # print(hpad, vpad)\n", + " adjustedrect = [croprect[0]-hpad, croprect[1]-vpad, croprect[2]+(2*hpad), croprect[3]+(2*vpad)]\n", + " adjustedrect = [int(s) for s in adjustedrect]\n", + " leftpad = -min(0, adjustedrect[0])\n", + " toppad = -min(0, adjustedrect[1])\n", + " rightpad = -min(0, baseimage.shape[1]-(adjustedrect[0]+adjustedrect[2]))\n", + " bottompad = -min(0, baseimage.shape[0]-(adjustedrect[1]+adjustedrect[3]))\n", + " \n", + " adjustedrect = [max(0, s) for s in adjustedrect]\n", + " \n", + " # print(leftpad, rightpad, toppad, bottompad)\n", + " ## DETERMINE PADDING AMOUNTS BECAUSE I THEN NEED TO PAD THE ORIGINAL IMAGE, CROP IT TO THE FIRST LEVEL, ROTATE, AND THEN CROP AGAIN\n", + " # print(adjustedrect)\n", + " # adjustedrect = croprect\n", + " borderType = cv2.BORDER_CONSTANT\n", + " padded = cv2.copyMakeBorder(baseimage, toppad, bottompad, leftpad, rightpad, borderType)\n", + " croppedpaddedimage = padded[adjustedrect[1]:adjustedrect[1]+adjustedrect[3], adjustedrect[0]:adjustedrect[0]+adjustedrect[2], :]\n", + " # print(croprect[2], croprect[3])\n", + " # print(croppedpaddedimage.shape)\n", + " # croppedog = baseimage[croprect[1]:croprect[1]+croprect[3], croprect[0]:croprect[0]+croprect[2], :]\n", + " # return croppedpaddedimage, croppedog\n", + " \n", + " rotatedbaseimage = mf.rotate(croppedpaddedimage, rotationangle)\n", + " \n", + " # print(vpad, hpad)\n", + " # print(croprect)\n", + " rotatedbaseimage = rotatedbaseimage[vpad:vpad+croprect[1]+croprect[3], hpad:hpad+croprect[0]+croprect[2], :]\n", + " # return rotatedbaseimage, 5\n", + " \n", + " \n", + " # print(preppedimage.shape)\n", + " rotateddst1 = mf.rotate(preppedimage, rotationangle)\n", + " # print(rotateddst1.shape)\n", + " # print(rotatedbaseimage.shape)\n", + " # return rotatedbaseimage, 5\n", + "\n", + " \n", + " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", + " # print(sizemultiplier)\n", + " # return rotatedbaseimage, rotationangle\n", + "\n", + " croppedbaseimage = mf.houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)\n", + "\n", + " finalbaseimage = mf.contourcrop(croppedbaseimage)\n", + "\n", + "\n", + " return finalbaseimage, rotationangle" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "def houghlinedeskewandcrop(image):\n", + " canny, ogimage, rect = mf.prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done\n", + " # return canny, croppedogimage\n", + " # print(canny.shape)\n", + " # print(croppedogimage.shape)\n", + "\n", + " ## -----------------finding angle to deskew-----------------\n", + " rotationangle = mf.houghlinedeskewangle(canny)\n", + " # print(rotationangle)\n", + "\n", + " \n", + " # rotatorrect = findcroprectforangle(rect, angle)\n", + "\n", + " # -----------------end of finding angle to deskew-----------------\n", + "\n", + " ## -----------------deskewing and then cropping-----------------\n", + " outimg, angle = houghlinedeskewthencrop(ogimage, canny, rotationangle, rect)\n", + " return outimg, angle" + ] + }, + { + "cell_type": "code", + "execution_count": 31, "metadata": {}, "outputs": [], "source": [ "def houghlineprocessing(image):\n", - " croppedanddeskewed, angle = mf.houghlinedeskewandcrop(image)\n", - " # return croppedanddeskewed\n", + " croppedanddeskewed, angle = houghlinedeskewandcrop(image)\n", + " return croppedanddeskewed\n", " \n", " \n", " postprocessed = cropclarifying(croppedanddeskewed)\n", @@ -520,7 +586,16 @@ }, { "cell_type": "code", - "execution_count": 3041, + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "# print(img.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, "metadata": {}, "outputs": [], "source": [ @@ -535,7 +610,7 @@ }, { "cell_type": "code", - "execution_count": 3042, + "execution_count": 34, "metadata": {}, "outputs": [], "source": [ @@ -549,7 +624,7 @@ }, { "cell_type": "code", - "execution_count": 3043, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -561,7 +636,7 @@ }, { "cell_type": "code", - "execution_count": 3044, + "execution_count": 36, "metadata": {}, "outputs": [], "source": [ @@ -570,7 +645,7 @@ }, { "cell_type": "code", - "execution_count": 3045, + "execution_count": 37, "metadata": {}, "outputs": [], "source": [ @@ -580,7 +655,7 @@ }, { "cell_type": "code", - "execution_count": 3046, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ @@ -610,14 +685,14 @@ }, { "cell_type": "code", - "execution_count": 3047, + "execution_count": 39, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "average time: 8.021544203162193(s)\n" + "average time: 0.1456429362297058(s)\n" ] } ], @@ -628,7 +703,7 @@ }, { "cell_type": "code", - "execution_count": 3048, + "execution_count": 40, "metadata": {}, "outputs": [], "source": [ @@ -638,7 +713,7 @@ }, { "cell_type": "code", - "execution_count": 3049, + "execution_count": 41, "metadata": {}, "outputs": [], "source": [ @@ -647,7 +722,7 @@ }, { "cell_type": "code", - "execution_count": 3050, + "execution_count": 42, "metadata": {}, "outputs": [], "source": [ diff --git a/code/autocropper/myfunctions.py b/code/autocropper/myfunctions.py index 8dd24af..ac55487 100644 --- a/code/autocropper/myfunctions.py +++ b/code/autocropper/myfunctions.py @@ -91,10 +91,6 @@ def mergecontours(contours): finalcontour = cv2.convexHull(cont) return finalcontour -def padWithColour(img, hpadding=0, vpadding=0, fill=(0,0,0)): - borderType = cv2.BORDER_CONSTANT - out = cv2.copyMakeBorder(img, vpadding, vpadding, hpadding, hpadding, borderType, None, fill) - return out # funtion to correct the median-angle to give it to the cv2.warpaffine() function @@ -346,7 +342,7 @@ def rotateLine(img, line, angle, returnint=True): pt2 = rotatePoint(img, (line[2],line[3]), angle, returnint) return (pt1[0], pt1[1], pt2[0], pt2[1]) -def prepimageforhoughline(image): +def prepimageforhoughline(image, returnrect=True): prepped, scaler, hp, vp = squareandthenresize(image, fill=255, width=1000, returnscalerinfo=True) prepped, croprect = premorphCrop(prepped) if (prepped.shape[1] > prepped.shape[0]): @@ -368,9 +364,12 @@ def prepimageforhoughline(image): dst1 = cv2.Canny(dst1, 0, 500, None, 3) # return dst1 - accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :] - # accompaniedimage = squarepad(accompaniedimage, fill=255) - return dst1, accompaniedimage + if returnrect: + return dst1, image, finalcroprect + else: + accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :] + # accompaniedimage = squarepad(accompaniedimage, fill=255) + return dst1, accompaniedimage def houghlinedeskewangle(image): lines = cv2.HoughLines(image, 1, np.pi/180, int(max(image.shape[0], image.shape[1])/6), None, 0, 0) @@ -399,8 +398,7 @@ def houghlinedeskewangle(image): rotationangle = np.rad2deg(mode) return rotationangle -def determineextrapadding(img, angle): - h, w = img.shape[0], img.shape[1] +def determineextrapadding(h,w, angle): radangle = abs(np.deg2rad(angle)) # print(radangle) totalheightrot = w*np.sin(radangle) + h*np.cos(radangle) @@ -413,7 +411,8 @@ def determineextrapadding(img, angle): return hpad, vpad def rotatewithexactpadding(img, angle, fill=(0,0,0)): - hpad, vpad = determineextrapadding(img, angle) + h, w = img[0], img[1] + hpad, vpad = determineextrapadding(w,h, angle) baseimage = padWithColour(img, hpad, vpad, fill=fill) rotatedimg = rotate(baseimage, angle) return rotatedimg @@ -477,8 +476,21 @@ def contourcrop(baseimage): return finalbaseimage def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle): - rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=0) - rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0)) + # rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=0) + # rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0)) + hpad, vpad = mf.determineextrapadding(rect[3], rect[2], rotationangle) + adjustedrect = [rect[0]-hpad, rect[1]-vpad, rect[2]+(2*hpad), rect[3]+(2*vpad)] + croppedogimage = croppedogimage[adjustedrect[1]:adjustedrect[1]+adjustedrect[3], adjustedrect[0]:adjustedrect[0]+adjustedrect[2], :] + + houghlinerotate() + + + + print(preppedimage.shape) + rotateddst1 = rotate(preppedimage, rotationangle) + print(rotateddst1.shape) + rotatedbaseimage = rotate(baseimage, rotationangle) + sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0] # print(sizemultiplier) # return rotatedbaseimage, rotationangle From b149e869636e79e2c90a9efba6c1ecb8d183faae Mon Sep 17 00:00:00 2001 From: Ethan Wellenreiter Date: Sun, 26 Nov 2023 17:40:32 -0500 Subject: [PATCH 5/8] Autocrop Checkpoint with fixed rotation. Need to make sure that houghlinepcrop still works correctly after the adjustments. Signed-off-by: Ethan Wellenreiter --- code/autocropper/houghlinedevspace.ipynb | 144 ++++++++++++++++------- code/autocropper/myfunctions.py | 15 ++- 2 files changed, 111 insertions(+), 48 deletions(-) diff --git a/code/autocropper/houghlinedevspace.ipynb b/code/autocropper/houghlinedevspace.ipynb index 585ffd6..6eccd36 100644 --- a/code/autocropper/houghlinedevspace.ipynb +++ b/code/autocropper/houghlinedevspace.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 22, + "execution_count": 123, "metadata": {}, "outputs": [], "source": [ @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 124, "metadata": {}, "outputs": [], "source": [ @@ -55,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 125, "metadata": {}, "outputs": [], "source": [ @@ -77,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 126, "metadata": {}, "outputs": [], "source": [ @@ -91,18 +91,18 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 127, "metadata": {}, "outputs": [], "source": [ - "img = cv2.imread('/mnt/dataset/baseimages/15.jpg')\n", + "img = cv2.imread('/mnt/dataset/baseimages/6.jpg')\n", "# img = cv2.imread('/mnt/code/autocropper/test_images/IMG_7605.jpg')\n", "testall = False" ] }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 128, "metadata": {}, "outputs": [], "source": [ @@ -448,7 +448,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 129, "metadata": {}, "outputs": [], "source": [ @@ -468,11 +468,55 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 130, "metadata": {}, "outputs": [], "source": [ "def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle, croprect):\n", + " imgcopy = baseimage.copy()\n", + " # sizemultiplier = croprect[3]/preppedimage.shape[0]\n", + " \n", + " ##adjust the rotation angle if it causes the rectangle to flip w and h size ordering. That is, if it will cause the width to be greater than the height or something. let me think about it for a second more.\n", + " # print(baseimage.shape[:2])\n", + " mask = np.full(baseimage.shape[:2], fill_value=255, dtype=np.uint8)\n", + " mask = cv2.rectangle(mask, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=0, thickness=cv2.FILLED)\n", + " \n", + " \n", + " \n", + " # rotatedmask = mf.rotate(mask, rotationangle, fill=255)\n", + " # print(mask.shape)\n", + " # return mask, 5\n", + " rotatedmask = mf.rotatewithexactpadding(mask, rotationangle, fill=255)\n", + " rotatedbaseimage = mf.rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n", + " # return preppedimage, 5\n", + " rotateddst1 = mf.rotatewithexactpadding(preppedimage, rotationangle, fill=(0,0,0))\n", + " fcr = mf.croptoblack(rotatedmask, extraborder=0, returnrect=True) #finalcroprect\n", + " # return rotatedmask, 5\n", + " # return rotatedbaseimage, 5\n", + " # print(fcr)\n", + " rotatedbaseimage = rotatedbaseimage[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", + " # rotateddst1 = rotateddst1[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", + " # return mask, 5\n", + " # return rotatedbaseimage, 5\n", + " \n", + " \n", + " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", + " print(sizemultiplier)\n", + " # return rotatedbaseimage, rotationangle\n", + "\n", + " croppedbaseimage = mf.houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)\n", + " \n", + " return croppedbaseimage, rotationangle\n", + "\n", + " # finalbaseimage = mf.contourcrop(croppedbaseimage)\n", + "\n", + "\n", + " return finalbaseimage, rotationangle\n", + " \n", + " \n", + " \n", + " \n", + " \n", " # rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=0)\n", " # rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n", " # print(croprect)\n", @@ -481,31 +525,43 @@ " # print(hpad, vpad)\n", " adjustedrect = [croprect[0]-hpad, croprect[1]-vpad, croprect[2]+(2*hpad), croprect[3]+(2*vpad)]\n", " adjustedrect = [int(s) for s in adjustedrect]\n", - " leftpad = -min(0, adjustedrect[0])\n", - " toppad = -min(0, adjustedrect[1])\n", - " rightpad = -min(0, baseimage.shape[1]-(adjustedrect[0]+adjustedrect[2]))\n", - " bottompad = -min(0, baseimage.shape[0]-(adjustedrect[1]+adjustedrect[3]))\n", - " \n", - " adjustedrect = [max(0, s) for s in adjustedrect]\n", + "\n", + " # print(adjustedrect)\n", + " leftpad = abs(min(0, adjustedrect[0]))\n", + " toppad = abs(min(0, adjustedrect[1]))\n", + " rightpad = abs(min(0, baseimage.shape[1]-(adjustedrect[0]+adjustedrect[2])))\n", + " bottompad = abs(min(0, baseimage.shape[0]-(adjustedrect[1]+adjustedrect[3])))\n", " \n", " # print(leftpad, rightpad, toppad, bottompad)\n", + " \n", + " borderType = cv2.BORDER_CONSTANT\n", + " # padded = cv2.copyMakeBorder(baseimage, toppad, bottompad, leftpad, rightpad, borderType)\n", + " # imgcopy = padded.copy()\n", + " imgcopy = cv2.rectangle(imgcopy, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=(0,255,0), thickness=3)\n", + " imgcopy = cv2.rectangle(imgcopy, (adjustedrect[0], adjustedrect[1]), (adjustedrect[0]+adjustedrect[2], adjustedrect[1]+adjustedrect[3]), color=(0,0,255), thickness=3)\n", + " \n", + " adjustedrect = [max(0, s) for s in adjustedrect]\n", + " imgcopy = cv2.rectangle(imgcopy, (adjustedrect[0], adjustedrect[1]), (adjustedrect[0]+adjustedrect[2], adjustedrect[1]+adjustedrect[3]), color=(255,0,0), thickness=3)\n", + " return imgcopy, 5\n", + " \n", + "\n", " ## DETERMINE PADDING AMOUNTS BECAUSE I THEN NEED TO PAD THE ORIGINAL IMAGE, CROP IT TO THE FIRST LEVEL, ROTATE, AND THEN CROP AGAIN\n", " # print(adjustedrect)\n", " # adjustedrect = croprect\n", - " borderType = cv2.BORDER_CONSTANT\n", - " padded = cv2.copyMakeBorder(baseimage, toppad, bottompad, leftpad, rightpad, borderType)\n", + " # borderType = cv2.BORDER_CONSTANT\n", + " # padded = cv2.copyMakeBorder(baseimage, toppad, bottompad, leftpad, rightpad, borderType)\n", " croppedpaddedimage = padded[adjustedrect[1]:adjustedrect[1]+adjustedrect[3], adjustedrect[0]:adjustedrect[0]+adjustedrect[2], :]\n", " # print(croprect[2], croprect[3])\n", " # print(croppedpaddedimage.shape)\n", " # croppedog = baseimage[croprect[1]:croprect[1]+croprect[3], croprect[0]:croprect[0]+croprect[2], :]\n", - " # return croppedpaddedimage, croppedog\n", + " # return croppedpaddedimage, 5\n", " \n", " rotatedbaseimage = mf.rotate(croppedpaddedimage, rotationangle)\n", " \n", " # print(vpad, hpad)\n", " # print(croprect)\n", - " rotatedbaseimage = rotatedbaseimage[vpad:vpad+croprect[1]+croprect[3], hpad:hpad+croprect[0]+croprect[2], :]\n", - " # return rotatedbaseimage, 5\n", + " # rotatedbaseimage = rotatedbaseimage[vpad:vpad+croprect[3], hpad:hpad+croprect[2], :]\n", + " return rotatedbaseimage, 5\n", " \n", " \n", " # print(preppedimage.shape)\n", @@ -529,7 +585,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 131, "metadata": {}, "outputs": [], "source": [ @@ -555,7 +611,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 132, "metadata": {}, "outputs": [], "source": [ @@ -586,7 +642,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 133, "metadata": {}, "outputs": [], "source": [ @@ -595,9 +651,17 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 134, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1.695\n" + ] + } + ], "source": [ "# prepped, scaler, hp, vp = mf.squareandthenresize(img, fill=255, width=1000, returnscalerinfo=True)\n", "outs = houghlineprocessing(img)\n", @@ -610,7 +674,7 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 135, "metadata": {}, "outputs": [], "source": [ @@ -624,7 +688,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 136, "metadata": {}, "outputs": [], "source": [ @@ -636,16 +700,16 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 137, "metadata": {}, "outputs": [], "source": [ - "testall = True" + "# testall = True" ] }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 138, "metadata": {}, "outputs": [], "source": [ @@ -655,7 +719,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 139, "metadata": {}, "outputs": [], "source": [ @@ -685,17 +749,9 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 140, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "average time: 0.1456429362297058(s)\n" - ] - } - ], + "outputs": [], "source": [ "if testall:\n", " results = testondataset(\"/mnt/dataset/baseimages/\", houghlineprocessing)" @@ -703,7 +759,7 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 141, "metadata": {}, "outputs": [], "source": [ @@ -713,7 +769,7 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 142, "metadata": {}, "outputs": [], "source": [ @@ -722,7 +778,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 143, "metadata": {}, "outputs": [], "source": [ diff --git a/code/autocropper/myfunctions.py b/code/autocropper/myfunctions.py index ac55487..c44fd77 100644 --- a/code/autocropper/myfunctions.py +++ b/code/autocropper/myfunctions.py @@ -400,6 +400,8 @@ def houghlinedeskewangle(image): def determineextrapadding(h,w, angle): radangle = abs(np.deg2rad(angle)) + # print(type(h), type(w), type(angle)) + # print(h, w, angle) # print(radangle) totalheightrot = w*np.sin(radangle) + h*np.cos(radangle) # print(h, totalheightrot) @@ -411,10 +413,13 @@ def determineextrapadding(h,w, angle): return hpad, vpad def rotatewithexactpadding(img, angle, fill=(0,0,0)): - h, w = img[0], img[1] - hpad, vpad = determineextrapadding(w,h, angle) + h, w = img.shape[0], img.shape[1] + hpad, vpad = determineextrapadding(h=h,w=w, angle=angle) + # fill1 = fill + # print(fill) baseimage = padWithColour(img, hpad, vpad, fill=fill) - rotatedimg = rotate(baseimage, angle) + # return baseimage + rotatedimg = rotate(baseimage, angle,fill=fill) return rotatedimg def houghlinepcrop(baseimage, preppedimage, scalingmultiplier): @@ -727,7 +732,7 @@ def cropclarifying(image): return lineout # implement a function that's called refine text -def croptoblack(image, extraborder=10): +def croptoblack(image, extraborder=10, returnrect=False): invertedimage = cv2.bitwise_not(image) blackpixels = cv2.findNonZero(invertedimage) mins = np.min(blackpixels, axis=0) @@ -737,6 +742,8 @@ def croptoblack(image, extraborder=10): maxx = min(maxs[0][0]+extraborder, image.shape[1]) maxy = min(maxs[0][1]+extraborder, image.shape[0]) # print(blackpixels) + if (returnrect): + return [minx,miny,maxx-minx,maxy-miny] return image[miny:maxy, minx:maxx] def reduceColours(x, centering=127): From d9aa07121acbe598679070cb932a09d06298e7d6 Mon Sep 17 00:00:00 2001 From: Ethan Wellenreiter Date: Mon, 27 Nov 2023 15:54:23 -0500 Subject: [PATCH 6/8] Fixed the houghline cropping and deskewing for no black rotation lines Adjusted the process so that the black lines left after the houghline cropping and rotating are no longer there. Signed-off-by: Ethan Wellenreiter --- code/autocropper/houghlinedevspace.ipynb | 184 +++++++++++++++++------ code/autocropper/myfunctions.py | 53 +++---- 2 files changed, 159 insertions(+), 78 deletions(-) diff --git a/code/autocropper/houghlinedevspace.ipynb b/code/autocropper/houghlinedevspace.ipynb index 6eccd36..c36a482 100644 --- a/code/autocropper/houghlinedevspace.ipynb +++ b/code/autocropper/houghlinedevspace.ipynb @@ -2,9 +2,20 @@ "cells": [ { "cell_type": "code", - "execution_count": 123, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/torchvision/datapoints/__init__.py:12: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n", + " warnings.warn(_BETA_TRANSFORMS_WARNING)\n", + "/usr/local/lib/python3.10/dist-packages/torchvision/transforms/v2/__init__.py:54: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n", + " warnings.warn(_BETA_TRANSFORMS_WARNING)\n" + ] + } + ], "source": [ "import cv2\n", "import myfunctions as mf\n", @@ -15,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -55,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -77,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -91,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -448,7 +459,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -468,47 +479,121 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def prepimageforhoughline(image, returnrect=True):\n", + " prepped, scaler, hp, vp = mf.squareandthenresize(image, fill=255, width=1000, returnscalerinfo=True)\n", + " ogpreppedshape = prepped.shape\n", + " # print(ogpreppedshape)\n", + " prepped, croprect = mf.premorphCrop(prepped)\n", + " # print(prepped.shape)\n", + " if (prepped.shape[1] > prepped.shape[0]):\n", + " prepped, preppedscaler = mf.ResizeWithAspectRatio(prepped, width=1000, retscale=True)\n", + " else:\n", + " prepped, preppedscaler = mf.ResizeWithAspectRatio(prepped, height=1000, retscale=True)\n", + " # print(prepped.shape)\n", + " # print(preppedscaler)\n", + " finalcroprect = (int(croprect[0]*scaler - hp), int(croprect[1]*scaler - vp), int(croprect[2]*scaler), int(croprect[3]*scaler))\n", + " gray1 = cv2.cvtColor(prepped, cv2.COLOR_BGR2GRAY)\n", + "\n", + " dst1 = cv2.Canny(gray1, 0, 500, None, 3)\n", + "\n", + " \n", + " kernel = np.ones((5,5), np.uint8)\n", + " out = cv2.morphologyEx(dst1, cv2.MORPH_DILATE, kernel)\n", + " out = cv2.blur(out, (5,5))\n", + " kernel = np.ones((6,6), np.uint8)\n", + " dst1 = cv2.morphologyEx(out, cv2.MORPH_ERODE, kernel)\n", + " # return dst1\n", + "\n", + " dst1 = cv2.Canny(dst1, 0, 500, None, 3)\n", + " # return dst1\n", + " accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :]\n", + " if returnrect:\n", + " borderType = cv2.BORDER_CONSTANT\n", + " ## first the og padding when we include the padding added by squaring\n", + " preppadding = [croprect[0], croprect[1], ogpreppedshape[1]-(croprect[0]+croprect[2]), ogpreppedshape[0]-(croprect[1]+croprect[3])]\n", + " # print(croprect)\n", + " # print(dst1.shape)\n", + " # print(preppadding)\n", + " preppadding = [int(s/preppedscaler) for s in preppadding]\n", + " # print(preppadding)\n", + " ## now adjust for any padding. hp and vp are for the full sized image so they need to be scaled down first.\n", + " # adjustedhp = int(hp/scaler)\n", + " # adjustedvp = int(vp/scaler)\n", + " # preppadding[0] -= adjustedhp\n", + " # preppadding[2] -= adjustedhp\n", + " # preppadding[1] -= adjustedvp\n", + " # preppadding[3] -= adjustedvp\n", + " \n", + " paddedprepped = cv2.copyMakeBorder(dst1, preppadding[1], preppadding[3], preppadding[0], preppadding[2], borderType, 0)\n", + " # paddedprepped = dst1\n", + " squaredimage = mf.squarepad(image, fill=0)\n", + " return dst1, accompaniedimage, paddedprepped, squaredimage, finalcroprect\n", + " else:\n", + " # accompaniedimage = squarepad(accompaniedimage, fill=255)\n", + " return dst1, accompaniedimage" + ] + }, + { + "cell_type": "code", + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle, croprect):\n", - " imgcopy = baseimage.copy()\n", + " # imglist = []\n", + " # print(preppedimage.shape)\n", + " # print(baseimage.shape)\n", + " \n", + " # imgcopy = baseimage.copy()\n", " # sizemultiplier = croprect[3]/preppedimage.shape[0]\n", " \n", " ##adjust the rotation angle if it causes the rectangle to flip w and h size ordering. That is, if it will cause the width to be greater than the height or something. let me think about it for a second more.\n", " # print(baseimage.shape[:2])\n", - " mask = np.full(baseimage.shape[:2], fill_value=255, dtype=np.uint8)\n", - " mask = cv2.rectangle(mask, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=0, thickness=cv2.FILLED)\n", - " \n", - " \n", + " # mask = np.full(baseimage.shape[:2], fill_value=255, dtype=np.uint8)\n", + " # mask = cv2.rectangle(mask, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=0, thickness=cv2.FILLED)\n", + " # temp = baseimage[croprect[1]:croprect[1]+croprect[3], croprect[0]:croprect[0]+croprect[2], :]\n", " \n", " # rotatedmask = mf.rotate(mask, rotationangle, fill=255)\n", " # print(mask.shape)\n", " # return mask, 5\n", - " rotatedmask = mf.rotatewithexactpadding(mask, rotationangle, fill=255)\n", + " # rotatedmask = mf.rotatewithexactpadding(mask, rotationangle, fill=255)\n", " rotatedbaseimage = mf.rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n", + " # rotatedtemp = mf.rotatewithexactpadding(temp, rotationangle, fill=(0,0,0))\n", " # return preppedimage, 5\n", " rotateddst1 = mf.rotatewithexactpadding(preppedimage, rotationangle, fill=(0,0,0))\n", - " fcr = mf.croptoblack(rotatedmask, extraborder=0, returnrect=True) #finalcroprect\n", + " \n", + " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", + " \n", + " # fcr = mf.croptoblack(rotatedmask, extraborder=0, returnrect=True) #finalcroprect\n", + " \n", + " # print(sizemultiplier)\n", + " # imglist.append(rotatedmask)\n", + " # imglist.append(rotatedbaseimage)\n", + " # imglist.append(rotateddst1)\n", + " # return imglist, 5\n", " # return rotatedmask, 5\n", " # return rotatedbaseimage, 5\n", + " # return rotateddst1, 5\n", " # print(fcr)\n", - " rotatedbaseimage = rotatedbaseimage[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", + " # rotatedbaseimage = rotatedbaseimage[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", " # rotateddst1 = rotateddst1[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", " # return mask, 5\n", " # return rotatedbaseimage, 5\n", " \n", " \n", - " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", - " print(sizemultiplier)\n", + " # sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", + " # print(sizemultiplier)\n", " # return rotatedbaseimage, rotationangle\n", "\n", " croppedbaseimage = mf.houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)\n", " \n", - " return croppedbaseimage, rotationangle\n", + " # return croppedbaseimage, rotationangle\n", "\n", - " # finalbaseimage = mf.contourcrop(croppedbaseimage)\n", + " finalbaseimage = mf.contourcrop(croppedbaseimage)\n", "\n", "\n", " return finalbaseimage, rotationangle\n", @@ -585,18 +670,18 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def houghlinedeskewandcrop(image):\n", - " canny, ogimage, rect = mf.prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done\n", - " # return canny, croppedogimage\n", + " croppedcanny, croppedimage, canny, ogimage, rect = prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done\n", + " # return canny, ogimage\n", " # print(canny.shape)\n", " # print(croppedogimage.shape)\n", "\n", " ## -----------------finding angle to deskew-----------------\n", - " rotationangle = mf.houghlinedeskewangle(canny)\n", + " rotationangle = mf.houghlinedeskewangle(croppedcanny)\n", " # print(rotationangle)\n", "\n", " \n", @@ -611,12 +696,12 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "def houghlineprocessing(image):\n", - " croppedanddeskewed, angle = houghlinedeskewandcrop(image)\n", + " croppedanddeskewed, angle = mf.houghlinedeskewandcrop(image)\n", " return croppedanddeskewed\n", " \n", " \n", @@ -642,7 +727,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -651,20 +736,13 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 13, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.695\n" - ] - } - ], + "outputs": [], "source": [ "# prepped, scaler, hp, vp = mf.squareandthenresize(img, fill=255, width=1000, returnscalerinfo=True)\n", "outs = houghlineprocessing(img)\n", + "# outs = prepimageforhoughline(img, returnrect=True)\n", "# print(img.shape)\n", "# outs = houghlinedeskewandcrop(img)\n", "# outs = outs[0]\n", @@ -674,7 +752,7 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -688,7 +766,7 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -700,16 +778,16 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ - "# testall = True" + "testall = True" ] }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -719,7 +797,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -749,9 +827,17 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "average time: 0.1962958723306656(s)\n" + ] + } + ], "source": [ "if testall:\n", " results = testondataset(\"/mnt/dataset/baseimages/\", houghlineprocessing)" @@ -759,7 +845,7 @@ }, { "cell_type": "code", - "execution_count": 141, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -769,7 +855,7 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -778,7 +864,7 @@ }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ diff --git a/code/autocropper/myfunctions.py b/code/autocropper/myfunctions.py index c44fd77..fddf559 100644 --- a/code/autocropper/myfunctions.py +++ b/code/autocropper/myfunctions.py @@ -344,11 +344,12 @@ def rotateLine(img, line, angle, returnint=True): def prepimageforhoughline(image, returnrect=True): prepped, scaler, hp, vp = squareandthenresize(image, fill=255, width=1000, returnscalerinfo=True) + ogpreppedshape = prepped.shape prepped, croprect = premorphCrop(prepped) if (prepped.shape[1] > prepped.shape[0]): - prepped = ResizeWithAspectRatio(prepped, width=1000) + prepped, preppedscaler = ResizeWithAspectRatio(prepped, width=1000, retscale=True) else: - prepped = ResizeWithAspectRatio(prepped, height=1000) + prepped, preppedscaler = ResizeWithAspectRatio(prepped, height=1000, retscale=True) finalcroprect = (int(croprect[0]*scaler - hp), int(croprect[1]*scaler - vp), int(croprect[2]*scaler), int(croprect[3]*scaler)) gray1 = cv2.cvtColor(prepped, cv2.COLOR_BGR2GRAY) @@ -364,11 +365,17 @@ def prepimageforhoughline(image, returnrect=True): dst1 = cv2.Canny(dst1, 0, 500, None, 3) # return dst1 + accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :] if returnrect: - return dst1, image, finalcroprect + borderType = cv2.BORDER_CONSTANT + preppadding = [croprect[0], croprect[1], ogpreppedshape[1]-(croprect[0]+croprect[2]), ogpreppedshape[0]-(croprect[1]+croprect[3])] + preppadding = [int(s/preppedscaler) for s in preppadding] + paddedprepped = cv2.copyMakeBorder(dst1, preppadding[1], preppadding[3], preppadding[0], preppadding[2], borderType, 0) + + squaredimage = squarepad(image, fill=0) + + return dst1, accompaniedimage, paddedprepped, squaredimage, finalcroprect else: - accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :] - # accompaniedimage = squarepad(accompaniedimage, fill=255) return dst1, accompaniedimage def houghlinedeskewangle(image): @@ -480,47 +487,35 @@ def contourcrop(baseimage): finalbaseimage = baseimage[scaledmx[1]:scaledmx[1]+scaledmx[3], scaledmx[0]:scaledmx[0]+scaledmx[2], :] return finalbaseimage -def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle): - # rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=0) - # rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0)) - hpad, vpad = mf.determineextrapadding(rect[3], rect[2], rotationangle) - adjustedrect = [rect[0]-hpad, rect[1]-vpad, rect[2]+(2*hpad), rect[3]+(2*vpad)] - croppedogimage = croppedogimage[adjustedrect[1]:adjustedrect[1]+adjustedrect[3], adjustedrect[0]:adjustedrect[0]+adjustedrect[2], :] - - houghlinerotate() - - - - print(preppedimage.shape) - rotateddst1 = rotate(preppedimage, rotationangle) - print(rotateddst1.shape) - rotatedbaseimage = rotate(baseimage, rotationangle) - +def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle, croprect): + rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0)) + rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=(0,0,0)) sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0] - # print(sizemultiplier) - # return rotatedbaseimage, rotationangle + croppedbaseimage = houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier) finalbaseimage = contourcrop(croppedbaseimage) - - + return finalbaseimage, rotationangle def houghlinedeskewandcrop(image): - canny, croppedogimage = prepimageforhoughline(image) ## scaling and cropping occurs. need to also return the changes done - # return canny, croppedogimage + croppedcanny, croppedimage, canny, ogimage, rect = prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done + # return canny, ogimage # print(canny.shape) # print(croppedogimage.shape) ## -----------------finding angle to deskew----------------- - rotationangle = houghlinedeskewangle(canny) + rotationangle = houghlinedeskewangle(croppedcanny) # print(rotationangle) + + # rotatorrect = findcroprectforangle(rect, angle) + # -----------------end of finding angle to deskew----------------- ## -----------------deskewing and then cropping----------------- - outimg, angle = houghlinedeskewthencrop(croppedogimage, canny, rotationangle) + outimg, angle = houghlinedeskewthencrop(ogimage, canny, rotationangle, rect) return outimg, angle def bruteforceprocessrects(greaterrects, lesserrects): From 6a27931e8b11700b50e7f593804c4fd2b0fd58aa Mon Sep 17 00:00:00 2001 From: Ethan Wellenreiter Date: Fri, 1 Dec 2023 19:02:53 -0500 Subject: [PATCH 7/8] Random whiteout checkpoint Signed-off-by: Ethan Wellenreiter --- code/autocropper/houghlinedevspace.ipynb | 543 +++++++++++------------ 1 file changed, 261 insertions(+), 282 deletions(-) diff --git a/code/autocropper/houghlinedevspace.ipynb b/code/autocropper/houghlinedevspace.ipynb index c36a482..b74e731 100644 --- a/code/autocropper/houghlinedevspace.ipynb +++ b/code/autocropper/houghlinedevspace.ipynb @@ -2,20 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 1895, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.10/dist-packages/torchvision/datapoints/__init__.py:12: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n", - " warnings.warn(_BETA_TRANSFORMS_WARNING)\n", - "/usr/local/lib/python3.10/dist-packages/torchvision/transforms/v2/__init__.py:54: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n", - " warnings.warn(_BETA_TRANSFORMS_WARNING)\n" - ] - } - ], + "outputs": [], "source": [ "import cv2\n", "import myfunctions as mf\n", @@ -26,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1896, "metadata": {}, "outputs": [], "source": [ @@ -66,7 +55,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1897, "metadata": {}, "outputs": [], "source": [ @@ -88,7 +77,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 1898, "metadata": {}, "outputs": [], "source": [ @@ -102,18 +91,18 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 1899, "metadata": {}, "outputs": [], "source": [ - "img = cv2.imread('/mnt/dataset/baseimages/6.jpg')\n", + "img = cv2.imread('/mnt/dataset/baseimages/12.jpg')\n", "# img = cv2.imread('/mnt/code/autocropper/test_images/IMG_7605.jpg')\n", "testall = False" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 1900, "metadata": {}, "outputs": [], "source": [ @@ -121,52 +110,209 @@ "\n", "\n", "def whiteoutbackground(image):\n", - " sdim = int(min(image.shape[0], image.shape[1])/10)\n", + " ogshape = image.shape\n", + " shrunkdim=1000\n", + " if (image.shape[1] > image.shape[0]):\n", + " shrunkimg, scaler = mf.ResizeWithAspectRatio(image, width=shrunkdim, retscale=True)\n", + " else:\n", + " shrunkimg, scaler = mf.ResizeWithAspectRatio(image, height=shrunkdim, retscale=True)\n", + " \n", + " mainimage = shrunkimg\n", + " \n", + " sdim = int(min(mainimage.shape[0], mainimage.shape[1])/5)\n", " srkernel = cv2.getStructuringElement(cv2.MORPH_RECT, (sdim, sdim))\n", " skernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (sdim, sdim))\n", " \n", " \n", - " lab = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)\n", + " lab = cv2.cvtColor(mainimage, cv2.COLOR_BGR2LAB)\n", " \n", " imglist = []\n", - " imglist.append(image)\n", + " # imglist.append(mainimage)\n", " \n", - " # currentimgofatype = lab[:,:,0]\n", - " # imglist.append(currentimgofatype)\n", + " labl = lab[:,:,0]\n", + " # imglist.append(labl)\n", " # imglist.append(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY))\n", " laba = lab[:,:,1]\n", - " # imglist.append(currentimgofatype)\n", + " # imglist.append(laba)\n", " labb = lab[:,:,2]\n", - " # imglist.append(currentimgofatype)\n", - "\n", - "\n", - " blurimage1 = cv2.blur(laba, (5,5))\n", - " blurimage2 = cv2.blur(labb, (5,5))\n", + " # imglist.append(labb)\n", " \n", + " \n", + " # canny = cv2.Canny(labl, 0, 500)\n", + " threshl = cv2.threshold(labl, 0, 255, cv2.THRESH_OTSU)[1]\n", + " # return threshl\n", + " \n", + " \n", + " dim = int(min(mainimage.shape[0], mainimage.shape[1])/100)\n", + " # dim = 2\n", + " # dim = dotsize\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", + " kernelell = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dim, dim))\n", + " \n", + " paddedl = mf.padWithColour(threshl, sdim*2, sdim*2, fill=0)\n", + " # return paddedl\n", + " \n", + " morphedl = paddedl\n", + " morphedl = cv2.morphologyEx(morphedl, cv2.MORPH_OPEN, kernel, iterations=1)\n", + " \n", + " # return morphedl\n", + " \n", + " canny = cv2.Canny(morphedl, 0, 500)\n", + " # return canny\n", + "\n", + " minlength = max(mainimage.shape[0], mainimage.shape[1])//7\n", + " mingap = max(mainimage.shape[0], mainimage.shape[1])//20\n", + " linesP = cv2.HoughLinesP(canny, 1, np.pi / 180, 30, None, minlength, mingap)\n", + " # print(linesP)\n", + " \n", + " vmarginlines = mf.WithinXDegrees(linesP, 10)\n", + " hmarginlines = mf.WithinXDegrees(linesP, 10, baseangle=90)\n", + " \n", + " marginlines = np.append(vmarginlines, hmarginlines, axis=0)\n", + " \n", + " # colourdst = cv2.cvtColor(morphedl, cv2.COLOR_GRAY2BGR)\n", + " # if marginlines is not None:\n", + " # for l in marginlines:\n", + " # cv2.line(colourdst, (int(l[0]), int(l[1])), (int(l[2]), int(l[3])), (0,0,255), 3, cv2.LINE_AA)\n", + " # return colourdst\n", + "\n", + " # morphedl = 255-cv2.morphologyEx(255-threshl, cv2.MORPH_OPEN, kernel, iterations=3)\n", + " morphedl = paddedl\n", + " morphedl = cv2.morphologyEx(morphedl, cv2.MORPH_ERODE, kernel, iterations=1)\n", + " morphedl = cv2.morphologyEx(morphedl, cv2.MORPH_ERODE, kernelell, iterations=1)\n", + "\n", + " # return morphedl\n", + " \n", + " contours, heirarchy = cv2.findContours(morphedl, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " print(contours)\n", + " biggestcontour = max(contours, key=cv2.contourArea)\n", + " return canny\n", + " \n", + " \n", + " blank = np.full(labl.shape, 255, dtype=np.uint8)\n", + " mask1 = blank.copy()\n", + " mask1 = mf.padWithColour(mask1, sdim*2, sdim*2, fill=255)\n", + " mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", + " \n", + " \n", + " mask1 = mask1[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " \n", + " \n", + " # resizemask = cv2.resize(mask1, (ogshape[1], ogshape[0]))\n", + " # return resizemask\n", + " maskc = cv2.cvtColor(mask1, cv2.COLOR_GRAY2BGR)\n", + " # print(maskc.shape)\n", + " # print(image.shape)\n", + " whitedbackground = cv2.bitwise_or(mainimage, maskc)\n", + " # return whitedbackground\n", + " \n", + " \n", + " lab2 = cv2.cvtColor(whitedbackground, cv2.COLOR_BGR2LAB)\n", + " \n", + " lab2l = lab2[:,:,0]\n", + " \n", + " \n", + " otsu2 = cv2.threshold(lab2l, 0, 255, cv2.THRESH_OTSU)[1]\n", + " \n", + " expandedmask1 = cv2.morphologyEx(mask1, cv2.MORPH_DILATE, kernel, iterations=1)\n", + " expandedmask1 = cv2.morphologyEx(expandedmask1, cv2.MORPH_DILATE, kernelell, iterations=1)\n", + " # return expandedmask1\n", + " \n", + " maskmerge = cv2.bitwise_and(otsu2, 255-expandedmask1)\n", + " return mask1\n", + " return maskmerge\n", + " \n", + " # return otsu2\n", + " \n", + " mpad = mf.padWithColour(maskmerge, sdim*2, sdim*2, fill=0)\n", + " return mpad\n", + " \n", + " #MORPHOLOGIES \n", + " morphed2 = cv2.morphologyEx(mpad, cv2.MORPH_ERODE, kernel, iterations=1)\n", + " morphed2 = cv2.morphologyEx(morphed2, cv2.MORPH_ERODE, kernelell, iterations=1)\n", + " return morphed2\n", + " \n", + " contours, heirarchy = cv2.findContours(morphed2, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " biggestcontour = max(contours, key=cv2.contourArea)\n", + " \n", + " \n", + " mask2 = blank.copy()\n", + " mask2 = mf.padWithColour(mask2, sdim*2, sdim*2, fill=255)\n", + " mask2 = cv2.drawContours(mask2, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", + " \n", + " \n", + " mask2 = mask2[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " \n", + " return mask2\n", + " \n", + " test = cv2.inpaint(whitedbackground, resizemask, 3, cv2.INPAINT_TELEA)\n", + " \n", + " return test\n", + " \n", + " contours, heirarchy = cv2.findContours(255-labl, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)\n", + " \n", + " imgout = cv2.drawContours(mainimage, contours, -1, (0,255,0), thickness=3)\n", + " return imgout\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + "\n", + " blurimagel = cv2.blur(labl, (7,7))\n", + " blurimagea = cv2.blur(laba, (7,7))\n", + " blurimageb = cv2.blur(labb, (7,7))\n", + " \n", + " \n", + " window = lab.shape[1]//5\n", + " if window % 2 == 0:\n", + " window += 1\n", + " threshl = 255-cv2.adaptiveThreshold(blurimagel, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 2)\n", " window = lab.shape[1]//3\n", " if window % 2 == 0:\n", " window += 1\n", - " thresh1 = cv2.adaptiveThreshold(blurimage1, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", + " thresha = cv2.adaptiveThreshold(blurimagea, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", " window = lab.shape[1]//10\n", " if window % 2 == 0:\n", " window += 1\n", - " thresh2 = cv2.adaptiveThreshold(blurimage2, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", - " # return thresh1\n", - " imglist.append(thresh1)\n", - " imglist.append(thresh2)\n", - " threshmerge = cv2.bitwise_or(thresh1, thresh2)\n", - " imglist.append(threshmerge)\n", + " threshb = cv2.adaptiveThreshold(blurimageb, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", + " # return threshl\n", + " # imglist.append(threshl)\n", + " # imglist.append(thresha)\n", + " # imglist.append(threshb)\n", + " # threshmerge = cv2.bitwise_or(thresha, threshb)\n", + " # imglist.append(threshmerge)\n", + " # threshmerge = cv2.bitwise_or(threshl, threshmerge)\n", + " # imglist.append(threshmerge)\n", + " # return imglist\n", " # return threshmerge\n", + " threshmerge = threshl\n", " \n", " dim = int(min(threshmerge.shape[0], threshmerge.shape[1])/100)\n", " # dim = 2\n", " # dim = dotsize\n", " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", " \n", - " padthresh = mf.padWithColour(threshmerge, sdim, sdim, fill=255)\n", + " padthresh = mf.padWithColour(threshmerge, sdim*2, sdim*2, fill=255)\n", " mask1t = cv2.morphologyEx(padthresh, cv2.MORPH_OPEN, kernel, iterations=1)\n", " # mask1t = cv2.morphologyEx(mask1t, cv2.MORPH_OPEN, skernel)\n", - " imglist.append(mask1t)\n", + " # imglist.append(mask1t)\n", " # return mask1t\n", " \n", " contours, heirarchy = cv2.findContours(255-mask1t, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", @@ -174,27 +320,77 @@ " \n", " blank = np.full(threshmerge.shape, 255, dtype=np.uint8)\n", " mask1 = blank.copy()\n", - " mask1 = mf.padWithColour(mask1, sdim, sdim, fill=255)\n", + " mask1 = mf.padWithColour(mask1, sdim*2, sdim*2, fill=255)\n", " mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", " # temp = cv2.drawContours(image, [biggestcontour], -1, 255, thickness=3)\n", " # imglist.append(temp)\n", - " imglist.append(mask1)\n", + " # imglist.append(mask1)\n", " # return mask1\n", " \n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel)\n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=1)\n", + " # return mask1\n", " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", - " mask1 = mask1[sdim:-sdim, sdim:-sdim]\n", - " imglist.append(mask1)\n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel, iterations=1)\n", + " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", + " mask1 = mask1[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " # imglist.append(mask1)\n", " # return mask1\n", " \n", - " maskc = cv2.cvtColor(mask1, cv2.COLOR_GRAY2BGR)\n", - " whitedbackground = cv2.bitwise_or(image, maskc)\n", - " imglist.append(whitedbackground)\n", + " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_DILATE, kernel)\n", + " \n", + " # resizemask = cv2.resize(mask1, (ogshape[1], ogshape[0]))\n", + " resizemask = mask1\n", + " # return resizemask\n", + " maskc = cv2.cvtColor(resizemask, cv2.COLOR_GRAY2BGR)\n", + " # print(maskc.shape)\n", + " # print(image.shape)\n", + " whitedbackground = cv2.bitwise_or(shrunkimg, maskc)\n", + " # imglist.append(whitedbackground)\n", + " # return whitedbackground\n", + " \n", + " test = cv2.inpaint(whitedbackground, resizemask, 3, cv2.INPAINT_TELEA)\n", + " # return test\n", + " # if (test.shape[1] > test.shape[0]):\n", + " # shrunktest = mf.ResizeWithAspectRatio(image, width=shrunkdim)\n", + " # else:\n", + " # shrunktest = mf.ResizeWithAspectRatio(image, height=shrunkdim)\n", + " \n", + " sdim = int(min(mainimage.shape[0], mainimage.shape[1])/40)\n", + " srkernel = cv2.getStructuringElement(cv2.MORPH_RECT, (sdim, sdim))\n", + " skernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (sdim, sdim))\n", + " \n", + " \n", + " lab2 = cv2.cvtColor(test, cv2.COLOR_BGR2LAB)\n", + " lab2l = lab2[:,:,0]\n", + " lab2a = lab2[:,:,1]\n", + " lab2b = lab2[:,:,2]\n", + " blurimage2l = cv2.blur(lab2l, (3,3))\n", + " # blurimage2l = lab2l\n", + " \n", + " # window = lab.shape[1]//10\n", + " # if window % 2 == 0:\n", + " # window += 1\n", + " # thresh2l = 255-cv2.adaptiveThreshold(blurimage2l, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", + " thresh2l = cv2.threshold(blurimage2l, 0, 255, cv2.THRESH_OTSU)[1]\n", + " \n", + " morphed2l = 255-cv2.morphologyEx(255-thresh2l, cv2.MORPH_OPEN, kernel, iterations=3)\n", + " padded2l = mf.padWithColour(morphed2l, sdim*2, sdim*2, fill=0)\n", + " \n", + " mask2 = cv2.morphologyEx(padded2l, cv2.MORPH_OPEN, srkernel)\n", + " # mask2 = padded2l\n", + " \n", + " mask2 = 255-mask2[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " \n", + " ##IT'S GOOD HERE. NOW, I CAN IMPLEMENT THE SHRINKING AGAIN, ADD PADDING, MORPHOLOGY TO CONNECT THE SHIT TO THE EDGE, AND THEN GET THE BIGGEST CONTOUR AGAIN\n", + " ## shrinking part \"implemented\" (it wasn't expanded to begin with), the padding and morphology+ still needs to be done\n", + " # return mask2\n", + "\n", + " mask2c = cv2.cvtColor(mask2, cv2.COLOR_GRAY2BGR)\n", + " whitedbackground = cv2.bitwise_or(test, mask2c)\n", " return whitedbackground\n", - " \n", - " \n", + "\n", + " \n", " return imglist\n", " \n", " \n", @@ -459,7 +655,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 1901, "metadata": {}, "outputs": [], "source": [ @@ -479,230 +675,13 @@ }, { "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "def prepimageforhoughline(image, returnrect=True):\n", - " prepped, scaler, hp, vp = mf.squareandthenresize(image, fill=255, width=1000, returnscalerinfo=True)\n", - " ogpreppedshape = prepped.shape\n", - " # print(ogpreppedshape)\n", - " prepped, croprect = mf.premorphCrop(prepped)\n", - " # print(prepped.shape)\n", - " if (prepped.shape[1] > prepped.shape[0]):\n", - " prepped, preppedscaler = mf.ResizeWithAspectRatio(prepped, width=1000, retscale=True)\n", - " else:\n", - " prepped, preppedscaler = mf.ResizeWithAspectRatio(prepped, height=1000, retscale=True)\n", - " # print(prepped.shape)\n", - " # print(preppedscaler)\n", - " finalcroprect = (int(croprect[0]*scaler - hp), int(croprect[1]*scaler - vp), int(croprect[2]*scaler), int(croprect[3]*scaler))\n", - " gray1 = cv2.cvtColor(prepped, cv2.COLOR_BGR2GRAY)\n", - "\n", - " dst1 = cv2.Canny(gray1, 0, 500, None, 3)\n", - "\n", - " \n", - " kernel = np.ones((5,5), np.uint8)\n", - " out = cv2.morphologyEx(dst1, cv2.MORPH_DILATE, kernel)\n", - " out = cv2.blur(out, (5,5))\n", - " kernel = np.ones((6,6), np.uint8)\n", - " dst1 = cv2.morphologyEx(out, cv2.MORPH_ERODE, kernel)\n", - " # return dst1\n", - "\n", - " dst1 = cv2.Canny(dst1, 0, 500, None, 3)\n", - " # return dst1\n", - " accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :]\n", - " if returnrect:\n", - " borderType = cv2.BORDER_CONSTANT\n", - " ## first the og padding when we include the padding added by squaring\n", - " preppadding = [croprect[0], croprect[1], ogpreppedshape[1]-(croprect[0]+croprect[2]), ogpreppedshape[0]-(croprect[1]+croprect[3])]\n", - " # print(croprect)\n", - " # print(dst1.shape)\n", - " # print(preppadding)\n", - " preppadding = [int(s/preppedscaler) for s in preppadding]\n", - " # print(preppadding)\n", - " ## now adjust for any padding. hp and vp are for the full sized image so they need to be scaled down first.\n", - " # adjustedhp = int(hp/scaler)\n", - " # adjustedvp = int(vp/scaler)\n", - " # preppadding[0] -= adjustedhp\n", - " # preppadding[2] -= adjustedhp\n", - " # preppadding[1] -= adjustedvp\n", - " # preppadding[3] -= adjustedvp\n", - " \n", - " paddedprepped = cv2.copyMakeBorder(dst1, preppadding[1], preppadding[3], preppadding[0], preppadding[2], borderType, 0)\n", - " # paddedprepped = dst1\n", - " squaredimage = mf.squarepad(image, fill=0)\n", - " return dst1, accompaniedimage, paddedprepped, squaredimage, finalcroprect\n", - " else:\n", - " # accompaniedimage = squarepad(accompaniedimage, fill=255)\n", - " return dst1, accompaniedimage" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle, croprect):\n", - " # imglist = []\n", - " # print(preppedimage.shape)\n", - " # print(baseimage.shape)\n", - " \n", - " # imgcopy = baseimage.copy()\n", - " # sizemultiplier = croprect[3]/preppedimage.shape[0]\n", - " \n", - " ##adjust the rotation angle if it causes the rectangle to flip w and h size ordering. That is, if it will cause the width to be greater than the height or something. let me think about it for a second more.\n", - " # print(baseimage.shape[:2])\n", - " # mask = np.full(baseimage.shape[:2], fill_value=255, dtype=np.uint8)\n", - " # mask = cv2.rectangle(mask, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=0, thickness=cv2.FILLED)\n", - " # temp = baseimage[croprect[1]:croprect[1]+croprect[3], croprect[0]:croprect[0]+croprect[2], :]\n", - " \n", - " # rotatedmask = mf.rotate(mask, rotationangle, fill=255)\n", - " # print(mask.shape)\n", - " # return mask, 5\n", - " # rotatedmask = mf.rotatewithexactpadding(mask, rotationangle, fill=255)\n", - " rotatedbaseimage = mf.rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n", - " # rotatedtemp = mf.rotatewithexactpadding(temp, rotationangle, fill=(0,0,0))\n", - " # return preppedimage, 5\n", - " rotateddst1 = mf.rotatewithexactpadding(preppedimage, rotationangle, fill=(0,0,0))\n", - " \n", - " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", - " \n", - " # fcr = mf.croptoblack(rotatedmask, extraborder=0, returnrect=True) #finalcroprect\n", - " \n", - " # print(sizemultiplier)\n", - " # imglist.append(rotatedmask)\n", - " # imglist.append(rotatedbaseimage)\n", - " # imglist.append(rotateddst1)\n", - " # return imglist, 5\n", - " # return rotatedmask, 5\n", - " # return rotatedbaseimage, 5\n", - " # return rotateddst1, 5\n", - " # print(fcr)\n", - " # rotatedbaseimage = rotatedbaseimage[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", - " # rotateddst1 = rotateddst1[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", - " # return mask, 5\n", - " # return rotatedbaseimage, 5\n", - " \n", - " \n", - " # sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", - " # print(sizemultiplier)\n", - " # return rotatedbaseimage, rotationangle\n", - "\n", - " croppedbaseimage = mf.houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)\n", - " \n", - " # return croppedbaseimage, rotationangle\n", - "\n", - " finalbaseimage = mf.contourcrop(croppedbaseimage)\n", - "\n", - "\n", - " return finalbaseimage, rotationangle\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " # rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=0)\n", - " # rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n", - " # print(croprect)\n", - " hpad, vpad = mf.determineextrapadding(croprect[3], croprect[2], rotationangle)\n", - " # print(rotationangle)\n", - " # print(hpad, vpad)\n", - " adjustedrect = [croprect[0]-hpad, croprect[1]-vpad, croprect[2]+(2*hpad), croprect[3]+(2*vpad)]\n", - " adjustedrect = [int(s) for s in adjustedrect]\n", - "\n", - " # print(adjustedrect)\n", - " leftpad = abs(min(0, adjustedrect[0]))\n", - " toppad = abs(min(0, adjustedrect[1]))\n", - " rightpad = abs(min(0, baseimage.shape[1]-(adjustedrect[0]+adjustedrect[2])))\n", - " bottompad = abs(min(0, baseimage.shape[0]-(adjustedrect[1]+adjustedrect[3])))\n", - " \n", - " # print(leftpad, rightpad, toppad, bottompad)\n", - " \n", - " borderType = cv2.BORDER_CONSTANT\n", - " # padded = cv2.copyMakeBorder(baseimage, toppad, bottompad, leftpad, rightpad, borderType)\n", - " # imgcopy = padded.copy()\n", - " imgcopy = cv2.rectangle(imgcopy, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=(0,255,0), thickness=3)\n", - " imgcopy = cv2.rectangle(imgcopy, (adjustedrect[0], adjustedrect[1]), (adjustedrect[0]+adjustedrect[2], adjustedrect[1]+adjustedrect[3]), color=(0,0,255), thickness=3)\n", - " \n", - " adjustedrect = [max(0, s) for s in adjustedrect]\n", - " imgcopy = cv2.rectangle(imgcopy, (adjustedrect[0], adjustedrect[1]), (adjustedrect[0]+adjustedrect[2], adjustedrect[1]+adjustedrect[3]), color=(255,0,0), thickness=3)\n", - " return imgcopy, 5\n", - " \n", - "\n", - " ## DETERMINE PADDING AMOUNTS BECAUSE I THEN NEED TO PAD THE ORIGINAL IMAGE, CROP IT TO THE FIRST LEVEL, ROTATE, AND THEN CROP AGAIN\n", - " # print(adjustedrect)\n", - " # adjustedrect = croprect\n", - " # borderType = cv2.BORDER_CONSTANT\n", - " # padded = cv2.copyMakeBorder(baseimage, toppad, bottompad, leftpad, rightpad, borderType)\n", - " croppedpaddedimage = padded[adjustedrect[1]:adjustedrect[1]+adjustedrect[3], adjustedrect[0]:adjustedrect[0]+adjustedrect[2], :]\n", - " # print(croprect[2], croprect[3])\n", - " # print(croppedpaddedimage.shape)\n", - " # croppedog = baseimage[croprect[1]:croprect[1]+croprect[3], croprect[0]:croprect[0]+croprect[2], :]\n", - " # return croppedpaddedimage, 5\n", - " \n", - " rotatedbaseimage = mf.rotate(croppedpaddedimage, rotationangle)\n", - " \n", - " # print(vpad, hpad)\n", - " # print(croprect)\n", - " # rotatedbaseimage = rotatedbaseimage[vpad:vpad+croprect[3], hpad:hpad+croprect[2], :]\n", - " return rotatedbaseimage, 5\n", - " \n", - " \n", - " # print(preppedimage.shape)\n", - " rotateddst1 = mf.rotate(preppedimage, rotationangle)\n", - " # print(rotateddst1.shape)\n", - " # print(rotatedbaseimage.shape)\n", - " # return rotatedbaseimage, 5\n", - "\n", - " \n", - " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", - " # print(sizemultiplier)\n", - " # return rotatedbaseimage, rotationangle\n", - "\n", - " croppedbaseimage = mf.houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)\n", - "\n", - " finalbaseimage = mf.contourcrop(croppedbaseimage)\n", - "\n", - "\n", - " return finalbaseimage, rotationangle" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "def houghlinedeskewandcrop(image):\n", - " croppedcanny, croppedimage, canny, ogimage, rect = prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done\n", - " # return canny, ogimage\n", - " # print(canny.shape)\n", - " # print(croppedogimage.shape)\n", - "\n", - " ## -----------------finding angle to deskew-----------------\n", - " rotationangle = mf.houghlinedeskewangle(croppedcanny)\n", - " # print(rotationangle)\n", - "\n", - " \n", - " # rotatorrect = findcroprectforangle(rect, angle)\n", - "\n", - " # -----------------end of finding angle to deskew-----------------\n", - "\n", - " ## -----------------deskewing and then cropping-----------------\n", - " outimg, angle = houghlinedeskewthencrop(ogimage, canny, rotationangle, rect)\n", - " return outimg, angle" - ] - }, - { - "cell_type": "code", - "execution_count": 11, + "execution_count": 1902, "metadata": {}, "outputs": [], "source": [ "def houghlineprocessing(image):\n", " croppedanddeskewed, angle = mf.houghlinedeskewandcrop(image)\n", - " return croppedanddeskewed\n", + " # return croppedanddeskewed\n", " \n", " \n", " postprocessed = cropclarifying(croppedanddeskewed)\n", @@ -727,7 +706,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 1903, "metadata": {}, "outputs": [], "source": [ @@ -736,7 +715,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 1904, "metadata": {}, "outputs": [], "source": [ @@ -752,7 +731,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 1905, "metadata": {}, "outputs": [], "source": [ @@ -766,7 +745,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 1906, "metadata": {}, "outputs": [], "source": [ @@ -778,7 +757,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 1907, "metadata": {}, "outputs": [], "source": [ @@ -787,7 +766,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 1908, "metadata": {}, "outputs": [], "source": [ @@ -797,7 +776,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 1909, "metadata": {}, "outputs": [], "source": [ @@ -827,14 +806,14 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 1910, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "average time: 0.1962958723306656(s)\n" + "average time: 0.20301511883735657(s)\n" ] } ], @@ -845,7 +824,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 1911, "metadata": {}, "outputs": [], "source": [ @@ -855,7 +834,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 1912, "metadata": {}, "outputs": [], "source": [ @@ -864,7 +843,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 1913, "metadata": {}, "outputs": [], "source": [ From 8cb39fbbf11b338272bc62ffafe54ddcde2d6a50 Mon Sep 17 00:00:00 2001 From: Ethan Wellenreiter Date: Tue, 5 Dec 2023 14:47:02 -0500 Subject: [PATCH 8/8] Temp final point in trying to get a black and white version Have to move on to training and working on the app. have it crop and rotate and will let yolov7 train on that and I would imagine it will work fine. Don't have the time to try figure out the whiteout still. Signed-off-by: Ethan Wellenreiter --- code/autocropper/houghlinedevspace.ipynb | 729 +++++++++-------------- code/autocropper/myfunctions.py | 42 +- 2 files changed, 322 insertions(+), 449 deletions(-) diff --git a/code/autocropper/houghlinedevspace.ipynb b/code/autocropper/houghlinedevspace.ipynb index b74e731..3321130 100644 --- a/code/autocropper/houghlinedevspace.ipynb +++ b/code/autocropper/houghlinedevspace.ipynb @@ -2,9 +2,20 @@ "cells": [ { "cell_type": "code", - "execution_count": 1895, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/torchvision/datapoints/__init__.py:12: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n", + " warnings.warn(_BETA_TRANSFORMS_WARNING)\n", + "/usr/local/lib/python3.10/dist-packages/torchvision/transforms/v2/__init__.py:54: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n", + " warnings.warn(_BETA_TRANSFORMS_WARNING)\n" + ] + } + ], "source": [ "import cv2\n", "import myfunctions as mf\n", @@ -15,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 1896, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -55,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 1897, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -77,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 1898, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -91,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 1899, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 1900, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -152,29 +163,92 @@ " paddedl = mf.padWithColour(threshl, sdim*2, sdim*2, fill=0)\n", " # return paddedl\n", " \n", - " morphedl = paddedl\n", - " morphedl = cv2.morphologyEx(morphedl, cv2.MORPH_OPEN, kernel, iterations=1)\n", " \n", + " # morphedl = 255-cv2.morphologyEx(255-threshl, cv2.MORPH_OPEN, kernel, iterations=3)\n", + " morphedl = paddedl\n", + " # morphedl = cv2.morphologyEx(morphedl, cv2.MORPH_ERODE, kernel, iterations=1)\n", + " morphed1l = cv2.morphologyEx(morphedl, cv2.MORPH_ERODE, kernelell, iterations=1)\n", + "\n", " # return morphedl\n", " \n", - " canny = cv2.Canny(morphedl, 0, 500)\n", + " contours, heirarchy = cv2.findContours(morphed1l, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " biggestcontour = max(contours, key=cv2.contourArea)\n", + " \n", + " \n", + " blank = np.full(labl.shape, 255, dtype=np.uint8)\n", + " mask1 = blank.copy()\n", + " mask1 = mf.padWithColour(mask1, sdim*2, sdim*2, fill=255)\n", + " mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", + " \n", + " \n", + " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_DILATE, kernelell, iterations=2)\n", + " \n", + " \n", + " # mask1 = mask1[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " # return mask1\n", + " \n", + " # morphed2l = mf.padWithColour(morphedl, sdim*2, sdim*2, fill=255)\n", + " morphed2l = cv2.morphologyEx(morphedl, cv2.MORPH_OPEN, kernel, iterations=1)\n", + " # morphed2l = morphed2l[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " \n", + " # return morphed2l\n", + " # print(mask1.shape)\n", + " # print(morphed2l.shape)\n", + " morphed2l = cv2.bitwise_or(morphed2l, 255-mask1)\n", + " # return morphed2l\n", + " \n", + " morphed2l = morphed2l[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " temp_final = cv2.bitwise_or(threshl, 255-morphed2l)\n", + " return temp_final\n", + " \n", + " canny = cv2.Canny(morphed2l, 0, 500)\n", " # return canny\n", "\n", - " minlength = max(mainimage.shape[0], mainimage.shape[1])//7\n", - " mingap = max(mainimage.shape[0], mainimage.shape[1])//20\n", - " linesP = cv2.HoughLinesP(canny, 1, np.pi / 180, 30, None, minlength, mingap)\n", + " vminlength = mainimage.shape[0]//10\n", + " vmaxgap = mainimage.shape[0]//50\n", + " vlinesP = cv2.HoughLinesP(canny, 1, np.pi / 180, 10, None, vminlength, vmaxgap)\n", + " \n", + " hminlength = mainimage.shape[1]//15\n", + " hmaxgap = mainimage.shape[1]//40\n", + " hlinesP = cv2.HoughLinesP(canny, 1, np.pi / 180, 10, None, hminlength, hmaxgap)\n", " # print(linesP)\n", " \n", - " vmarginlines = mf.WithinXDegrees(linesP, 10)\n", - " hmarginlines = mf.WithinXDegrees(linesP, 10, baseangle=90)\n", + " vmarginlines = mf.WithinXDegrees(vlinesP, 15)\n", + " hmarginlines = mf.WithinXDegrees(hlinesP, 15, baseangle=90)\n", " \n", " marginlines = np.append(vmarginlines, hmarginlines, axis=0)\n", + " # marginlines = marginlines.astype(int)\n", + " # # print(marginlines)\n", + " # reshaped = np.reshape(marginlines, (-1,1, 2))\n", + " # # reshaped = cv2.convexHull(reshaped)\n", + " # # print(reshaped)\n", " \n", - " # colourdst = cv2.cvtColor(morphedl, cv2.COLOR_GRAY2BGR)\n", - " # if marginlines is not None:\n", - " # for l in marginlines:\n", - " # cv2.line(colourdst, (int(l[0]), int(l[1])), (int(l[2]), int(l[3])), (0,0,255), 3, cv2.LINE_AA)\n", - " # return colourdst\n", + " \n", + " \n", + " colourdst = cv2.cvtColor(morphedl, cv2.COLOR_GRAY2BGR)\n", + " # out = cv2.drawContours(colourdst, [reshaped], -1, (0,255,0), thickness=3)\n", + " # return out\n", + " \n", + " \n", + " #### NEW IDEA: MERGE THE WHITEOUT BACKGROUND AND TEXT CLARIFICATION STEP BECAUSE DOING THE OTSU THRESHOLD SEEMS TO WORK PRETTY WELL AND IF I JUST WHITE OUT THE OUTER AREA (ACTUALLY WHITE)\n", + " # THEN I HAVE JUST THE TEXT\n", + " \n", + "\n", + " if marginlines is not None:\n", + " for l in marginlines:\n", + " cv2.line(colourdst, (int(l[0]), int(l[1])), (int(l[2]), int(l[3])), (0,0,255), 3, cv2.LINE_AA)\n", + " return colourdst\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " ## IDEA:\n", + " # MASK OUT THE WORDS USING OUR MASKS MADE FROM THE STUFF BELOW. THEN WHEN CANNY IS DONE TO IT, IT SHOULDN'T HAVE A WHOLE BUNCH OF SHIT IN THE CENTER. STILL NEED TO FIGURE OUT HOW TO LINK THE HOUGH LINES AROUND THE RECEIPT\n", + "\n", + "\n", + "\n", + "\n", "\n", " # morphedl = 255-cv2.morphologyEx(255-threshl, cv2.MORPH_OPEN, kernel, iterations=3)\n", " morphedl = paddedl\n", @@ -184,7 +258,8 @@ " # return morphedl\n", " \n", " contours, heirarchy = cv2.findContours(morphedl, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " print(contours)\n", + " # print(contours[0].shape)\n", + " print(contours[0])\n", " biggestcontour = max(contours, key=cv2.contourArea)\n", " return canny\n", " \n", @@ -253,418 +328,161 @@ " \n", " imgout = cv2.drawContours(mainimage, contours, -1, (0,255,0), thickness=3)\n", " return imgout\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "\n", - "\n", - " blurimagel = cv2.blur(labl, (7,7))\n", - " blurimagea = cv2.blur(laba, (7,7))\n", - " blurimageb = cv2.blur(labb, (7,7))\n", - " \n", - " \n", - " window = lab.shape[1]//5\n", - " if window % 2 == 0:\n", - " window += 1\n", - " threshl = 255-cv2.adaptiveThreshold(blurimagel, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 2)\n", - " window = lab.shape[1]//3\n", - " if window % 2 == 0:\n", - " window += 1\n", - " thresha = cv2.adaptiveThreshold(blurimagea, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", - " window = lab.shape[1]//10\n", - " if window % 2 == 0:\n", - " window += 1\n", - " threshb = cv2.adaptiveThreshold(blurimageb, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", - " # return threshl\n", - " # imglist.append(threshl)\n", - " # imglist.append(thresha)\n", - " # imglist.append(threshb)\n", - " # threshmerge = cv2.bitwise_or(thresha, threshb)\n", - " # imglist.append(threshmerge)\n", - " # threshmerge = cv2.bitwise_or(threshl, threshmerge)\n", - " # imglist.append(threshmerge)\n", - " # return imglist\n", - " # return threshmerge\n", - " threshmerge = threshl\n", - " \n", - " dim = int(min(threshmerge.shape[0], threshmerge.shape[1])/100)\n", - " # dim = 2\n", - " # dim = dotsize\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " \n", - " padthresh = mf.padWithColour(threshmerge, sdim*2, sdim*2, fill=255)\n", - " mask1t = cv2.morphologyEx(padthresh, cv2.MORPH_OPEN, kernel, iterations=1)\n", - " # mask1t = cv2.morphologyEx(mask1t, cv2.MORPH_OPEN, skernel)\n", - " # imglist.append(mask1t)\n", - " # return mask1t\n", - " \n", - " contours, heirarchy = cv2.findContours(255-mask1t, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " biggestcontour = max(contours, key=cv2.contourArea)\n", - " \n", - " blank = np.full(threshmerge.shape, 255, dtype=np.uint8)\n", - " mask1 = blank.copy()\n", - " mask1 = mf.padWithColour(mask1, sdim*2, sdim*2, fill=255)\n", - " mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", - " # temp = cv2.drawContours(image, [biggestcontour], -1, 255, thickness=3)\n", - " # imglist.append(temp)\n", - " # imglist.append(mask1)\n", - " # return mask1\n", - " \n", - " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=1)\n", - " # return mask1\n", - " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel, iterations=1)\n", - " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=2)\n", - " mask1 = mask1[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", - " # imglist.append(mask1)\n", - " # return mask1\n", - " \n", - " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_DILATE, kernel)\n", - " \n", - " # resizemask = cv2.resize(mask1, (ogshape[1], ogshape[0]))\n", - " resizemask = mask1\n", - " # return resizemask\n", - " maskc = cv2.cvtColor(resizemask, cv2.COLOR_GRAY2BGR)\n", - " # print(maskc.shape)\n", - " # print(image.shape)\n", - " whitedbackground = cv2.bitwise_or(shrunkimg, maskc)\n", - " # imglist.append(whitedbackground)\n", - " # return whitedbackground\n", - " \n", - " test = cv2.inpaint(whitedbackground, resizemask, 3, cv2.INPAINT_TELEA)\n", - " # return test\n", - " # if (test.shape[1] > test.shape[0]):\n", - " # shrunktest = mf.ResizeWithAspectRatio(image, width=shrunkdim)\n", - " # else:\n", - " # shrunktest = mf.ResizeWithAspectRatio(image, height=shrunkdim)\n", - " \n", - " sdim = int(min(mainimage.shape[0], mainimage.shape[1])/40)\n", - " srkernel = cv2.getStructuringElement(cv2.MORPH_RECT, (sdim, sdim))\n", - " skernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (sdim, sdim))\n", - " \n", - " \n", - " lab2 = cv2.cvtColor(test, cv2.COLOR_BGR2LAB)\n", - " lab2l = lab2[:,:,0]\n", - " lab2a = lab2[:,:,1]\n", - " lab2b = lab2[:,:,2]\n", - " blurimage2l = cv2.blur(lab2l, (3,3))\n", - " # blurimage2l = lab2l\n", - " \n", - " # window = lab.shape[1]//10\n", - " # if window % 2 == 0:\n", - " # window += 1\n", - " # thresh2l = 255-cv2.adaptiveThreshold(blurimage2l, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 0)\n", - " thresh2l = cv2.threshold(blurimage2l, 0, 255, cv2.THRESH_OTSU)[1]\n", - " \n", - " morphed2l = 255-cv2.morphologyEx(255-thresh2l, cv2.MORPH_OPEN, kernel, iterations=3)\n", - " padded2l = mf.padWithColour(morphed2l, sdim*2, sdim*2, fill=0)\n", - " \n", - " mask2 = cv2.morphologyEx(padded2l, cv2.MORPH_OPEN, srkernel)\n", - " # mask2 = padded2l\n", - " \n", - " mask2 = 255-mask2[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", - " \n", - " ##IT'S GOOD HERE. NOW, I CAN IMPLEMENT THE SHRINKING AGAIN, ADD PADDING, MORPHOLOGY TO CONNECT THE SHIT TO THE EDGE, AND THEN GET THE BIGGEST CONTOUR AGAIN\n", - " ## shrinking part \"implemented\" (it wasn't expanded to begin with), the padding and morphology+ still needs to be done\n", - " # return mask2\n", - "\n", - " mask2c = cv2.cvtColor(mask2, cv2.COLOR_GRAY2BGR)\n", - " whitedbackground = cv2.bitwise_or(test, mask2c)\n", - " return whitedbackground\n", - "\n", - " \n", - " return imglist\n", - " \n", - " \n", - " \n", - " imagecpy = image.copy()\n", - " blurimage = cv2.blur(imagecpy, (13,13))\n", - " \n", - "\n", - " \n", - " \n", - " # ogcover = np.full(imagecpy.shape, 0, dtype=np.uint8)\n", - " # ogcover = mf.padWithColour(ogcover, sdim, sdim, fill=(255,255,255))\n", - " # ogcover = cv2.cvtColor(ogcover, cv2.COLOR_BGR2GRAY)\n", - " \n", - " # blurimage = mf.padWithColour(blurimage, sdim, sdim, fill=(255,255,255))\n", - " # imagecpy = cv2.inpaint(blurimage, ogcover, 3, cv2.INPAINT_TELEA)\n", - " # imagecpy = cv2.copyMakeBorder(blurimage, sdim, sdim, sdim, sdim, cv2.BORDER_REPLICATE)\n", - " # return imagecpy\n", - " # imagecpy = mf.padWithColour(image, hpadding=sdim, vpadding=sdim)\n", - " \n", - " lab = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)\n", - " gray = lab[:,:,0]\n", - " # gray = cv2.cvtColor(blurimage, cv2.COLOR_BGR2GRAY)\n", - " \n", - " # blur = cv2.blur(gray, (7,7))\n", - " \n", - " # window = 51\n", - " window = gray.shape[1]//7\n", - " if window % 2 == 0:\n", - " window += 1\n", - " thresh = cv2.adaptiveThreshold(gray, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 2)\n", - " # thresh2 = cv2.threshold(gray, 0, 255, cv2.THRESH_OTSU)[1]\n", - " # thresh = cv2.bitwise_and(thresh1, thresh2)\n", - " # return thresh\n", - " \n", - " # initialcontours, heirarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " \n", - " # contourrects = np.array([list(cv2.boundingRect(contour)) for contour in initialcontours])\n", - " # rectdims = contourrects[:,2:]\n", - " # flatrectdims = rectdims.flatten()\n", - " # dotsize = int(np.median(flatrectdims)//2)\n", - " # # print(contourrects)\n", - " # # print(rectdims)\n", - " # # print(dotsize)\n", - " \n", - " \n", - "\n", - " # dim = int(min(thresh.shape[0], thresh.shape[1])/400)\n", - " dim = 2\n", - " # dim = dotsize\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " morphedthresh = cv2.morphologyEx(thresh, cv2.MORPH_ERODE, kernel)\n", - " return morphedthresh\n", - " \n", - " \n", - " contours, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " biggestcontour = max(contours, key=cv2.contourArea)\n", - " \n", - " \n", - " # epsilon = 0.0005*cv2.arcLength(biggestcontour,True)\n", - " # approx = cv2.approxPolyDP(biggestcontour,epsilon,True)\n", - " \n", - " blank = np.full(thresh.shape, 255, dtype=np.uint8)\n", - " mask1 = 255 - blank.copy()\n", - " mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 255, thickness=cv2.FILLED)\n", - " \n", - " mask1 = mf.padWithColour(mask1, sdim, sdim, fill=0)\n", - " \n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_CLOSE, skernel, iterations=1)\n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel, iterations=1)\n", - " # mask1 = cv2.morphologyEx(mask1, cv2.MORPH_OPEN, skernel)\n", - " # return mask1\n", - " \n", - " \n", - " mask1contours, heirarchy = cv2.findContours(mask1, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " \n", - " biggestm1contour = max(mask1contours, key=cv2.contourArea)\n", - " \n", - " mask1_1 = 255-blank.copy()\n", - " mask1_1 = mf.padWithColour(mask1_1, sdim, sdim, fill=0)\n", - " mask1_1 = cv2.drawContours(mask1_1, [biggestm1contour], -1, 255, thickness=cv2.FILLED)\n", - " mask1_1 = 255-mask1_1\n", - " \n", - " dim = max(2,int(min(image.shape[0], image.shape[1])/100))\n", - " # print(dim)\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " mask1_1 = cv2.morphologyEx(mask1_1, cv2.MORPH_DILATE, kernel, iterations=1)\n", - " mask1_1 = mask1_1[sdim:-sdim, sdim:-sdim]\n", - " \n", - " # epsilon = 0.01*cv2.arcLength(biggestm1contour,True)\n", - " # approxm1 = cv2.approxPolyDP(biggestm1contour,epsilon,True)\n", - " \n", - " # hull = cv2.convexHull(biggestcontour)\n", - " # hull = cv2.convexHull(approx)\n", - " # mask2 = 255 - blank.copy()\n", - " # mask2 = cv2.drawContours(mask2, [hull], -1, 255, thickness=cv2.FILLED)\n", - " \n", - " # imagecpy = cv2.drawContours(imagecpy, [biggestcontour], -1, (255,0,0), thickness=3)\n", - " # # imagecpy = cv2.drawContours(imagecpy, [approx], -1, (0,255,0), thickness=3)\n", - " # # imagecpy = cv2.drawContours(imagecpy, [hull], -1, (0,255,255), thickness=3)\n", - " # imagecpy = cv2.drawContours(imagecpy, [biggestm1contour], -1, (0,0,255), thickness=3)\n", - " # # imagecpy = cv2.drawContours(imagecpy, [approxm1], -1, (255,0,255), thickness=3)\n", - " # return imagecpy\n", - " \n", - " # mask = 255 - cv2.bitwise_and(mask1, mask2)\n", - " # return mask1_1\n", - " \n", - " maskc = cv2.cvtColor(mask1_1, cv2.COLOR_GRAY2BGR)\n", - " # return maskc\n", - " # return blurimage\n", - " # print(blurimage)\n", - " # print(maskc)\n", - " whitedbackground = cv2.bitwise_or(image, maskc)\n", - " return whitedbackground\n", - "\n", - "\n", - "\n", - "\n", - "##### NEED TO FIX THE CURVE OF CONNECTING TO THE EDGES DUE TO THE ELLIPSE KERNEL\n", - " \n", - " graywhitedbackground = cv2.cvtColor(whitedbackground, cv2.COLOR_BGR2GRAY)\n", - " # thresh2 = cv2.threshold(graywhitedbackground, 0, 255, cv2.THRESH_OTSU)[1]\n", - " blurredgraywhitedbackground = cv2.blur(graywhitedbackground, (11,11))\n", - " # blurredgraywhitedbackground = graywhitedbackground\n", - " window = blurredgraywhitedbackground.shape[1]//15\n", - " if window % 2 == 0:\n", - " window += 1\n", - " thresh2 = cv2.adaptiveThreshold(blurredgraywhitedbackground, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 2)\n", - " # return thresh2\n", - " \n", - " paddedthresh2 = mf.padWithColour(thresh2, sdim, sdim, fill=255)\n", - " \n", - " \n", - " dim = max(2,int(min(image.shape[0], image.shape[1])/50))\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " t2morph = cv2.morphologyEx(paddedthresh2, cv2.MORPH_OPEN, kernel, iterations=2)\n", - " dim = int(window*1.5)\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " t2morph = cv2.morphologyEx(t2morph, cv2.MORPH_CLOSE, kernel, iterations=1)\n", - " \n", - " t2morph = t2morph[sdim:-sdim, sdim:-sdim]\n", - " # return t2morph\n", - "\n", - " # print(mask1_1.shape)\n", - " # print(t2morph.shape)\n", - " mask2 = 255-cv2.bitwise_or(mask1_1, 255-t2morph)\n", - " # return mask2\n", - " \n", - " \n", - " \n", - " mask2contours, heirarchy = cv2.findContours(mask2, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " \n", - " biggestm2contour = max(mask2contours, key=cv2.contourArea)\n", - " \n", - " mask2_1 = 255 - blank.copy()\n", - " mask2_1 = cv2.drawContours(mask2_1, [biggestm2contour], -1, 255, thickness=cv2.FILLED)\n", - " \n", - " # return mask2_1\n", - " \n", - " mask2_1 = mf.padWithColour(mask2_1, sdim, sdim, fill=0)\n", - " \n", - " mask2_2 = cv2.morphologyEx(mask2_1, cv2.MORPH_OPEN, kernel)\n", - " \n", - " mask2_2 = 255-mask2_2[sdim:-sdim, sdim:-sdim]\n", - " \n", - " # return mask2_2\n", - "\n", - " mask2c = cv2.cvtColor(mask2_2, cv2.COLOR_GRAY2BGR)\n", - " # return maskc\n", - " # return blurimage\n", - " # print(blurimage)\n", - " # print(maskc)\n", - " whitedbackground2 = cv2.bitwise_or(image, mask2c)\n", - " return whitedbackground2\n", - " \n", - " \n", - " \n", - " \n", - " canny = cv2.Canny(graywhitedbackground, 0, 500)\n", - " return canny\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " contours1, heirarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " contours2, heirarchy = cv2.findContours(morphedthresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " \n", - "\n", - "\n", - " biggestcontour2 = max(contours1, key=cv2.contourArea)\n", - " biggestcontour1 = max(contours2, key=cv2.contourArea)\n", - " \n", - " epsilon = 0.0005*cv2.arcLength(biggestcontour1,True)\n", - " approx = cv2.approxPolyDP(biggestcontour1,epsilon,True)\n", - " # approx = cv2.convexHull(approx)\n", - " epsilon = 0.005*cv2.arcLength(approx,True)\n", - " approx = cv2.approxPolyDP(approx,epsilon,True)\n", - " # approx = cv2.convexHull(biggestcontour1)\n", - " # print(approx)\n", - " \n", - " imagecpy = cv2.drawContours(imagecpy, [biggestcontour1], -1, (0,255,0), thickness=3)\n", - " imagecpy = cv2.drawContours(imagecpy, [biggestcontour2], -1, (0,0,255), thickness=3)\n", - " \n", - " imagecpy = cv2.drawContours(imagecpy, [approx], -1, (255,0,0), thickness=3)\n", - " # return imagecpy\n", - " \n", - " blank = np.full(thresh.shape, 255, dtype=np.uint8)\n", - " mask = blank.copy()\n", - " mask = cv2.drawContours(mask, [biggestcontour1], -1, (0,0,0), thickness=cv2.FILLED)\n", - " # mask = cv2.drawContours(mask, [approx], -1, (0,0,0), thickness=cv2.FILLED)\n", - " # mask = cv2.drawContours(mask, [biggestcontour2], -1, (0,0,0), thickness=cv2.FILLED)\n", - "\n", - " # return mask\n", - "\n", - " invertmask = 255 - mask\n", - " \n", - " \n", - " dim = int(min(invertmask.shape[0], invertmask.shape[1])/200)\n", - " # # dim = 21\n", - " # print(dim)\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " # invertmask = cv2.morphologyEx(invertmask, cv2.MORPH_DILATE, kernel)\n", - " mask = 255 - cv2.morphologyEx(invertmask, cv2.MORPH_ERODE, kernel, iterations=1)\n", - " return mask\n", - " \n", - " maskcontours, heirarchy = cv2.findContours(255-mask, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", - " \n", - " mainmaskcontour = max(maskcontours, key=cv2.contourArea)\n", - " mask1 = blank.copy()\n", - " mask1 = cv2.drawContours(mask1, [mainmaskcontour], -1, (0,0,0), thickness=cv2.FILLED)\n", - " mask1 = cv2.morphologyEx(mask1, cv2.MORPH_ERODE, kernel, iterations=1)\n", - " # temp = cv2.drawContours(image, [mainmaskcontour], -1, (0,255,0), thickness=3)\n", - " # return temp\n", - " \n", - " mask1c = cv2.cvtColor(mask1, cv2.COLOR_GRAY2BGR)\n", - " whitedbackground = cv2.bitwise_or(image, mask1c)\n", - " # return whitedbackground\n", - " \n", - " mask2 = blank.copy()\n", - " mask2 = 255-cv2.drawContours(mask2, [approx], -1, (0,0,0), thickness=cv2.FILLED)\n", - " \n", - " dim = int(min(mask2.shape[0], mask2.shape[1])/50)\n", - " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", - " morphedmask = 255-cv2.morphologyEx(mask2, cv2.MORPH_OPEN, kernel, iterations=7)\n", - " # return morphedmask\n", - " \n", - " finalmask = cv2.bitwise_or(mask1, morphedmask)\n", - " \n", - " \n", - " finalmaskbgr = cv2.cvtColor(finalmask, cv2.COLOR_GRAY2BGR)\n", - " # return finalmaskbgr\n", - "\n", - " whitedbackground = cv2.bitwise_or(whitedbackground, finalmaskbgr)\n", - " # return whitedbackground\n", - " \n", - " test = cv2.inpaint(whitedbackground, finalmask, 3, cv2.INPAINT_TELEA)\n", - " return test" + " \n" ] }, { "cell_type": "code", - "execution_count": 1901, + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "def textleaver(image):\n", + " ogshape = image.shape\n", + " shrunkdim=1000\n", + " if (image.shape[1] > image.shape[0]):\n", + " shrunkimg, scaler = mf.ResizeWithAspectRatio(image, width=shrunkdim, retscale=True)\n", + " else:\n", + " shrunkimg, scaler = mf.ResizeWithAspectRatio(image, height=shrunkdim, retscale=True)\n", + " \n", + " mainimage = shrunkimg\n", + " \n", + " sdim = int(min(mainimage.shape[0], mainimage.shape[1])/5)\n", + " srkernel = cv2.getStructuringElement(cv2.MORPH_RECT, (sdim, sdim))\n", + " skernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (sdim, sdim))\n", + " \n", + " oglab = cv2.cvtColor(image, cv2.COLOR_BGR2LAB)\n", + " lab = cv2.cvtColor(mainimage, cv2.COLOR_BGR2LAB)\n", + " \n", + " imglist = []\n", + " # imglist.append(mainimage)\n", + " \n", + " labl = lab[:,:,0]\n", + " oglabl = oglab[:,:,0]\n", + " # # imglist.append(labl)\n", + " # # imglist.append(cv2.cvtColor(image, cv2.COLOR_BGR2GRAY))\n", + " # laba = lab[:,:,1]\n", + " # # imglist.append(laba)\n", + " # labb = lab[:,:,2]\n", + " # # imglist.append(labb)\n", + " \n", + " divisor = 1.5\n", + " window = int(min(labl.shape)/divisor)\n", + " window = window if window%2 == 1 else window + 1\n", + " # canny = cv2.Canny(labl, 0, 500)\n", + " ethreshl = cv2.threshold(labl, 0, 255, cv2.THRESH_OTSU)[1]\n", + " threshl = cv2.threshold(labl, 0, 255, cv2.THRESH_OTSU)[1]\n", + " # threshl = cv2.adaptiveThreshold(labl, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, window, 35)\n", + " \n", + " \n", + " ogwindow = int(min(oglabl.shape)/divisor)\n", + " ogwindow = window if window%2 == 1 else window + 1\n", + " print(ogwindow)\n", + " ogthreshl = cv2.threshold(oglabl, 0, 255, cv2.THRESH_TRIANGLE)[1]\n", + " return ogthreshl\n", + " # ogthreshl = cv2.adaptiveThreshold(oglabl, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, ogwindow, 35)\n", + " # return threshl\n", + " \n", + " colourthresh = cv2.cvtColor(threshl, cv2.COLOR_GRAY2BGR)\n", + " \n", + " dim = int(min(mainimage.shape[0], mainimage.shape[1])/100)\n", + " # dim = 2\n", + " # dim = dotsize\n", + " dim = max(3,dim)\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (dim, dim))\n", + " kernelell = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dim, dim))\n", + " \n", + " # paddedl = mf.padWithColour(threshl, sdim*2, sdim*2, fill=0)\n", + " paddedl = threshl\n", + " # return paddedl\n", + " \n", + " \n", + " # morphedl = 255-cv2.morphologyEx(255-threshl, cv2.MORPH_OPEN, kernel, iterations=3)\n", + " morphedl = paddedl\n", + " morphed1l = cv2.morphologyEx(morphedl, cv2.MORPH_ERODE, kernel, iterations=1)\n", + " # morphed1l = cv2.morphologyEx(morphed1l, cv2.MORPH_OPEN, kernel, iterations=1)\n", + " # morphed1l = cv2.morphologyEx(morphed1l, cv2.MORPH_OPEN, kernel, iterations=1)\n", + " # morphed1l = cv2.morphologyEx(morphedl, cv2.MORPH_ERODE, kernelell, iterations=2)\n", + " \n", + " emorphed1l = cv2.morphologyEx(ethreshl, cv2.MORPH_ERODE, kernel, iterations=1)\n", + "\n", + " # return morphedl\n", + " \n", + " contours, heirarchy = cv2.findContours(morphed1l, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " biggestcontour = max(contours, key=cv2.contourArea)\n", + " \n", + " # temp = cv2.drawContours(colourthresh, [biggestcontour], -1, (0,255,0), thickness=1)\n", + " # return temp\n", + " \n", + " \n", + " blank = np.full(labl.shape, 255, dtype=np.uint8)\n", + " mask1 = blank.copy()\n", + " # mask1 = mf.padWithColour(mask1, sdim*2, sdim*2, fill=255)\n", + " mask1 = cv2.drawContours(mask1, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", + " ## need to change the erosion so that if the paper goes to the edge, it doesn't get eroded in (because that means the paper is right to the edge and writing may be close)\n", + " \n", + " contours, heirarchy = cv2.findContours(morphed1l, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)\n", + " biggestcontour = max(contours, key=cv2.contourArea)\n", + " \n", + " emask1 = blank.copy()\n", + " emask1 = cv2.drawContours(emask1, [biggestcontour], -1, 0, thickness=cv2.FILLED)\n", + " \n", + " mask1 = 255-cv2.morphologyEx(255-mask1, cv2.MORPH_ERODE, kernel, iterations=2)\n", + " \n", + " emask1 = 255-cv2.morphologyEx(255-emask1, cv2.MORPH_ERODE, kernel, iterations=2)\n", + " \n", + " \n", + " # mask1 = mask1[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " # return mask1\n", + " \n", + " # morphed2l = mf.padWithColour(morphedl, sdim*2, sdim*2, fill=255)\n", + " morphed2l = cv2.morphologyEx(morphedl, cv2.MORPH_OPEN, kernel, iterations=1)\n", + " morphed2l = cv2.morphologyEx(morphedl, cv2.MORPH_ERODE, kernel, iterations=1)\n", + " # morphed2l = morphed2l[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " \n", + " # return morphed2l\n", + " # print(mask1.shape)\n", + " # print(morphed2l.shape)\n", + " morphed2l = cv2.bitwise_or(morphed2l, 255-mask1)\n", + " # return morphed2l\n", + "\n", + " # paddedthreshl = mf.padWithColour(morphed2l, sdim*2, sdim*2, fill=255)\n", + " # temp = cv2.drawContours(colourthresh, [biggestcontour], -1, (0,255,0), thickness=1)\n", + " # return temp\n", + "\n", + "\n", + " morphed2l = cv2.morphologyEx(morphed2l, cv2.MORPH_ERODE, kernel, iterations=1)\n", + " morphed2l = cv2.morphologyEx(morphed2l, cv2.MORPH_ERODE, kernelell, iterations=1)\n", + " # return morphed2l\n", + " # morphed2l = cv2.bitwise_or(morphed2l, 255-emask1)\n", + " \n", + " # morphed2l = morphed2l[(sdim*2):-(sdim*2), (sdim*2):-(sdim*2)]\n", + " \n", + " resizedmask = cv2.resize(255-morphed2l, (ogshape[1], ogshape[0]))\n", + " temp_final = cv2.bitwise_or(ogthreshl, resizedmask)\n", + " \n", + " dim=3\n", + " kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (dim, dim))\n", + " temp_final = cv2.morphologyEx(temp_final, cv2.MORPH_OPEN, kernel)\n", + " temp_final = cv2.morphologyEx(temp_final, cv2.MORPH_OPEN, kernel)\n", + " # temp_final = cv2.morphologyEx(temp_final, cv2.MORPH_CLOSE, kernel)\n", + " # temp_final = cv2.morphologyEx(temp_final, cv2.MORPH_OPEN, kernel)\n", + " return temp_final" + ] + }, + { + "cell_type": "code", + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "def cropclarifying(image):\n", - " whitedbackground = whiteoutbackground(image)\n", - " return whitedbackground\n", + " # whitedbackground = whiteoutbackground(image)\n", + " # return whitedbackground\n", "\n", - " textrefined = mf.textClarifying(whitedbackground)\n", - " # return textrefined\n", + " # textrefined = mf.textClarifying(whitedbackground)\n", + " textrefined = textleaver(image)\n", + " return textrefined\n", " #maybe now is when I put in the line removing function\n", "\n", " lineout = mf.removeLinesFromText(textrefined)\n", @@ -675,7 +493,7 @@ }, { "cell_type": "code", - "execution_count": 1902, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -684,15 +502,18 @@ " # return croppedanddeskewed\n", " \n", " \n", - " postprocessed = cropclarifying(croppedanddeskewed)\n", - " return postprocessed\n", - " postprocessed = mf.croptoblack(postprocessed)\n", + " # postprocessed = cropclarifying(croppedanddeskewed)\n", + " postprocessed = croppedanddeskewed\n", + " # return postprocessed\n", + " # postprocessed = mf.croptoblack(postprocessed)\n", " \n", " # postprocessed = cv2.cvtColor(postprocessed, cv2.COLOR_GRAY2BGR)\n", " # return postprocessed\n", " \n", " # final = mf.externaldeskew(postprocessed, fill=(255,255,255))\n", - " final = mf.receipttextdeskew(postprocessed, fill=(255,255,255))\n", + " # rotangle = mf.receipttextdeskew(postprocessed, fill=(255,255,255), returnangle=True)\n", + " final = postprocessed\n", + " \n", " \n", " # final = mf.croptoblack(final)\n", " \n", @@ -706,7 +527,7 @@ }, { "cell_type": "code", - "execution_count": 1903, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -715,9 +536,17 @@ }, { "cell_type": "code", - "execution_count": 1904, + "execution_count": 11, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.0\n" + ] + } + ], "source": [ "# prepped, scaler, hp, vp = mf.squareandthenresize(img, fill=255, width=1000, returnscalerinfo=True)\n", "outs = houghlineprocessing(img)\n", @@ -731,7 +560,7 @@ }, { "cell_type": "code", - "execution_count": 1905, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -745,7 +574,7 @@ }, { "cell_type": "code", - "execution_count": 1906, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -757,7 +586,7 @@ }, { "cell_type": "code", - "execution_count": 1907, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -766,7 +595,7 @@ }, { "cell_type": "code", - "execution_count": 1908, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -776,7 +605,7 @@ }, { "cell_type": "code", - "execution_count": 1909, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -806,14 +635,30 @@ }, { "cell_type": "code", - "execution_count": 1910, + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "average time: 0.20301511883735657(s)\n" + "0.9740282517223996\n", + "-2.0053522829578814\n", + "-0.9740282517223996\n", + "0.0\n", + "0.9740282517223996\n", + "-0.9740282517223996\n", + "-0.011669615052326776\n", + "2.0053522829578814\n", + "0.0\n", + "0.0\n", + "0.0\n", + "-2.979380534680281\n", + "0.0\n", + "0.0\n", + "-2.0053522829578814\n", + "-11.000789666511807\n", + "average time: 0.19967518746852875(s)\n" ] } ], @@ -824,7 +669,7 @@ }, { "cell_type": "code", - "execution_count": 1911, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -834,7 +679,7 @@ }, { "cell_type": "code", - "execution_count": 1912, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ @@ -843,7 +688,7 @@ }, { "cell_type": "code", - "execution_count": 1913, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ diff --git a/code/autocropper/myfunctions.py b/code/autocropper/myfunctions.py index fddf559..491be81 100644 --- a/code/autocropper/myfunctions.py +++ b/code/autocropper/myfunctions.py @@ -507,6 +507,19 @@ def houghlinedeskewandcrop(image): ## -----------------finding angle to deskew----------------- rotationangle = houghlinedeskewangle(croppedcanny) + # print(croppedcanny.shape) + # print(abs(rotationangle)) + if (croppedcanny.shape[0] > croppedcanny.shape[1]): + if (rotationangle > 45): + rotationangle -= 90 + elif rotationangle < -45: + rotationangle += 90 + # print(rotationangle) + # elif (croppedcanny.shape[1] > croppedcanny.shape[0]): + # if (rotationangle > 45): + # rotationangle -= 90 + # elif rotationangle < -45: + # rotationangle += 90 # print(rotationangle) @@ -1117,26 +1130,41 @@ def getreceipttextAngle(cvImage) -> float: # print(angle) return angle -def receipttextdeskew(img, fill=(0,0,0)): +def receipttextdeskew(img, fill=(0,0,0), returnangle=False): colourimg = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) angle = getreceipttextAngle(colourimg) - padimg = padWithColour(img, hpadding=50, vpadding=50, fill=fill) - rotated = rotate(padimg, angle, fill=fill) + if returnangle: + return angle + # padimg = padWithColour(img, hpadding=50, vpadding=50, fill=fill) + # print(img.shape) + # grayfill = int((fill[0]*0.299) + (fill[1]*0.587) + (fill[2]*0.114)) + rotated = rotatewithexactpadding(colourimg, angle, fill=fill) + grayrotated = cv2.cvtColor(rotated, cv2.COLOR_BGR2GRAY) + # print(grayrotated) + croprect = croptoblack(grayrotated, returnrect=True) + # rotated = cv2.cvtColor(rotated, cv2.COLOR_GRAY2BGR) + rotated = rotated[croprect[1]:croprect[1]+croprect[3], croprect[0]:croprect[0]+croprect[2], :] + rotated = padWithColour(rotated, hpadding=50, vpadding=50, fill=fill) return rotated ## ------------------------------Full deskewing and cropping------------------------------ def houghlineprocessing(image): - croppedanddeskewed, angle = houghlinedeskewandcrop(image) + croppedanddeskewed, angle = mf.houghlinedeskewandcrop(image) + # return croppedanddeskewed - postprocessed = cropclarifying(croppedanddeskewed) + + # postprocessed = cropclarifying(croppedanddeskewed) + postprocessed = croppedanddeskewed # return postprocessed - postprocessed = croptoblack(postprocessed) + # postprocessed = mf.croptoblack(postprocessed) # postprocessed = cv2.cvtColor(postprocessed, cv2.COLOR_GRAY2BGR) # return postprocessed # final = mf.externaldeskew(postprocessed, fill=(255,255,255)) - final = receipttextdeskew(postprocessed, fill=(255,255,255)) + # rotangle = mf.receipttextdeskew(postprocessed, fill=(255,255,255), returnangle=True) + final = postprocessed + # final = mf.croptoblack(final)