From d9aa07121acbe598679070cb932a09d06298e7d6 Mon Sep 17 00:00:00 2001 From: Ethan Wellenreiter Date: Mon, 27 Nov 2023 15:54:23 -0500 Subject: [PATCH] Fixed the houghline cropping and deskewing for no black rotation lines Adjusted the process so that the black lines left after the houghline cropping and rotating are no longer there. Signed-off-by: Ethan Wellenreiter --- code/autocropper/houghlinedevspace.ipynb | 184 +++++++++++++++++------ code/autocropper/myfunctions.py | 53 +++---- 2 files changed, 159 insertions(+), 78 deletions(-) diff --git a/code/autocropper/houghlinedevspace.ipynb b/code/autocropper/houghlinedevspace.ipynb index 6eccd36..c36a482 100644 --- a/code/autocropper/houghlinedevspace.ipynb +++ b/code/autocropper/houghlinedevspace.ipynb @@ -2,9 +2,20 @@ "cells": [ { "cell_type": "code", - "execution_count": 123, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/torchvision/datapoints/__init__.py:12: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n", + " warnings.warn(_BETA_TRANSFORMS_WARNING)\n", + "/usr/local/lib/python3.10/dist-packages/torchvision/transforms/v2/__init__.py:54: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n", + " warnings.warn(_BETA_TRANSFORMS_WARNING)\n" + ] + } + ], "source": [ "import cv2\n", "import myfunctions as mf\n", @@ -15,7 +26,7 @@ }, { "cell_type": "code", - "execution_count": 124, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -55,7 +66,7 @@ }, { "cell_type": "code", - "execution_count": 125, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ @@ -77,7 +88,7 @@ }, { "cell_type": "code", - "execution_count": 126, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -91,7 +102,7 @@ }, { "cell_type": "code", - "execution_count": 127, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -102,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 128, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ @@ -448,7 +459,7 @@ }, { "cell_type": "code", - "execution_count": 129, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -468,47 +479,121 @@ }, { "cell_type": "code", - "execution_count": 130, + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def prepimageforhoughline(image, returnrect=True):\n", + " prepped, scaler, hp, vp = mf.squareandthenresize(image, fill=255, width=1000, returnscalerinfo=True)\n", + " ogpreppedshape = prepped.shape\n", + " # print(ogpreppedshape)\n", + " prepped, croprect = mf.premorphCrop(prepped)\n", + " # print(prepped.shape)\n", + " if (prepped.shape[1] > prepped.shape[0]):\n", + " prepped, preppedscaler = mf.ResizeWithAspectRatio(prepped, width=1000, retscale=True)\n", + " else:\n", + " prepped, preppedscaler = mf.ResizeWithAspectRatio(prepped, height=1000, retscale=True)\n", + " # print(prepped.shape)\n", + " # print(preppedscaler)\n", + " finalcroprect = (int(croprect[0]*scaler - hp), int(croprect[1]*scaler - vp), int(croprect[2]*scaler), int(croprect[3]*scaler))\n", + " gray1 = cv2.cvtColor(prepped, cv2.COLOR_BGR2GRAY)\n", + "\n", + " dst1 = cv2.Canny(gray1, 0, 500, None, 3)\n", + "\n", + " \n", + " kernel = np.ones((5,5), np.uint8)\n", + " out = cv2.morphologyEx(dst1, cv2.MORPH_DILATE, kernel)\n", + " out = cv2.blur(out, (5,5))\n", + " kernel = np.ones((6,6), np.uint8)\n", + " dst1 = cv2.morphologyEx(out, cv2.MORPH_ERODE, kernel)\n", + " # return dst1\n", + "\n", + " dst1 = cv2.Canny(dst1, 0, 500, None, 3)\n", + " # return dst1\n", + " accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :]\n", + " if returnrect:\n", + " borderType = cv2.BORDER_CONSTANT\n", + " ## first the og padding when we include the padding added by squaring\n", + " preppadding = [croprect[0], croprect[1], ogpreppedshape[1]-(croprect[0]+croprect[2]), ogpreppedshape[0]-(croprect[1]+croprect[3])]\n", + " # print(croprect)\n", + " # print(dst1.shape)\n", + " # print(preppadding)\n", + " preppadding = [int(s/preppedscaler) for s in preppadding]\n", + " # print(preppadding)\n", + " ## now adjust for any padding. hp and vp are for the full sized image so they need to be scaled down first.\n", + " # adjustedhp = int(hp/scaler)\n", + " # adjustedvp = int(vp/scaler)\n", + " # preppadding[0] -= adjustedhp\n", + " # preppadding[2] -= adjustedhp\n", + " # preppadding[1] -= adjustedvp\n", + " # preppadding[3] -= adjustedvp\n", + " \n", + " paddedprepped = cv2.copyMakeBorder(dst1, preppadding[1], preppadding[3], preppadding[0], preppadding[2], borderType, 0)\n", + " # paddedprepped = dst1\n", + " squaredimage = mf.squarepad(image, fill=0)\n", + " return dst1, accompaniedimage, paddedprepped, squaredimage, finalcroprect\n", + " else:\n", + " # accompaniedimage = squarepad(accompaniedimage, fill=255)\n", + " return dst1, accompaniedimage" + ] + }, + { + "cell_type": "code", + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle, croprect):\n", - " imgcopy = baseimage.copy()\n", + " # imglist = []\n", + " # print(preppedimage.shape)\n", + " # print(baseimage.shape)\n", + " \n", + " # imgcopy = baseimage.copy()\n", " # sizemultiplier = croprect[3]/preppedimage.shape[0]\n", " \n", " ##adjust the rotation angle if it causes the rectangle to flip w and h size ordering. That is, if it will cause the width to be greater than the height or something. let me think about it for a second more.\n", " # print(baseimage.shape[:2])\n", - " mask = np.full(baseimage.shape[:2], fill_value=255, dtype=np.uint8)\n", - " mask = cv2.rectangle(mask, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=0, thickness=cv2.FILLED)\n", - " \n", - " \n", + " # mask = np.full(baseimage.shape[:2], fill_value=255, dtype=np.uint8)\n", + " # mask = cv2.rectangle(mask, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=0, thickness=cv2.FILLED)\n", + " # temp = baseimage[croprect[1]:croprect[1]+croprect[3], croprect[0]:croprect[0]+croprect[2], :]\n", " \n", " # rotatedmask = mf.rotate(mask, rotationangle, fill=255)\n", " # print(mask.shape)\n", " # return mask, 5\n", - " rotatedmask = mf.rotatewithexactpadding(mask, rotationangle, fill=255)\n", + " # rotatedmask = mf.rotatewithexactpadding(mask, rotationangle, fill=255)\n", " rotatedbaseimage = mf.rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n", + " # rotatedtemp = mf.rotatewithexactpadding(temp, rotationangle, fill=(0,0,0))\n", " # return preppedimage, 5\n", " rotateddst1 = mf.rotatewithexactpadding(preppedimage, rotationangle, fill=(0,0,0))\n", - " fcr = mf.croptoblack(rotatedmask, extraborder=0, returnrect=True) #finalcroprect\n", + " \n", + " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", + " \n", + " # fcr = mf.croptoblack(rotatedmask, extraborder=0, returnrect=True) #finalcroprect\n", + " \n", + " # print(sizemultiplier)\n", + " # imglist.append(rotatedmask)\n", + " # imglist.append(rotatedbaseimage)\n", + " # imglist.append(rotateddst1)\n", + " # return imglist, 5\n", " # return rotatedmask, 5\n", " # return rotatedbaseimage, 5\n", + " # return rotateddst1, 5\n", " # print(fcr)\n", - " rotatedbaseimage = rotatedbaseimage[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", + " # rotatedbaseimage = rotatedbaseimage[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", " # rotateddst1 = rotateddst1[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n", " # return mask, 5\n", " # return rotatedbaseimage, 5\n", " \n", " \n", - " sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", - " print(sizemultiplier)\n", + " # sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n", + " # print(sizemultiplier)\n", " # return rotatedbaseimage, rotationangle\n", "\n", " croppedbaseimage = mf.houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)\n", " \n", - " return croppedbaseimage, rotationangle\n", + " # return croppedbaseimage, rotationangle\n", "\n", - " # finalbaseimage = mf.contourcrop(croppedbaseimage)\n", + " finalbaseimage = mf.contourcrop(croppedbaseimage)\n", "\n", "\n", " return finalbaseimage, rotationangle\n", @@ -585,18 +670,18 @@ }, { "cell_type": "code", - "execution_count": 131, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "def houghlinedeskewandcrop(image):\n", - " canny, ogimage, rect = mf.prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done\n", - " # return canny, croppedogimage\n", + " croppedcanny, croppedimage, canny, ogimage, rect = prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done\n", + " # return canny, ogimage\n", " # print(canny.shape)\n", " # print(croppedogimage.shape)\n", "\n", " ## -----------------finding angle to deskew-----------------\n", - " rotationangle = mf.houghlinedeskewangle(canny)\n", + " rotationangle = mf.houghlinedeskewangle(croppedcanny)\n", " # print(rotationangle)\n", "\n", " \n", @@ -611,12 +696,12 @@ }, { "cell_type": "code", - "execution_count": 132, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "def houghlineprocessing(image):\n", - " croppedanddeskewed, angle = houghlinedeskewandcrop(image)\n", + " croppedanddeskewed, angle = mf.houghlinedeskewandcrop(image)\n", " return croppedanddeskewed\n", " \n", " \n", @@ -642,7 +727,7 @@ }, { "cell_type": "code", - "execution_count": 133, + "execution_count": 12, "metadata": {}, "outputs": [], "source": [ @@ -651,20 +736,13 @@ }, { "cell_type": "code", - "execution_count": 134, + "execution_count": 13, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1.695\n" - ] - } - ], + "outputs": [], "source": [ "# prepped, scaler, hp, vp = mf.squareandthenresize(img, fill=255, width=1000, returnscalerinfo=True)\n", "outs = houghlineprocessing(img)\n", + "# outs = prepimageforhoughline(img, returnrect=True)\n", "# print(img.shape)\n", "# outs = houghlinedeskewandcrop(img)\n", "# outs = outs[0]\n", @@ -674,7 +752,7 @@ }, { "cell_type": "code", - "execution_count": 135, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ @@ -688,7 +766,7 @@ }, { "cell_type": "code", - "execution_count": 136, + "execution_count": 15, "metadata": {}, "outputs": [], "source": [ @@ -700,16 +778,16 @@ }, { "cell_type": "code", - "execution_count": 137, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ - "# testall = True" + "testall = True" ] }, { "cell_type": "code", - "execution_count": 138, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ @@ -719,7 +797,7 @@ }, { "cell_type": "code", - "execution_count": 139, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ @@ -749,9 +827,17 @@ }, { "cell_type": "code", - "execution_count": 140, + "execution_count": 19, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "average time: 0.1962958723306656(s)\n" + ] + } + ], "source": [ "if testall:\n", " results = testondataset(\"/mnt/dataset/baseimages/\", houghlineprocessing)" @@ -759,7 +845,7 @@ }, { "cell_type": "code", - "execution_count": 141, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -769,7 +855,7 @@ }, { "cell_type": "code", - "execution_count": 142, + "execution_count": 21, "metadata": {}, "outputs": [], "source": [ @@ -778,7 +864,7 @@ }, { "cell_type": "code", - "execution_count": 143, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ diff --git a/code/autocropper/myfunctions.py b/code/autocropper/myfunctions.py index c44fd77..fddf559 100644 --- a/code/autocropper/myfunctions.py +++ b/code/autocropper/myfunctions.py @@ -344,11 +344,12 @@ def rotateLine(img, line, angle, returnint=True): def prepimageforhoughline(image, returnrect=True): prepped, scaler, hp, vp = squareandthenresize(image, fill=255, width=1000, returnscalerinfo=True) + ogpreppedshape = prepped.shape prepped, croprect = premorphCrop(prepped) if (prepped.shape[1] > prepped.shape[0]): - prepped = ResizeWithAspectRatio(prepped, width=1000) + prepped, preppedscaler = ResizeWithAspectRatio(prepped, width=1000, retscale=True) else: - prepped = ResizeWithAspectRatio(prepped, height=1000) + prepped, preppedscaler = ResizeWithAspectRatio(prepped, height=1000, retscale=True) finalcroprect = (int(croprect[0]*scaler - hp), int(croprect[1]*scaler - vp), int(croprect[2]*scaler), int(croprect[3]*scaler)) gray1 = cv2.cvtColor(prepped, cv2.COLOR_BGR2GRAY) @@ -364,11 +365,17 @@ def prepimageforhoughline(image, returnrect=True): dst1 = cv2.Canny(dst1, 0, 500, None, 3) # return dst1 + accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :] if returnrect: - return dst1, image, finalcroprect + borderType = cv2.BORDER_CONSTANT + preppadding = [croprect[0], croprect[1], ogpreppedshape[1]-(croprect[0]+croprect[2]), ogpreppedshape[0]-(croprect[1]+croprect[3])] + preppadding = [int(s/preppedscaler) for s in preppadding] + paddedprepped = cv2.copyMakeBorder(dst1, preppadding[1], preppadding[3], preppadding[0], preppadding[2], borderType, 0) + + squaredimage = squarepad(image, fill=0) + + return dst1, accompaniedimage, paddedprepped, squaredimage, finalcroprect else: - accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :] - # accompaniedimage = squarepad(accompaniedimage, fill=255) return dst1, accompaniedimage def houghlinedeskewangle(image): @@ -480,47 +487,35 @@ def contourcrop(baseimage): finalbaseimage = baseimage[scaledmx[1]:scaledmx[1]+scaledmx[3], scaledmx[0]:scaledmx[0]+scaledmx[2], :] return finalbaseimage -def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle): - # rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=0) - # rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0)) - hpad, vpad = mf.determineextrapadding(rect[3], rect[2], rotationangle) - adjustedrect = [rect[0]-hpad, rect[1]-vpad, rect[2]+(2*hpad), rect[3]+(2*vpad)] - croppedogimage = croppedogimage[adjustedrect[1]:adjustedrect[1]+adjustedrect[3], adjustedrect[0]:adjustedrect[0]+adjustedrect[2], :] - - houghlinerotate() - - - - print(preppedimage.shape) - rotateddst1 = rotate(preppedimage, rotationangle) - print(rotateddst1.shape) - rotatedbaseimage = rotate(baseimage, rotationangle) - +def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle, croprect): + rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0)) + rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=(0,0,0)) sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0] - # print(sizemultiplier) - # return rotatedbaseimage, rotationangle + croppedbaseimage = houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier) finalbaseimage = contourcrop(croppedbaseimage) - - + return finalbaseimage, rotationangle def houghlinedeskewandcrop(image): - canny, croppedogimage = prepimageforhoughline(image) ## scaling and cropping occurs. need to also return the changes done - # return canny, croppedogimage + croppedcanny, croppedimage, canny, ogimage, rect = prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done + # return canny, ogimage # print(canny.shape) # print(croppedogimage.shape) ## -----------------finding angle to deskew----------------- - rotationangle = houghlinedeskewangle(canny) + rotationangle = houghlinedeskewangle(croppedcanny) # print(rotationangle) + + # rotatorrect = findcroprectforangle(rect, angle) + # -----------------end of finding angle to deskew----------------- ## -----------------deskewing and then cropping----------------- - outimg, angle = houghlinedeskewthencrop(croppedogimage, canny, rotationangle) + outimg, angle = houghlinedeskewthencrop(ogimage, canny, rotationangle, rect) return outimg, angle def bruteforceprocessrects(greaterrects, lesserrects):