ewellenr/receipt_indexer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed the houghline cropping and deskewing for no black rotation lines

Browse Source 
Adjusted the process so that the black lines left after the houghline
cropping and rotating are no longer there.

Signed-off-by: Ethan Wellenreiter <ewellenreiter@gmail.com>
This commit is contained in:
Ethan Wellenreiter 2023-11-27 15:54:23 -05:00
parent b149e86963
commit d9aa07121a
2 changed files with 159 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

184
code/autocropper/houghlinedevspace.ipynb
View File

@ -2,9 +2,20 @@
"cells": [
{
"cell_type": "code",
"execution_count": 123,
"execution_count": 1,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/usr/local/lib/python3.10/dist-packages/torchvision/datapoints/__init__.py:12: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n",
" warnings.warn(_BETA_TRANSFORMS_WARNING)\n",
"/usr/local/lib/python3.10/dist-packages/torchvision/transforms/v2/__init__.py:54: UserWarning: The torchvision.datapoints and torchvision.transforms.v2 namespaces are still Beta. While we do not expect major breaking changes, some APIs may still change according to user feedback. Please submit any feedback you may have in this issue: https://github.com/pytorch/vision/issues/6753, and you can also check out https://github.com/pytorch/vision/issues/7319 to learn more about the APIs that we suspect might involve future changes. You can silence this warning by calling torchvision.disable_beta_transforms_warning().\n",
" warnings.warn(_BETA_TRANSFORMS_WARNING)\n"
]
}
],
"source": [
"import cv2\n",
"import myfunctions as mf\n",
@ -15,7 +26,7 @@
},
{
"cell_type": "code",
"execution_count": 124,
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@ -55,7 +66,7 @@
},
{
"cell_type": "code",
"execution_count": 125,
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
@ -77,7 +88,7 @@
},
{
"cell_type": "code",
"execution_count": 126,
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
@ -91,7 +102,7 @@
},
{
"cell_type": "code",
"execution_count": 127,
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
@ -102,7 +113,7 @@
},
{
"cell_type": "code",
"execution_count": 128,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
@ -448,7 +459,7 @@
},
{
"cell_type": "code",
"execution_count": 129,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
@ -468,47 +479,121 @@
},
{
"cell_type": "code",
"execution_count": 130,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"def prepimageforhoughline(image, returnrect=True):\n",
" prepped, scaler, hp, vp = mf.squareandthenresize(image, fill=255, width=1000, returnscalerinfo=True)\n",
" ogpreppedshape = prepped.shape\n",
" # print(ogpreppedshape)\n",
" prepped, croprect = mf.premorphCrop(prepped)\n",
" # print(prepped.shape)\n",
" if (prepped.shape[1] > prepped.shape[0]):\n",
" prepped, preppedscaler = mf.ResizeWithAspectRatio(prepped, width=1000, retscale=True)\n",
" else:\n",
" prepped, preppedscaler = mf.ResizeWithAspectRatio(prepped, height=1000, retscale=True)\n",
" # print(prepped.shape)\n",
" # print(preppedscaler)\n",
" finalcroprect = (int(croprect[0]*scaler - hp), int(croprect[1]*scaler - vp), int(croprect[2]*scaler), int(croprect[3]*scaler))\n",
" gray1 = cv2.cvtColor(prepped, cv2.COLOR_BGR2GRAY)\n",
"\n",
" dst1 = cv2.Canny(gray1, 0, 500, None, 3)\n",
"\n",
" \n",
" kernel = np.ones((5,5), np.uint8)\n",
" out = cv2.morphologyEx(dst1, cv2.MORPH_DILATE, kernel)\n",
" out = cv2.blur(out, (5,5))\n",
" kernel = np.ones((6,6), np.uint8)\n",
" dst1 = cv2.morphologyEx(out, cv2.MORPH_ERODE, kernel)\n",
" # return dst1\n",
"\n",
" dst1 = cv2.Canny(dst1, 0, 500, None, 3)\n",
" # return dst1\n",
" accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :]\n",
" if returnrect:\n",
" borderType = cv2.BORDER_CONSTANT\n",
" ## first the og padding when we include the padding added by squaring\n",
" preppadding = [croprect[0], croprect[1], ogpreppedshape[1]-(croprect[0]+croprect[2]), ogpreppedshape[0]-(croprect[1]+croprect[3])]\n",
" # print(croprect)\n",
" # print(dst1.shape)\n",
" # print(preppadding)\n",
" preppadding = [int(s/preppedscaler) for s in preppadding]\n",
" # print(preppadding)\n",
" ## now adjust for any padding. hp and vp are for the full sized image so they need to be scaled down first.\n",
" # adjustedhp = int(hp/scaler)\n",
" # adjustedvp = int(vp/scaler)\n",
" # preppadding[0] -= adjustedhp\n",
" # preppadding[2] -= adjustedhp\n",
" # preppadding[1] -= adjustedvp\n",
" # preppadding[3] -= adjustedvp\n",
" \n",
" paddedprepped = cv2.copyMakeBorder(dst1, preppadding[1], preppadding[3], preppadding[0], preppadding[2], borderType, 0)\n",
" # paddedprepped = dst1\n",
" squaredimage = mf.squarepad(image, fill=0)\n",
" return dst1, accompaniedimage, paddedprepped, squaredimage, finalcroprect\n",
" else:\n",
" # accompaniedimage = squarepad(accompaniedimage, fill=255)\n",
" return dst1, accompaniedimage"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle, croprect):\n",
" imgcopy = baseimage.copy()\n",
" # imglist = []\n",
" # print(preppedimage.shape)\n",
" # print(baseimage.shape)\n",
" \n",
" # imgcopy = baseimage.copy()\n",
" # sizemultiplier = croprect[3]/preppedimage.shape[0]\n",
" \n",
" ##adjust the rotation angle if it causes the rectangle to flip w and h size ordering. That is, if it will cause the width to be greater than the height or something. let me think about it for a second more.\n",
" # print(baseimage.shape[:2])\n",
" mask = np.full(baseimage.shape[:2], fill_value=255, dtype=np.uint8)\n",
" mask = cv2.rectangle(mask, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=0, thickness=cv2.FILLED)\n",
" \n",
" \n",
" # mask = np.full(baseimage.shape[:2], fill_value=255, dtype=np.uint8)\n",
" # mask = cv2.rectangle(mask, (croprect[0], croprect[1]), (croprect[0]+croprect[2], croprect[1]+croprect[3]), color=0, thickness=cv2.FILLED)\n",
" # temp = baseimage[croprect[1]:croprect[1]+croprect[3], croprect[0]:croprect[0]+croprect[2], :]\n",
" \n",
" # rotatedmask = mf.rotate(mask, rotationangle, fill=255)\n",
" # print(mask.shape)\n",
" # return mask, 5\n",
" rotatedmask = mf.rotatewithexactpadding(mask, rotationangle, fill=255)\n",
" # rotatedmask = mf.rotatewithexactpadding(mask, rotationangle, fill=255)\n",
" rotatedbaseimage = mf.rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))\n",
" # rotatedtemp = mf.rotatewithexactpadding(temp, rotationangle, fill=(0,0,0))\n",
" # return preppedimage, 5\n",
" rotateddst1 = mf.rotatewithexactpadding(preppedimage, rotationangle, fill=(0,0,0))\n",
" fcr = mf.croptoblack(rotatedmask, extraborder=0, returnrect=True) #finalcroprect\n",
" \n",
" sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n",
" \n",
" # fcr = mf.croptoblack(rotatedmask, extraborder=0, returnrect=True) #finalcroprect\n",
" \n",
" # print(sizemultiplier)\n",
" # imglist.append(rotatedmask)\n",
" # imglist.append(rotatedbaseimage)\n",
" # imglist.append(rotateddst1)\n",
" # return imglist, 5\n",
" # return rotatedmask, 5\n",
" # return rotatedbaseimage, 5\n",
" # return rotateddst1, 5\n",
" # print(fcr)\n",
" rotatedbaseimage = rotatedbaseimage[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n",
" # rotatedbaseimage = rotatedbaseimage[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n",
" # rotateddst1 = rotateddst1[fcr[1]:fcr[1]+fcr[3], fcr[0]:fcr[0]+fcr[2]]\n",
" # return mask, 5\n",
" # return rotatedbaseimage, 5\n",
" \n",
" \n",
" sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n",
" print(sizemultiplier)\n",
" # sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]\n",
" # print(sizemultiplier)\n",
" # return rotatedbaseimage, rotationangle\n",
"\n",
" croppedbaseimage = mf.houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)\n",
" \n",
" return croppedbaseimage, rotationangle\n",
" # return croppedbaseimage, rotationangle\n",
"\n",
" # finalbaseimage = mf.contourcrop(croppedbaseimage)\n",
" finalbaseimage = mf.contourcrop(croppedbaseimage)\n",
"\n",
"\n",
" return finalbaseimage, rotationangle\n",
@ -585,18 +670,18 @@
},
{
"cell_type": "code",
"execution_count": 131,
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"def houghlinedeskewandcrop(image):\n",
" canny, ogimage, rect = mf.prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done\n",
" # return canny, croppedogimage\n",
" croppedcanny, croppedimage, canny, ogimage, rect = prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done\n",
" # return canny, ogimage\n",
" # print(canny.shape)\n",
" # print(croppedogimage.shape)\n",
"\n",
" ## -----------------finding angle to deskew-----------------\n",
" rotationangle = mf.houghlinedeskewangle(canny)\n",
" rotationangle = mf.houghlinedeskewangle(croppedcanny)\n",
" # print(rotationangle)\n",
"\n",
" \n",
@ -611,12 +696,12 @@
},
{
"cell_type": "code",
"execution_count": 132,
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"def houghlineprocessing(image):\n",
" croppedanddeskewed, angle = houghlinedeskewandcrop(image)\n",
" croppedanddeskewed, angle = mf.houghlinedeskewandcrop(image)\n",
" return croppedanddeskewed\n",
" \n",
" \n",
@ -642,7 +727,7 @@
},
{
"cell_type": "code",
"execution_count": 133,
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
@ -651,20 +736,13 @@
},
{
"cell_type": "code",
"execution_count": 134,
"execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1.695\n"
]
}
],
"outputs": [],
"source": [
"# prepped, scaler, hp, vp = mf.squareandthenresize(img, fill=255, width=1000, returnscalerinfo=True)\n",
"outs = houghlineprocessing(img)\n",
"# outs = prepimageforhoughline(img, returnrect=True)\n",
"# print(img.shape)\n",
"# outs = houghlinedeskewandcrop(img)\n",
"# outs = outs[0]\n",
@ -674,7 +752,7 @@
},
{
"cell_type": "code",
"execution_count": 135,
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
@ -688,7 +766,7 @@
},
{
"cell_type": "code",
"execution_count": 136,
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
@ -700,16 +778,16 @@
},
{
"cell_type": "code",
"execution_count": 137,
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
"# testall = True"
"testall = True"
]
},
{
"cell_type": "code",
"execution_count": 138,
"execution_count": 17,
"metadata": {},
"outputs": [],
"source": [
@ -719,7 +797,7 @@
},
{
"cell_type": "code",
"execution_count": 139,
"execution_count": 18,
"metadata": {},
"outputs": [],
"source": [
@ -749,9 +827,17 @@
},
{
"cell_type": "code",
"execution_count": 140,
"execution_count": 19,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"average time: 0.1962958723306656(s)\n"
]
}
],
"source": [
"if testall:\n",
" results = testondataset(\"/mnt/dataset/baseimages/\", houghlineprocessing)"
@ -759,7 +845,7 @@
},
{
"cell_type": "code",
"execution_count": 141,
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
@ -769,7 +855,7 @@
},
{
"cell_type": "code",
"execution_count": 142,
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
@ -778,7 +864,7 @@
},
{
"cell_type": "code",
"execution_count": 143,
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [

53
code/autocropper/myfunctions.py
View File

@ -344,11 +344,12 @@ def rotateLine(img, line, angle, returnint=True):
def prepimageforhoughline(image, returnrect=True):
prepped, scaler, hp, vp = squareandthenresize(image, fill=255, width=1000, returnscalerinfo=True)
ogpreppedshape = prepped.shape
prepped, croprect = premorphCrop(prepped)
if (prepped.shape[1] > prepped.shape[0]):
prepped = ResizeWithAspectRatio(prepped, width=1000)
prepped, preppedscaler = ResizeWithAspectRatio(prepped, width=1000, retscale=True)
else:
prepped = ResizeWithAspectRatio(prepped, height=1000)
prepped, preppedscaler = ResizeWithAspectRatio(prepped, height=1000, retscale=True)
finalcroprect = (int(croprect[0]*scaler - hp), int(croprect[1]*scaler - vp), int(croprect[2]*scaler), int(croprect[3]*scaler))
gray1 = cv2.cvtColor(prepped, cv2.COLOR_BGR2GRAY)
@ -364,11 +365,17 @@ def prepimageforhoughline(image, returnrect=True):
dst1 = cv2.Canny(dst1, 0, 500, None, 3)
# return dst1
accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :]
if returnrect:
return dst1, image, finalcroprect
borderType = cv2.BORDER_CONSTANT
preppadding = [croprect[0], croprect[1], ogpreppedshape[1]-(croprect[0]+croprect[2]), ogpreppedshape[0]-(croprect[1]+croprect[3])]
preppadding = [int(s/preppedscaler) for s in preppadding]
paddedprepped = cv2.copyMakeBorder(dst1, preppadding[1], preppadding[3], preppadding[0], preppadding[2], borderType, 0)
squaredimage = squarepad(image, fill=0)
return dst1, accompaniedimage, paddedprepped, squaredimage, finalcroprect
else:
accompaniedimage = image[finalcroprect[1]:finalcroprect[1]+finalcroprect[3], finalcroprect[0]:finalcroprect[0]+finalcroprect[2], :]
# accompaniedimage = squarepad(accompaniedimage, fill=255)
return dst1, accompaniedimage
def houghlinedeskewangle(image):
@ -480,47 +487,35 @@ def contourcrop(baseimage):
finalbaseimage = baseimage[scaledmx[1]:scaledmx[1]+scaledmx[3], scaledmx[0]:scaledmx[0]+scaledmx[2], :]
return finalbaseimage
def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle):
# rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=0)
# rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))
hpad, vpad = mf.determineextrapadding(rect[3], rect[2], rotationangle)
adjustedrect = [rect[0]-hpad, rect[1]-vpad, rect[2]+(2*hpad), rect[3]+(2*vpad)]
croppedogimage = croppedogimage[adjustedrect[1]:adjustedrect[1]+adjustedrect[3], adjustedrect[0]:adjustedrect[0]+adjustedrect[2], :]
houghlinerotate()
print(preppedimage.shape)
rotateddst1 = rotate(preppedimage, rotationangle)
print(rotateddst1.shape)
rotatedbaseimage = rotate(baseimage, rotationangle)
def houghlinedeskewthencrop(baseimage, preppedimage, rotationangle, croprect):
rotatedbaseimage = rotatewithexactpadding(baseimage, rotationangle, fill=(0,0,0))
rotateddst1 = rotatewithexactpadding(preppedimage, rotationangle, fill=(0,0,0))
sizemultiplier = rotatedbaseimage.shape[0]/rotateddst1.shape[0]
# print(sizemultiplier)
# return rotatedbaseimage, rotationangle
croppedbaseimage = houghlinepcrop(rotatedbaseimage, rotateddst1, sizemultiplier)
finalbaseimage = contourcrop(croppedbaseimage)
return finalbaseimage, rotationangle
def houghlinedeskewandcrop(image):
canny, croppedogimage = prepimageforhoughline(image) ## scaling and cropping occurs. need to also return the changes done
# return canny, croppedogimage
croppedcanny, croppedimage, canny, ogimage, rect = prepimageforhoughline(image, returnrect=True) ## scaling and cropping occurs. need to also return the changes done
# return canny, ogimage
# print(canny.shape)
# print(croppedogimage.shape)
## -----------------finding angle to deskew-----------------
rotationangle = houghlinedeskewangle(canny)
rotationangle = houghlinedeskewangle(croppedcanny)
# print(rotationangle)
# rotatorrect = findcroprectforangle(rect, angle)
# -----------------end of finding angle to deskew-----------------
## -----------------deskewing and then cropping-----------------
outimg, angle = houghlinedeskewthencrop(croppedogimage, canny, rotationangle)
outimg, angle = houghlinedeskewthencrop(ogimage, canny, rotationangle, rect)
return outimg, angle
def bruteforceprocessrects(greaterrects, lesserrects):