boofcv.alg.shapes.polygon.BinaryPolygonDetector Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of feature Show documentation
Show all versions of feature Show documentation
BoofCV is an open source Java library for real-time computer vision and robotics applications.
/*
* Copyright (c) 2011-2016, Peter Abeles. All Rights Reserved.
*
* This file is part of BoofCV (http://boofcv.org).
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package boofcv.alg.shapes.polygon;
import boofcv.alg.InputSanityCheck;
import boofcv.alg.distort.DistortImageOps;
import boofcv.alg.filter.binary.Contour;
import boofcv.alg.filter.binary.LinearContourLabelChang2004;
import boofcv.alg.shapes.edge.PolygonEdgeIntensity;
import boofcv.alg.shapes.polyline.MinimizeEnergyPrune;
import boofcv.alg.shapes.polyline.RefinePolyLineCorner;
import boofcv.alg.shapes.polyline.SplitMergeLineFitLoop;
import boofcv.struct.ConnectRule;
import boofcv.struct.distort.PixelTransform_F32;
import boofcv.struct.image.GrayS32;
import boofcv.struct.image.GrayU8;
import boofcv.struct.image.ImageGray;
import georegression.geometry.UtilPolygons2D_F64;
import georegression.metric.Area2D_F64;
import georegression.struct.point.Point2D_F64;
import georegression.struct.point.Point2D_I32;
import georegression.struct.shapes.Polygon2D_F64;
import georegression.struct.shapes.RectangleLength2D_F32;
import org.ddogleg.struct.FastQueue;
import org.ddogleg.struct.GrowQueue_B;
import org.ddogleg.struct.GrowQueue_I32;
import java.util.ArrayList;
import java.util.List;
/**
*
* Detects convex polygons with the specified number of sides in an image. Shapes are assumed to be black shapes
* against a white background, allowing for thresholding to be used. Subpixel refinement is done using the
* provided implementation of {@link RefineBinaryPolygon}.
*
*
* Processing Steps:
*
* - First the input a gray scale image and a binarized version of it.
* - The contours of black blobs are found.
* - From the contours polygons are fitted and refined to pixel accuracy.
* - (Optional) Sub-pixel refinement of the polygon's edges and/or corners.
*
*
*
* The returned polygons will encompass the entire black polygon. Here is a simple example in 1D. If all pixels are
* white, but pixels ranging from 5 to 10, inclusive, then the returned boundaries would be 5.0 to 11.0. This
* means that coordinates 5.0 ≤ x < 11.0 are all black. 11.0 is included, but note that the entire pixel 11 is white.
*
*
* @author Peter Abeles
*/
public class BinaryPolygonDetector {
// minimum size of a shape's contour as a fraction of the image width
private double minContourFraction;
private int minimumContour; // this is image.width*minContourFraction
private double minimumArea; // computed from minimumContour
// does the polygon have to be convex
private boolean convex;
private LinearContourLabelChang2004 contourFinder = new LinearContourLabelChang2004(ConnectRule.FOUR);
private GrayS32 labeled = new GrayS32(1,1);
// finds the initial polygon around a target candidate
private SplitMergeLineFitLoop fitPolygon;
// removes extra corners
GrowQueue_I32 pruned = new GrowQueue_I32();
MinimizeEnergyPrune pruner;
// Improve the selection of corner pixels in the contour
private RefinePolyLineCorner improveContour = new RefinePolyLineCorner(true,20);
// Refines the estimate of the polygon's lines using a subpixel technique
private RefineBinaryPolygon refinePolygon;
// List of all squares that it finds
private FastQueue found = new FastQueue(Polygon2D_F64.class,true);
// extera information for found shapes
private FastQueue foundInfo = new FastQueue(Info.class,true);
// type of input image
private Class inputType;
// number of lines allowed in the polygon
private int minSides,maxSides;
// true if points touching the border are NOT pruned
private boolean canTouchBorder;
// work space for initial polygon
private Polygon2D_F64 workPoly = new Polygon2D_F64();
// should the order of the polygon be on clockwise order on output?
private boolean outputClockwise;
// storage for the contours associated with a found target. used for debugging
// All contours are in distorted pixel coordiantes
private List foundContours = new ArrayList();
// transforms which can be used to handle lens distortion
protected PixelTransform_F32 distToUndist, undistToDist;
boolean verbose = false;
// used to remove false positives
PolygonEdgeIntensity edgeIntensity;
double edgeThreshold;
// should it check the edge score before? With a chessboard pattern the initial guess is known to be very poor
// so it should only check the edge after. Otherwise its good to filter before optimization.
boolean checkEdgeBefore = true;
// helper used to customize low level behaviors internally
private PolygonHelper helper;
// storage for contour in undistorted image pixels
private FastQueue contourUndist = new FastQueue(Point2D_I32.class,true);
/**
* Configures the detector.
* @param minSides minimum number of sides
* @param maxSides maximum number of sides
* @param contourToPolygon Fits a crude polygon to the shape's binary contour
* @param refinePolygon (Optional) Refines the polygon's lines. Set to null to skip step
* @param minContourFraction Size of minimum contour as a fraction of the input image's width. Try 0.23
* @param outputClockwise If true then the order of the output polygons will be in clockwise order
* @param convex If true it will only return convex shapes
* @param touchBorder if true then shapes which touch the image border are allowed
* @param splitPenalty Penalty given to a line segment while splitting. See {@link MinimizeEnergyPrune}
* @param edgeThreshold Polygons with an edge intensity less than this are discarded.
* @param inputType Type of input image it's processing
*/
public BinaryPolygonDetector(int minSides, int maxSides,
SplitMergeLineFitLoop contourToPolygon,
RefineBinaryPolygon refinePolygon,
double minContourFraction,
boolean outputClockwise,
boolean convex,
boolean touchBorder, double splitPenalty,
double edgeThreshold,
Class inputType) {
setNumberOfSides(minSides,maxSides);
this.refinePolygon = refinePolygon;
this.edgeIntensity = new PolygonEdgeIntensity(1,1.5,15,inputType);
this.inputType = inputType;
this.minContourFraction = minContourFraction;
this.fitPolygon = contourToPolygon;
this.outputClockwise = outputClockwise;
this.convex = convex;
this.canTouchBorder = touchBorder;
this.edgeThreshold = edgeThreshold;
pruner = new MinimizeEnergyPrune(splitPenalty);
workPoly = new Polygon2D_F64(1);
}
/**
* Specifies transforms which can be used to change coordinates from distorted to undistorted and the opposite
* coordinates. The undistorted image is never explicitly created.
*
*
* WARNING: The undistorted image must have the same bounds as the distorted input image. This is because
* several of the bounds checks use the image shape. This are simplified greatly by this assumption.
*
*
* @param width Input image width. Used in sanity check only.
* @param height Input image height. Used in sanity check only.
* @param distToUndist Transform from distorted to undistorted image.
* @param undistToDist Transform from undistorted to distorted image.
*/
public void setLensDistortion( int width , int height ,
PixelTransform_F32 distToUndist , PixelTransform_F32 undistToDist ) {
this.distToUndist = distToUndist;
this.undistToDist = undistToDist;
// sanity check since I think many people will screw this up.
RectangleLength2D_F32 rect = DistortImageOps.boundBox_F32(width, height, distToUndist);
float x1 = rect.x0 + rect.width;
float y1 = rect.y0 + rect.height;
float tol = 1e-4f;
if( rect.getX() < -tol || rect.getY() < -tol || x1 > width+tol || y1 > height+tol ) {
throw new IllegalArgumentException("You failed the idiot test! RTFM! The undistorted image "+
"must be contained by the same bounds as the input distorted image");
}
if( refinePolygon != null ) {
refinePolygon.setLensDistortion(width, height, distToUndist, undistToDist);
}
edgeIntensity.setTransform(undistToDist);
}
/**
* Examines the undistorted gray scake input image for squares.
*
* @param gray Input image
*/
public void process(T gray, GrayU8 binary) {
if( verbose ) System.out.println("ENTER BinaryPolygonDetector.process()");
InputSanityCheck.checkSameShape(binary, gray);
if( labeled.width != gray.width || labeled.height == gray.width )
configure(gray.width,gray.height);
found.reset();
foundContours.clear();
foundInfo.reset();
edgeIntensity.setImage(gray);
findCandidateShapes(gray, binary);
if( verbose ) System.out.println("EXIT BinaryPolygonDetector.process()");
}
/**
* Specifies the image's intrinsic parameters and target size
*
* @param width Width of the input image
* @param height Height of the input image
*/
private void configure( int width , int height ) {
// resize storage images
labeled.reshape(width, height);
// adjust size based parameters based on image size
this.minimumContour = (int)(width*minContourFraction);
this.minimumArea = Math.pow(this.minimumContour /4.0,2);
if( helper != null )
helper.setImageShape(width,height);
}
/**
* Finds blobs in the binary image. Then looks for blobs that meet size and shape requirements. See code
* below for the requirements. Those that remain are considered to be target candidates.
*/
private void findCandidateShapes( T gray , GrayU8 binary ) {
int maxSidesConsider = (int)Math.ceil(maxSides*1.5);
// stop fitting the polygon if it clearly has way too many sides
fitPolygon.setAbortSplits(2*maxSides);
// find binary blobs
contourFinder.process(binary, labeled);
// find blobs where all 4 edges are lines
FastQueue blobs = contourFinder.getContours();
for (int i = 0; i < blobs.size; i++) {
Contour c = blobs.get(i);
if( c.external.size() >= minimumContour) {
// System.out.println("----- candidate "+c.external.size());
// ignore shapes which touch the image border
boolean touchesBorder = touchesBorder(c.external);
if( !canTouchBorder && touchesBorder ) {
if( verbose ) System.out.println("rejected polygon, touched border");
continue;
}
if( helper != null )
if( !helper.filterContour(c.external,touchesBorder,true) )
continue;
// remove lens distortion
List contourUndist;
if( distToUndist != null ) {
removeDistortionFromContour(c.external,this.contourUndist);
contourUndist = this.contourUndist.toList();
if( helper != null )
if( !helper.filterContour(contourUndist,touchesBorder,false) )
continue;
} else {
contourUndist = c.external;
}
if( !fitPolygon.process(contourUndist) ) {
if( verbose ) System.out.println("rejected polygon initial fit failed. contour size = "+c.external.size());
continue;
}
GrowQueue_I32 splits = fitPolygon.getSplits();
if( splits.size() > maxSidesConsider ) {
if( verbose ) System.out.println("Way too many corners, "+splits.size()+". Aborting before improve. Contour size "+c.external.size());
continue;
}
// Perform a local search and improve the corner placements
if( !improveContour.fit(contourUndist,splits) ) {
if( verbose ) System.out.println("rejected improve contour. contour size = "+c.external.size());
continue;
}
// reduce the number of corners based on an energy model
pruner.prune(c.external, splits, pruned);
splits = pruned;
// only accept polygons with the expected number of sides
if (!expectedNumberOfSides(splits)) {
// System.out.println("First point "+c.external.get(0));
if( verbose ) System.out.println("rejected number of sides. "+splits.size()+" contour "+c.external.size());
continue;
}
if( helper != null ) {
if( !helper.filterPixelPolygon(contourUndist,c.external,splits,touchesBorder) ) {
if( verbose ) System.out.println("rejected by helper.filterPixelPolygon()");
continue;
}
}
// convert the format of the initial crude polygon
workPoly.vertexes.resize(splits.size());
for (int j = 0; j < splits.size(); j++) {
Point2D_I32 p = contourUndist.get( splits.get(j));
workPoly.get(j).set(p.x,p.y);
}
if( helper != null ) {
helper.adjustBeforeOptimize(workPoly);
}
// Filter out polygons which are not convex if requested by the user
if( convex && !UtilPolygons2D_F64.isConvex(workPoly)) {
if( verbose ) System.out.println("Rejected not convex");
continue;
}
// make sure it's big enough
double area = Area2D_F64.polygonSimple(workPoly);
if( area < minimumArea ) {
if( verbose ) System.out.println("Rejected area");
continue;
}
// Test the edge quality and prune before performing an expensive optimization
if( checkEdgeBefore && !checkPolygonEdge(workPoly,workPoly.isCCW())) {
// if( verbose ) System.out.println("Rejected edge score before");
continue;
}
Polygon2D_F64 refined = found.grow();
refined.vertexes.resize(splits.size);
boolean success;
if( refinePolygon != null ) {
refinePolygon.setImage(gray);
success = refinePolygon.refine(workPoly,contourUndist,splits,refined);
if( verbose && !success ) System.out.println("Rejected after refinePolygon");
} else {
refined.set(workPoly);
success = true;
}
boolean refinedCCW = refined.isCCW();
// test it again with the full threshold
if( !checkPolygonEdge(refined,refinedCCW)) {
if( verbose ) System.out.println("Rejected edge score, after");
success = false;
}
// refine the polygon and add it to the found list
if( success ) {
if( outputClockwise == refinedCCW )
refined.flip();
// System.out.println("SUCCESS!!!\n");
c.id = found.size();
foundContours.add(c);
Info info = foundInfo.grow();
info.external = true;
info.borderCorners.reset();
if( touchesBorder ) {
// tolerance is a little bit above 0.5.pixels due to prior rounding to integer
determineCornersOnBorder(refined, info.borderCorners, 0.7f);
}
info.edgeInside = edgeIntensity.getAverageInside();
info.edgeOutside = edgeIntensity.getAverageOutside();
} else {
found.removeTail();
}
}
}
}
/**
* Check to see if corners are touching the image border
* @param polygon Refined polygon
* @param corners storage for corner indexes
*/
void determineCornersOnBorder( Polygon2D_F64 polygon , GrowQueue_B corners , float tol ) {
corners.reset();
for (int i = 0; i < polygon.size(); i++) {
corners.add(isUndistortedOnBorder(polygon.get(i),tol));
}
}
/**
* Coverts the point into distorted image coordinates and then checks to see if it is on the image border
* @param undistorted pixel in undistorted coordinates
* @param tol Tolerance for a point being on the image border
* @return true if on the border or false otherwise
*/
boolean isUndistortedOnBorder( Point2D_F64 undistorted , float tol ) {
float x,y;
if( undistToDist == null ) {
x = (float)undistorted.x;
y = (float)undistorted.y;
} else {
undistToDist.compute((int)Math.round(undistorted.x),(int)Math.round(undistorted.y));
x = undistToDist.distX;
y = undistToDist.distY;
}
return( x <= tol || y <= tol || x+tol >= labeled.width-1 || y+tol >= labeled.height-1 );
}
/**
* Checks to see if the edge is intense enough or if it should be discarded as noise
* @return true if the edge is strong enough
*/
private boolean checkPolygonEdge( Polygon2D_F64 polygon , boolean ccw ) {
if(!edgeIntensity.computeEdge(polygon,ccw)) {
if( verbose ) System.out.println("Can't compute polygon edge intensity");
return false;
}
if(!edgeIntensity.checkIntensity(true,edgeThreshold)) {
if( verbose ) {
double inside = edgeIntensity.getAverageInside();
double outside = edgeIntensity.getAverageOutside();
System.out.println("Rejected edge score inside: " + inside+" "+outside);
}
return false;
}
return true;
}
/**
* True if the number of sides found matches what it is looking for
*/
private boolean expectedNumberOfSides(GrowQueue_I32 splits) {
return splits.size() >= minSides && splits.size() <= maxSides;
}
/**
* Removes lens distortion from the found contour
*/
private void removeDistortionFromContour(List distorted, FastQueue undistorted ) {
undistorted.reset();
for (int j = 0; j < distorted.size(); j++) {
// remove distortion
Point2D_I32 p = distorted.get(j);
distToUndist.compute(p.x,p.y);
// round to minimize error
int x = Math.round(distToUndist.distX);
int y = Math.round(distToUndist.distY);
// save the results
undistorted.grow().set(x,y);
}
}
/**
* Checks to see if some part of the contour touches the image border. Most likely cropped
*/
protected final boolean touchesBorder( List contour ) {
int endX = labeled.width-1;
int endY = labeled.height-1;
for (int j = 0; j < contour.size(); j++) {
Point2D_I32 p = contour.get(j);
if( p.x == 0 || p.y == 0 || p.x == endX || p.y == endY )
{
return true;
}
}
return false;
}
public void setHelper(PolygonHelper helper) {
this.helper = helper;
}
public boolean isConvex() {
return convex;
}
public void setConvex(boolean convex) {
this.convex = convex;
}
public GrayS32 getLabeled() {
return labeled;
}
public boolean isOutputClockwise() {
return outputClockwise;
}
public FastQueue getFoundPolygons() {
return found;
}
public List getUsedContours(){return foundContours;}
public List getAllContours(){return contourFinder.getContours().toList();}
public Class getInputType() {
return inputType;
}
public void setNumberOfSides( int min , int max ) {
if( min < 3 )
throw new IllegalArgumentException("The min must be >= 3");
if( max < min )
throw new IllegalArgumentException("The max must be >= the min");
this.minSides = min;
this.maxSides = max;
}
public int getMinimumSides() {
return minSides;
}
public int getMaximumSides() {
return maxSides;
}
public void setVerbose(boolean verbose) {
this.verbose = verbose;
}
public boolean isCheckEdgeBefore() {
return checkEdgeBefore;
}
public RefineBinaryPolygon getRefinePolygon() {
return refinePolygon;
}
public void setRefinePolygon(RefineBinaryPolygon refinePolygon) {
this.refinePolygon = refinePolygon;
}
public double getEdgeThreshold() {
return edgeThreshold;
}
public void setEdgeThreshold(double edgeThreshold) {
this.edgeThreshold = edgeThreshold;
}
public PixelTransform_F32 getDistToUndist() {
return distToUndist;
}
public PixelTransform_F32 getUndistToDist() {
return undistToDist;
}
/**
* Returns additional information on the polygon
*/
public FastQueue getPolygonInfo() {
return foundInfo;
}
/**
* If set to true it will prune using polygons using their edge intensity before sub-pixel optimization.
* This should only be set to false if the initial edge is known to be off by a bit, like with a chessboard.
* @param checkEdgeBefore true for checking before and false for after.
*/
public void setCheckEdgeBefore(boolean checkEdgeBefore) {
this.checkEdgeBefore = checkEdgeBefore;
}
public static class Info
{
/**
* Was it created from an external or internal contour
*/
public boolean external;
/**
* Average pixel intensity score along the polygon's edge inside and outside
*/
public double edgeInside,edgeOutside;
/**
* Boolean value for each corner being along the border. If empty then non of the corners are long the border.
* true means the corner is a border corner.
*/
public GrowQueue_B borderCorners = new GrowQueue_B();
public boolean touchesBorder() {
if( borderCorners.size() == 0 )
return false;
else {
for (int i = 0; i < borderCorners.size(); i++) {
if( borderCorners.get(i))
return true;
}
}
return false;
}
}
}