com.seleniumtests.util.imaging.ImageDetector Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of core Show documentation
There is a newer version: 4.23.18
/**
 * Orignal work: Copyright 2015 www.seleniumtests.com
 * Modified work: Copyright 2016 www.infotel.com
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.seleniumtests.util.imaging;

import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;

import javax.swing.ImageIcon;
import javax.swing.JFrame;
import javax.swing.JLabel;
import javax.swing.WindowConstants;

import org.apache.log4j.Logger;
import org.opencv.calib3d.Calib3d;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.MatOfByte;
import org.opencv.core.MatOfDMatch;
import org.opencv.core.MatOfKeyPoint;
import org.opencv.core.MatOfPoint2f;
import org.opencv.core.Point;
import org.opencv.core.Scalar;
import org.opencv.features2d.DMatch;
import org.opencv.features2d.DescriptorExtractor;
import org.opencv.features2d.DescriptorMatcher;
import org.opencv.features2d.FeatureDetector;
import org.opencv.features2d.Features2d;
import org.opencv.features2d.KeyPoint;
import org.opencv.highgui.Highgui;
import org.openqa.selenium.Rectangle;

import com.seleniumtests.customexception.ImageSearchException;
import com.seleniumtests.util.logging.SeleniumRobotLogger;

/**
 * Class made for detecting an image inside an other one
 * It uses openCV to look for the objectImage inside the sceneImage
 * Rotations and resizing are supported
 * @author behe
 *
 */
public class ImageDetector {
	
	private Rectangle detectedRectangle;
	private boolean computed = false;
	private long rotationAngle;
	private File sceneImage;
	private File objectImage;
	private boolean debug = false;
	private double detectionThreshold = 0.05;
	private Mat imgMatch = new Mat();
	private double sizeRatio;
	private static Logger logger = SeleniumRobotLogger.getLogger(ImageDetector.class);
	
	// load openCV
	// In case of "UnsatisfiedLinkError, library already loaded in another class loader", during unit tests, check that 
	// this class or a calling one is not "prepared" through PowerMockito (which reloads the class in another class loader)
	static {
		nu.pattern.OpenCV.loadShared();
	}
	
	public ImageDetector() {
		// do nothing, only for test
	}
	
	public ImageDetector(File sceneImage, File objectImage) {
		this(sceneImage, objectImage, 0.05);
	}
	
	public ImageDetector(File sceneImage, File objectImage, double detectionThreshold) {
		setSceneImage(sceneImage);
		setObjectImage(objectImage);
		this.detectionThreshold = detectionThreshold;
	}
	
	/**
	 * Compute the rectangle where the searched picture is and the rotation angle between both images
	 * Throw {@link ImageSearchException} if picture is not found
	 * @return
	 */
	public void detectCorrespondingZone() {
		Mat objectImageMat = Highgui.imread(objectImage.getAbsolutePath(), Highgui.CV_LOAD_IMAGE_COLOR);
		Mat sceneImageMat = Highgui.imread(sceneImage.getAbsolutePath(), Highgui.CV_LOAD_IMAGE_COLOR);
		FeatureDetector surf = FeatureDetector.create(FeatureDetector.SURF);
		
		MatOfKeyPoint objectKeyPoints = new MatOfKeyPoint();
		MatOfKeyPoint sceneKeyPoints = new MatOfKeyPoint();
		
		surf.detect(objectImageMat, objectKeyPoints);
		surf.detect(sceneImageMat, sceneKeyPoints);
		
		DescriptorExtractor surfExtractor = DescriptorExtractor.create(DescriptorExtractor.SURF);
		Mat objectDescriptor = new Mat();
		Mat sceneDescriptor = new Mat();
		surfExtractor.compute(objectImageMat, objectKeyPoints, objectDescriptor);
		surfExtractor.compute(sceneImageMat, sceneKeyPoints, sceneDescriptor);
		
		// http://stackoverflow.com/questions/29828849/flann-for-opencv-java
		DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.FLANNBASED);
		MatOfDMatch matches = new MatOfDMatch();
		
		if (objectKeyPoints.toList().size() == 0) {
			throw new ImageSearchException("No keypoints in object to search, check it's not uniformly coloured: " + objectImage.getAbsolutePath());
		}
		if (sceneKeyPoints.toList().size() == 0) {
			throw new ImageSearchException("No keypoints in scene, check it's not uniformly coloured: " + sceneImage.getAbsolutePath());
		}
		
		matcher.match( objectDescriptor, sceneDescriptor, matches );
		
		double maxDist = 0; 
		double minDist = 100;
		
		for( int i = 0; i < objectDescriptor.rows(); i++ ) { 
			double dist = matches.toList().get(i).distance;
		    if ( dist < minDist ) {
		    	minDist = dist;
		    }
		    if( dist > maxDist )  {
		    	maxDist = dist;
		    }
		}
		
		logger.debug("-- Max dist : " + maxDist);
		logger.debug("-- Min dist : " + minDist);
		
		LinkedList goodMatches = new LinkedList<>();
		MatOfDMatch gm = new MatOfDMatch();

		for(int i = 0; i < objectDescriptor.rows(); i++){
		    if(matches.toList().get(i).distance < detectionThreshold){
		        goodMatches.addLast(matches.toList().get(i));
		    }
		}
		gm.fromList(goodMatches);

		Features2d.drawMatches(objectImageMat, objectKeyPoints, sceneImageMat, sceneKeyPoints, 
				gm, imgMatch, Scalar.all(-1), Scalar.all(-1), new MatOfByte(), Features2d.NOT_DRAW_SINGLE_POINTS);
		
		if (goodMatches.isEmpty()) {
			throw new ImageSearchException("Cannot find matching zone");
		}
		
		LinkedList objList = new LinkedList<>();
		LinkedList sceneList = new LinkedList<>();

		List objectKeyPointsList = objectKeyPoints.toList();
		List sceneKeyPointsList = sceneKeyPoints.toList();

		for(int i = 0; i 0.1) {
			throw new ImageSearchException("Aspect ratio between source and detected image is not the same");
		} else {
			logger.debug("Transform ratio is " + Math.round(widthRatio * 100) / 100.0);
			sizeRatio = widthRatio;
		}
	}
	
	/**
	 * Record detected zone as a rectangle
	 * Take into account the rotating angle so that resulting rectangle correspond to points (origin point depends on rotation)
	 * @param p1	corner corresponding to top left corner of origin possibly rotated
	 * @param p2	corner corresponding to top right corner of origin possibly rotated
	 * @param p3	corner corresponding to bottom right corner of origin possibly rotated
	 * @param p4	corner corresponding to bottom left corner of origin possibly rotated
	 */
	protected void recordDetectedRectangle(Point p1, Point p2, Point p3, Point p4) {
		switch ((int)rotationAngle) {
		case 0:
			detectedRectangle = new Rectangle((int)p1.x, (int)p1.y, (int)Math.abs(p4.y - p1.y), (int)Math.abs(p2.x - p1.x));
			break;
		case 90:
			detectedRectangle = new Rectangle((int)p4.x, (int)p4.y, (int)Math.abs(p3.y - p4.y), (int)Math.abs(p1.x - p4.x));
			break;
		case 180:
			detectedRectangle = new Rectangle((int)p3.x, (int)p3.y, (int)Math.abs(p2.y - p3.y), (int)Math.abs(p4.x - p3.x));
			break;
		case 270:
			detectedRectangle = new Rectangle((int)p2.x, (int)p2.y, (int)Math.abs(p1.y - p2.y), (int)Math.abs(p3.x - p2.x));
			break;
		default:
			break;
		}
	}
	
	private void showResultingPicture() throws IOException {
		String tempFile = File.createTempFile("img", ".png").getAbsolutePath();
		writeComparisonPictureToFile(tempFile);
		showResultingImage(tempFile);
	}
	
	/**
	 * File path should end with an image extension (jpg, png)
	 * @param filePath
	 */
	public void writeComparisonPictureToFile(String filePath) {
		if (filePath.toLowerCase().endsWith(".jpg") || filePath.toLowerCase().endsWith(".png")) {
			Highgui.imwrite(filePath, imgMatch);
		} else {
			throw new ImageSearchException("only .JPG and .PNG files are supported");
		}
	}
	
	/**
	 * Method to display the result of image detection
	 * @param imgStr
	 * @param m
	 */
	public void showResultingImage(String filePath) {
		JFrame frame = new JFrame("My GUI");
		frame.setDefaultCloseOperation(WindowConstants.DISPOSE_ON_CLOSE);
	
		frame.setResizable(true);
		frame.setLocationRelativeTo(null);
	
		// Inserts the image icon
		ImageIcon image = new ImageIcon(filePath);
		frame.setSize(image.getIconWidth()+10,image.getIconHeight()+35);
		// Draw the Image data into the BufferedImage
		JLabel label1 = new JLabel(" ", image, JLabel.CENTER);
		frame.getContentPane().add(label1);
	
		frame.validate();
		frame.setVisible(true);
	}
	

	public Rectangle getDetectedRectangle() {
		return detectedRectangle;
	}

	public boolean isComputed() {
		return computed;
	}

	public long getRotationAngle() {
		return rotationAngle;
	}

	public void setRotationAngle(long rotationAngle) {
		this.rotationAngle = rotationAngle;
	}

	/**
	 * Returns the ratio between the detected image (in scene) and the source image (to find)
	 * @return
	 */
	public double getSizeRatio() {
		return sizeRatio;
	}

	public void setDebug(boolean debug) {
		this.debug = debug;
	}

	public void setSceneImage(File sceneImage) {
		if (!sceneImage.exists()) {
			throw new ImageSearchException(String.format("File for object to detect %s does not exist", sceneImage));
		}
		this.sceneImage = sceneImage;
	}

	public void setObjectImage(File objectImage) {
		if (!objectImage.exists()) {
			throw new ImageSearchException(String.format("File for scene to detect object in %s does not exist", objectImage));
		}
		this.objectImage = objectImage;
	}
}