org.sikuli.script.ImageFind Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of sikulixapi Show documentation
Show all versions of sikulixapi Show documentation
... for visual testing and automation
/*
* Copyright 2010-2014, Sikuli.org, sikulix.com
* Released under the MIT License.
*
* modified RaiMan
*/
package org.sikuli.script;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.MatOfDouble;
import org.opencv.core.Rect;
import org.opencv.core.Size;
import org.opencv.imgproc.Imgproc;
import org.sikuli.basics.Debug;
import org.sikuli.basics.Settings;
/**
* UNDER DEVELOPMENT --- SURELY HAS BUGS ;-)
* Intended replacement for Finder together with ImageFinder
* completely implementing the OpenCV usage on the Java level.
*/
public class ImageFind implements Iterator{
private static String me = "ImageFind: ";
private static int lvl = 3;
private static void log(int level, String message, Object... args) {
Debug.logx(level, me + message, args);
}
private ImageFinder owner = null;
private boolean isValid = false;
private boolean isInnerFind = false;
private Image pImage = null;
private Mat probe = new Mat();
private boolean isPlainColor = false;
private boolean isBlack = false;
private double similarity = Settings.MinSimilarity;
private double waitingTime = Settings.AutoWaitTimeout;
private boolean shouldCheckLastSeen = Settings.CheckLastSeen;
private Object[] findArgs = null;
private int resizeMinDownSample = 12;
private double resizeFactor;
private float[] resizeLevels = new float[] {1f, 0.4f};
private int resizeMaxLevel = resizeLevels.length - 1;
private double resizeMinSim = 0.9;
private double resizeMinFactor = 1.5;
private Core.MinMaxLocResult findDownRes = null;
private int sorted;
public static final int AS_ROWS = 0;
public static final int AS_COLUMNS = 1;
public static final int BEST_FIRST = 2;
private int finding = -1;
public static final int FINDING_ANY = 0;
public static final int FINDING_SOME = 1;
public static final int FINDING_ALL = 2;
private int count = 0;
public static int SOME_COUNT = 5;
public static int ALL_MAX = 100;
private int allMax = 0;
private List matches = Collections.synchronizedList(new ArrayList());
private boolean repeating;
private long lastFindTime = 0;
private long lastSearchTime = 0;
public ImageFind() {
matches.add(null);
}
public boolean isValid() {
return true;
}
public void setIsInnerFind() {
isInnerFind = true;
}
void setSimilarity(double sim) {
similarity = sim;
}
public void setFindTimeout(double t) {
waitingTime = t;
}
public void setFinding(int ftyp) {
finding = ftyp;
}
public void setSorted(int styp) {
sorted = styp;
}
public void setCount(int c) {
count = c;
}
public List getMatches() {
return matches;
}
protected boolean checkFind(ImageFinder owner, Object pprobe, Object... args) {
if (owner.isValid()) {
this.owner = owner;
} else {
return false;
}
isValid = false;
shouldCheckLastSeen = Settings.CheckLastSeen;
if (pprobe instanceof String) {
pImage = Image.create((String) pprobe);
if (pImage.isValid()) {
isValid = true;
}
} else if (pprobe instanceof Image) {
if (((Image) pprobe).isValid()) {
isValid = true;
pImage = (Image) pprobe;
}
} else if (pprobe instanceof Pattern) {
if (((Pattern) pprobe).getImage().isValid()) {
isValid = true;
pImage = ((Pattern) pprobe).getImage();
similarity = ((Pattern) pprobe).getSimilar();
}
} else if (pprobe instanceof Mat) {
isValid = true;
probe = (Mat) pprobe;
waitingTime = 0.0;
shouldCheckLastSeen = false;
} else {
log(-1, "find(... some, any, all): probe invalid (not Pattern, String nor valid Image)");
return false;
}
if (probe.empty()) {
probe = Image.createMat(pImage.get());
}
checkProbe();
if (!owner.isImage()) {
if (args.length > 0) {
if (args[0] instanceof Integer) {
waitingTime = 0.0 + (Integer) args[0];
} else if (args[0] instanceof Double) {
waitingTime = (Double) args[0];
}
}
if (args.length > 1) {
findArgs = Arrays.copyOfRange(args, 1, args.length);
} else {
findArgs = null;
}
}
return isValid;
}
private void checkProbe() {
MatOfDouble pMean = new MatOfDouble();
MatOfDouble pStdDev = new MatOfDouble();
Core.meanStdDev(probe, pMean, pStdDev);
double min = 0.00001;
isPlainColor = false;
double sum = 0.0;
double arr[] = pStdDev.toArray();
for (int i = 0; i < arr.length; i++) {
sum += arr[i];
}
if (sum < min) {
isPlainColor = true;
}
sum = 0.0;
arr = pMean.toArray();
for (int i = 0; i < arr.length; i++) {
sum += arr[i];
}
if (sum < min && isPlainColor) {
isBlack = true;
}
resizeFactor = Math.min(((double) probe.width())/resizeMinDownSample, ((double) probe.height())/resizeMinDownSample);
resizeFactor = Math.max(1.0, resizeFactor);
}
protected ImageFind doFind() {
Debug.enter(me + ": doFind");
Core.MinMaxLocResult fres = null;
repeating = false;
long begin = (new Date()).getTime();
long lap;
while (true) {
lastFindTime = (new Date()).getTime();
if (shouldCheckLastSeen && !repeating && !owner.isImage && pImage.getLastSeen() != null) {
log(3, "checkLastSeen: trying ...");
ImageFinder f = new ImageFinder(new Region(pImage.getLastSeen()));
if (null != f.findInner(probe, pImage.getLastSeenScore() - 0.01)) {
log(lvl, "checkLastSeen: success");
set(f.next());
if (pImage != null) {
pImage.setLastSeen(get().getRect(), get().getScore());
}
break;
}
log(lvl, "checkLastSeen: not found");
}
if (!owner.isMultiFinder || owner.base.empty()) {
if (owner.isRegion) {
owner.setBase(owner.region.getScreen().capture(owner.region).getImage());
} else if (owner.isScreen) {
owner.setBase(owner.screen.capture().getImage());
}
}
if (!isInnerFind && resizeFactor > resizeMinFactor) {
log(3, "downsampling: trying ...");
doFindDown(0, resizeFactor);
fres = findDownRes;
}
if (fres == null) {
if (!isInnerFind) {
log(3, "downsampling: not found with (%f) - trying original size", resizeFactor);
}
fres = doFindDown(0, 0.0);
if(fres != null && fres.maxVal > similarity - 0.01) {
set(new Match((int) fres.maxLoc.x + owner.offX, (int) fres.maxLoc.y + owner.offY,
probe.width(), probe.height(), fres.maxVal, null, null));
}
} else {
log(lvl, "downsampling: success: adjusting match");
set(checkFound(fres));
}
lastFindTime = (new Date()).getTime() - lastFindTime;
if (hasNext()) {
get().setTimes(lastFindTime, lastSearchTime);
if (pImage != null) {
pImage.setLastSeen(get().getRect(), get().getScore());
}
break;
} else {
if (isInnerFind || owner.isImage()) {
break;
}
else {
if (waitingTime < 0.001 || (lap = (new Date()).getTime() - begin) > waitingTime * 1000) {
break;
}
if (owner.MaxTimePerScan > lap) {
try {
Thread.sleep(owner.MaxTimePerScan - lap);
} catch (Exception ex) {
}
}
repeating = true;
}
}
}
return this;
}
private Match checkFound(Core.MinMaxLocResult res) {
Match match = null;
ImageFinder f;
Rect r = null;
if (owner.isImage()) {
int off = ((int) resizeFactor) + 1;
r = getSubMatRect(owner.base, (int) res.maxLoc.x, (int) res.maxLoc.y,
probe.width(), probe.height(), off);
f = new ImageFinder(owner.base.submat(r));
} else {
f = new ImageFinder((new Region((int) res.maxLoc.x + owner.offX, (int) res.maxLoc.y + owner.offY,
probe.width(), probe.height())).grow(((int) resizeFactor) + 1));
}
if (null != f.findInner(probe, similarity)) {
log(lvl, "check after downsampling: success");
match = f.next();
if (owner.isImage()) {
match.x += r.x;
match.y += r.y;
}
}
return match;
}
private static Rect getSubMatRect(Mat mat, int x, int y, int w, int h, int margin) {
x = Math.max(0, x - margin);
y = Math.max(0, y - margin);
w = Math.min(w + 2 * margin, mat.width() - x);
h = Math.min(h + 2 * margin, mat.height()- y);
return new Rect(x, y, w, h);
}
private Core.MinMaxLocResult doFindDown(int level, double factor) {
Debug.enter(me + ": doFindDown (%d - 1/%.2f)", level, factor * resizeLevels[level]);
Debug timer = Debug.startTimer("doFindDown");
Mat b = new Mat();
Mat p = new Mat();
Core.MinMaxLocResult dres = null;
double rfactor;
if (factor > 0.0) {
rfactor = factor * resizeLevels[level];
if (rfactor < resizeMinFactor) return null;
Size sb = new Size(owner.base.cols()/rfactor, owner.base.rows()/factor);
Size sp = new Size(probe.cols()/rfactor, probe.rows()/factor);
Imgproc.resize(owner.base, b, sb, 0, 0, Imgproc.INTER_AREA);
Imgproc.resize(probe, p, sp, 0, 0, Imgproc.INTER_AREA);
dres = doFindMatch(b, p);
log(lvl, "doFindDown: score: %.2f at (%d, %d)", dres.maxVal,
(int) (dres.maxLoc.x * rfactor), (int) (dres.maxLoc.y * rfactor));
} else {
dres = doFindMatch(owner.base, probe);
timer.end();
return dres;
}
if (dres.maxVal < resizeMinSim) {
if (level == resizeMaxLevel) {
timer.end();
return null;
}
if (level == 0) {
findDownRes = null;
}
level++;
doFindDown(level, factor);
} else {
dres.maxLoc.x *= rfactor;
dres.maxLoc.y *= rfactor;
findDownRes = dres;
}
timer.end();
return null;
}
private Core.MinMaxLocResult doFindMatch(Mat base, Mat probe) {
Mat res = new Mat();
Mat bi = new Mat();
Mat pi = new Mat();
if (!isPlainColor) {
Imgproc.matchTemplate(base, probe, res, Imgproc.TM_CCOEFF_NORMED);
} else {
if (isBlack) {
Core.bitwise_not(base, bi);
Core.bitwise_not(probe, pi);
} else {
bi = base;
pi = probe;
}
Imgproc.matchTemplate(bi, pi, res, Imgproc.TM_SQDIFF_NORMED);
Core.subtract(Mat.ones(res.size(), CvType.CV_32F), res, res);
}
return Core.minMaxLoc(res);
}
@Override
public boolean hasNext() {
if (matches.size() > 0) {
return matches.get(0) != null;
}
return false;
}
@Override
public Match next() {
Match m = null;
if (matches.size() > 0) {
m = matches.get(0);
remove();
}
return m;
}
@Override
public void remove() {
if (matches.size() > 0) {
matches.remove(0);
}
}
public Match get() {
return get(0);
}
public Match get(int n) {
if (n < matches.size()) {
return matches.get(n);
}
return null;
}
private Match add(Match m) {
if (matches.add(m)) {
return m;
}
return null;
}
private Match set(Match m) {
if (matches.size() > 0) {
matches.set(0, m);
} else {
matches.add(m);
}
return m;
}
public int getSize() {
return matches.size();
}
}