• Home
• org.bytedeco
• javacv
• 0.11
• source code
• GNImageAligner.java

×

Project download

Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $

You can buy this project and download/modify it how often you want.

org.bytedeco.javacv.GNImageAligner Maven / Gradle / Ivy

/*
 * Copyright (C) 2009,2010,2011,2012 Samuel Audet
 *
 * This file is part of JavaCV.
 *
 * JavaCV is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 2 of the License, or
 * (at your option) any later version (subject to the "Classpath" exception
 * as provided in the LICENSE.txt file that accompanied this code).
 *
 * JavaCV is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with JavaCV.  If not, see .
 */

package org.bytedeco.javacv;

import java.util.Arrays;
import org.bytedeco.javacv.ImageTransformer.Data;
import org.bytedeco.javacv.ImageTransformer.Parameters;

import static org.bytedeco.javacpp.opencv_core.*;
import static org.bytedeco.javacpp.opencv_imgproc.*;

/**
 *
 * @author Samuel Audet
 */
public class GNImageAligner implements ImageAligner {
    public GNImageAligner(ImageTransformer transformer, Parameters initialParameters,
            IplImage template0, double[] roiPts, IplImage target0) {
        this(transformer, initialParameters, template0, roiPts, target0, new Settings());
    }
    public GNImageAligner(ImageTransformer transformer, Parameters initialParameters,
            IplImage template0, double[] roiPts, IplImage target0, Settings settings) {
        this(transformer, initialParameters);
        setSettings(settings);

        final int minLevel = settings.pyramidLevelMin;
        final int maxLevel = settings.pyramidLevelMax;

        this.template    = new IplImage[maxLevel+1];
        this.target      = new IplImage[maxLevel+1];
        this.transformed = new IplImage[maxLevel+1];
        this.residual    = new IplImage[maxLevel+1];
        this.mask        = new IplImage[maxLevel+1];
        int w = template0 != null ? template0.width()     : target0.width();
        int h = template0 != null ? template0.height()    : target0.height();
        int c = template0 != null ? template0.nChannels() : target0.nChannels();
        int o = template0 != null ? template0.origin()    : target0.origin();
        for (int i = minLevel; i <= maxLevel; i++) {
            if (i == minLevel && (template0 != null && template0.depth() == IPL_DEPTH_32F)) {
                template[i] = template0;
            } else {
                template[i] = IplImage.create(w, h, IPL_DEPTH_32F, c, o);
            }
            if (i == minLevel && (target0 != null && target0.depth() == IPL_DEPTH_32F)) {
                target[i] = target0;
            } else {
                target[i] = IplImage.create(w, h, IPL_DEPTH_32F, c, o);
            }
            transformed[i] = IplImage.create(w, h, IPL_DEPTH_32F, c, o);
            residual   [i] = IplImage.create(w, h, IPL_DEPTH_32F, c, o);
            mask       [i] = IplImage.create(w, h, IPL_DEPTH_8U,  1, o);
            w /= 2;
            h /= 2;
        }

        this.hessianGradientTransformerData = new Data[n];
        for (int i = 0; i < n; i++) {
            hessianGradientTransformerData[i] = new Data(template[pyramidLevel],
                    transformed[pyramidLevel], residual[pyramidLevel], mask[pyramidLevel],
                    0, 0, pyramidLevel, null, null, n);
        }
        this.residualTransformerData = new Data[] { new Data(template[pyramidLevel],
                    target[pyramidLevel], null, mask[pyramidLevel],
                    0, 0, pyramidLevel, transformed[pyramidLevel], residual[pyramidLevel], 1) };

        setConstrained(settings.constrained);
        setTemplateImage(template0, roiPts);
        setTargetImage(target0);
    }
    protected GNImageAligner(ImageTransformer transformer, Parameters initialParameters) {
        this.n = initialParameters.size();

        this.srcRoiPts = CvMat.create(4, 1, CV_64F, 2);
        this.dstRoiPts = CvMat.create(4, 1, CV_64F, 2);
        this.dstRoiPtsArray = new CvPoint(4);
        this.roi     = new CvRect();
        this.temproi = new CvRect();
        this.transformer = transformer;

        this.parameters      = initialParameters.clone();
        this.parametersArray = new Parameters[] { parameters };
        this.tempParameters  = new Parameters[n];
        for (int i = 0; i < tempParameters.length; i++) {
            this.tempParameters[i] = initialParameters.clone();
        }

        subspaceParameters = parameters.getSubspace();
        if (subspaceParameters != null) {
            tempSubspaceParameters = new double[Parallel.getNumThreads()][];
            for (int i = 0; i < tempSubspaceParameters.length; i++) {
                tempSubspaceParameters[i] = subspaceParameters.clone();
            }
//        for (double d : subspaceParameters) {
//            System.out.print(d + " ");
//        }
//        System.out.println();
        }
    }

    public static class Settings extends ImageAligner.Settings implements Cloneable {
        public Settings() { }
        public Settings(Settings s) {
            super(s);
            stepSize        = s.stepSize;
            lineSearch      = s.lineSearch;
            deltaMin        = s.deltaMin;
            deltaMax        = s.deltaMax;
            displacementMax = s.displacementMax;
            alphaSubspace   = s.alphaSubspace;
            alphaTikhonov   = s.alphaTikhonov;
            gammaTgamma     = s.gammaTgamma;
            constrained     = s.constrained;
        }

        double stepSize        = 0.1;
        double[] lineSearch    = {1.0, 0.25};
        double deltaMin        = 10;
        double deltaMax        = 300;
        double displacementMax = 0.2;
        double alphaSubspace   = 0.1;
        double alphaTikhonov   = 0;
        CvMat gammaTgamma      = null;
        boolean constrained    = false;

        public double getStepSize() {
            return stepSize;
        }
        public void setStepSize(double stepSize) {
            this.stepSize = stepSize;
        }

        public double[] getLineSearch() {
            return lineSearch;
        }
        public void setLineSearch(double[] lineSearch) {
            this.lineSearch = lineSearch;
        }

        public double getDeltaMin() {
            return deltaMin;
        }
        public void setDeltaMin(double deltaMin) {
            this.deltaMin = deltaMin;
        }

        public double getDeltaMax() {
            return deltaMax;
        }
        public void setDeltaMax(double deltaMax) {
            this.deltaMax = deltaMax;
        }

        public double getDisplacementMax() {
            return displacementMax;
        }
        public void setDisplacementMax(double displacementMax) {
            this.displacementMax = displacementMax;
        }

        public double getAlphaSubspace() {
            return alphaSubspace;
        }
        public void setAlphaSubspace(double alphaSubspace) {
            this.alphaSubspace = alphaSubspace;
        }

        public double getAlphaTikhonov() {
            return alphaTikhonov;
        }
        public void setAlphaTikhonov(double alphaTikhonov) {
            this.alphaTikhonov = alphaTikhonov;
        }

        public CvMat getGammaTgamma() {
            return gammaTgamma;
        }
        public void setGammaTgamma(CvMat gammaTgamma) {
            this.gammaTgamma = gammaTgamma;
        }

//        public boolean isConstrained() {
//            return constrained;
//        }
//        public void setConstrained(boolean constrained) {
//            this.constrained = constrained;
//        }

        @Override public Settings clone() {
            return new Settings(this);
        }
    }

    protected Settings settings;
    public Settings getSettings() {
        return settings;
    }
    public void setSettings(ImageAligner.Settings settings) {
        this.settings = (Settings)settings;
    }

    protected final int n;
    protected IplImage[] template, target, transformed, residual, mask;
    protected IplImage[] images = new IplImage[5];
    protected CvMat srcRoiPts, dstRoiPts;
    protected CvPoint dstRoiPtsArray;
    protected CvRect roi, temproi;
    protected ImageTransformer transformer;
    protected Data[] hessianGradientTransformerData, residualTransformerData;
    protected Parameters parameters, parametersArray[], tempParameters[], priorParameters;
    protected CvMat hessian, gradient, update, prior;
    protected double[] constraintGrad, subspaceResidual, subspaceJacobian[], updateScale;
    protected boolean[] subspaceCorrelated;
    protected int pyramidLevel;
    protected double RMSE;
    protected boolean residualUpdateNeeded = true;
    protected int lastLinePosition = 0;
    protected int trials = 0;
//    protected double prevOutlierRatio = 0;

    protected double[] subspaceParameters, tempSubspaceParameters[];

    public IplImage getTemplateImage() {
        return template[pyramidLevel];
    }
    public void setTemplateImage(IplImage template0, double[] roiPts) {
        final int minLevel = settings.pyramidLevelMin;
        final int maxLevel = settings.pyramidLevelMax;

        if (roiPts == null && template0 != null) {
            int w = template0.width()  << minLevel;
            int h = template0.height() << minLevel;
            this.srcRoiPts.put(0.0, 0.0,  w, 0.0,  w, h,  0, h);
        } else if (roiPts != null) {
            this.srcRoiPts.put(roiPts);
        }

        if (template0 == null) {
            return;
        }

        if (template0.depth() == IPL_DEPTH_32F) {
            template[minLevel] = template0;
        } else {
            cvConvertScale(template0, template[minLevel], 1.0/template0.highValue(), 0);
        }

        for (int i = minLevel+1; i <= maxLevel; i++) {
            cvPyrDown(template[i-1], template[i], CV_GAUSSIAN_5x5);
        }
        setPyramidLevel(maxLevel);
    }

    public IplImage getTargetImage() {
        return target[pyramidLevel];
    }
    public void setTargetImage(IplImage target0) {
        final int minLevel = settings.pyramidLevelMin;
        final int maxLevel = settings.pyramidLevelMax;

        if (target0 == null) {
            return;
        }

        if (target0.depth() == IPL_DEPTH_32F) {
            target[minLevel] = target0;
        }

        if (settings.displacementMax > 0) {
            transformer.transform(srcRoiPts, dstRoiPts, parameters, false);
            double[] pts = dstRoiPts.get();
            for (int i = 0; i < pts.length; i++) {
                pts[i] /= (1< prevRMSE; j++) {
            RMSE = prevRMSE;
            parameters.set(resetParameters);
            if (subspaceParameters != null) {
                System.arraycopy(resetSubspaceParameters, 0, subspaceParameters, 0, subspaceParameters.length);
            }
            lastLinePosition = j;
            for (int i = 0; i < n; i++) {
                parameters.set(i, parameters.get(i) + settings.lineSearch[j]*update.get(i)*updateScale[i]);
            }
            for (int i = n; i < update.length(); i++) {
                subspaceParameters[i-n] += settings.lineSearch[j]*update.get(i)*updateScale[i];
            }
            residualUpdateNeeded = true;
        }

        double deltaNorm = 0;
        if (delta != null) {
            for (int i = 0; i < delta.length && i < updateScale.length; i++) {
                delta[i] = settings.lineSearch[lastLinePosition]*update.get(i)*updateScale[i];
            }
            deltaNorm = JavaCV.norm(Arrays.copyOf(delta, n));
        }

        boolean invalid = getRMSE() > prevRMSE || deltaNorm > settings.deltaMax ||
                          Double.isNaN(RMSE) || Double.isInfinite(RMSE);
        if (invalid) {
            RMSE = prevRMSE;
            parameters.set(prevParameters);
            if (subspaceParameters != null) {
                System.arraycopy(prevSubspaceParameters, 0, subspaceParameters, 0, subspaceParameters.length);
            }
            residualUpdateNeeded = true;
        }
        if (invalid && deltaNorm > settings.deltaMin && ++trials < 2) {
            return false;
        } else if (invalid || deltaNorm < settings.deltaMin) {
            trials = 0;
            if (pyramidLevel > settings.pyramidLevelMin) {
                setPyramidLevel(pyramidLevel-1);
            } else {
                converged = true;
            }
        } else {
            trials = 0;
        }
        return converged;
    }

    protected void doHessianGradient(final double[] scale) {
        final double constraintError = parameters.getConstraintError();
        final double stepSize = settings.stepSize;

        cvSetZero(gradient);
        cvSetZero(hessian);

        Parallel.loop(0, n, new Parallel.Looper() {
        public void loop(int from, int to, int looperID) {
//        for (int i = 0; i < n; i++) {
        for (int i = from; i < to; i++) {
            tempParameters[i].set(parameters);
            tempParameters[i].set(i, tempParameters[i].get(i) + /*(1< 0 &&
                settings.alphaSubspace != 0.0) {
            final int m = subspaceParameters.length;
//            double[][] subspaceHessian  = new double[n+m][n+m];
//            double[] subspaceGradient   = new double[n+m];

            Arrays.fill(subspaceCorrelated, false);
            tempParameters[0].set(parameters);
            tempParameters[0].setSubspace(subspaceParameters);
            Parallel.loop(0, n+m, tempSubspaceParameters.length, new Parallel.Looper() {
            public void loop(int from, int to, int looperID) {
//            int looperID = 0;
//            for (int i = 0; i < n+m; i++) {
            for (int i = from; i < to; i++) {
                if (i < n) {
                    Arrays.fill(subspaceJacobian[i], 0);
                    subspaceJacobian[i][i] = scale[i];
                } else {
                    System.arraycopy(subspaceParameters, 0, tempSubspaceParameters[looperID], 0, m);
                    tempSubspaceParameters[looperID][i-n] += stepSize;
                    tempParameters[i-n+1].set(parameters);
                    tempParameters[i-n+1].setSubspace(tempSubspaceParameters[looperID]);
                    scale[i] = tempSubspaceParameters[looperID][i-n] - subspaceParameters[i-n];
                    for (int j = 0; j < n; j++) {
                        subspaceJacobian[i][j] = tempParameters[0].get(j) - tempParameters[i-n+1].get(j);
                        subspaceCorrelated[j] |= subspaceJacobian[i][j] != 0; // this may not work in parallel...
                    }
                }
            }}});

            int subspaceCorrelatedCount = 0;
//            double subspaceRMSE = 0;
            for (int i = 0; i < n; i++) {
                subspaceResidual[i] = parameters.get(i) - tempParameters[0].get(i);
//                subspaceRMSE += subspaceResidual[i]*subspaceResidual[i];

                if (subspaceCorrelated[i]) {
                    subspaceCorrelatedCount++;
                }
            }
//            subspaceRMSE = Math.sqrt(subspaceRMSE/n);
//System.out.println((float)RMSE + " " + (float)subspaceRMSE);
            final double K = settings.alphaSubspace*settings.alphaSubspace * RMSE*RMSE/
                    subspaceCorrelatedCount;//(subspaceRMSE*subspaceRMSE);

            Parallel.loop(0, n+m, new Parallel.Looper() {
            public void loop(int from, int to, int looperID) {
//            int looperID = 0;
//            for (int i = 0; i < n+m; i++) {
            for (int i = from; i < to; i++) {
                if (i < n && !subspaceCorrelated[i]) {
                    continue;
                }

                for (int j = i; j < n+m; j++) {
                    if (j < n && !subspaceCorrelated[j]) {
                        continue;
                    }
                    double h = 0;
                    for (int k = 0; k < n; k++) {
                        h += subspaceJacobian[i][k]*subspaceJacobian[j][k];
                    }
//                    subspaceHessian[i][j] = h;
                    h = hessian.get(i, j) + K*h;
                    hessian.put(i, j, h);
                    hessian.put(j, i, h);
                }

                double g = 0;
                for (int k = 0; k < n; k++) {
                    g -= subspaceJacobian[i][k]*subspaceResidual[k];
                }
//                subspaceGradient[i] = g;
                g = gradient.get(i) + K*g;
                gradient.put(i, g);
            }}});
        }

        // add Tikhonov regularization
        int rows = hessian.rows(), cols = hessian.cols();
        for (int i = 0; i < rows; i++) {
            for (int j = 0; j < cols; j++) {
                double h = hessian.get(i, j);
                double g = 0;
                if (settings.gammaTgamma != null && i < settings.gammaTgamma.rows() && j < settings.gammaTgamma.cols()) {
                    g = settings.gammaTgamma.get(i, j);
                }
                double a = 0;
                if (i == j && i < n) {
                    a = settings.alphaTikhonov * settings.alphaTikhonov;
                }
                hessian.put(i, j, h + g + a);
            }
        }
    }

    protected void doRoi() {
        transformer.transform(srcRoiPts, dstRoiPts, parameters, false);
        double[] pts = dstRoiPts.get();
        for (int i = 0; i < pts.length; i++) {
            pts[i] /= (1<