com.simiacryptus.mindseye.art.ops.PatternPCAMatcher Maven / Gradle / Ivy

Go to download
/*
 * Copyright (c) 2019 by Andrew Charneski.
 *
 * The author licenses this file to you under the
 * Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance
 * with the License.  You may obtain a copy
 * of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.simiacryptus.mindseye.art.ops;

import com.simiacryptus.mindseye.art.VisualModifier;
import com.simiacryptus.mindseye.art.VisualModifierParameters;
import com.simiacryptus.mindseye.art.util.PCA;
import com.simiacryptus.mindseye.lang.Layer;
import com.simiacryptus.mindseye.lang.Result;
import com.simiacryptus.mindseye.lang.Tensor;
import com.simiacryptus.mindseye.layers.ValueLayer;
import com.simiacryptus.mindseye.layers.cudnn.*;
import com.simiacryptus.mindseye.layers.cudnn.conv.ConvolutionLayer;
import com.simiacryptus.mindseye.layers.java.BoundedActivationLayer;
import com.simiacryptus.mindseye.layers.java.ImgBandScaleLayer;
import com.simiacryptus.mindseye.layers.java.NthPowerActivationLayer;
import com.simiacryptus.mindseye.network.DAGNode;
import com.simiacryptus.mindseye.network.PipelineNetwork;
import com.simiacryptus.ref.wrappers.RefArrays;
import com.simiacryptus.ref.wrappers.RefIntStream;
import com.simiacryptus.ref.wrappers.RefList;
import com.simiacryptus.ref.wrappers.RefString;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nonnull;

public class PatternPCAMatcher implements VisualModifier {
  private static final Logger log = LoggerFactory.getLogger(PatternPCAMatcher.class);
  private int minValue = -1;
  private int maxValue = 1;
  private boolean averaging = true;
  private int bands = 16;

  public int getBands() {
    return bands;
  }

  @Nonnull
  public PatternPCAMatcher setBands(int bands) {
    this.bands = bands;
    return this;
  }

  public int getMaxValue() {
    return maxValue;
  }

  @Nonnull
  public PatternPCAMatcher setMaxValue(int maxValue) {
    this.maxValue = maxValue;
    return this;
  }

  public int getMinValue() {
    return minValue;
  }

  @Nonnull
  public PatternPCAMatcher setMinValue(int minValue) {
    this.minValue = minValue;
    return this;
  }

  public boolean isAveraging() {
    return averaging;
  }

  @Nonnull
  public PatternPCAMatcher setAveraging(boolean averaging) {
    this.averaging = averaging;
    return this;
  }

  @Nonnull
  @Override
  public PipelineNetwork build(@Nonnull VisualModifierParameters visualModifierParameters) {
    PipelineNetwork network = visualModifierParameters.copyNetwork();
    Tensor baseContent = Result.getData0(network.eval(visualModifierParameters.getStyle()));
    visualModifierParameters.freeRef();
    int[] contentDimensions = baseContent.getDimensions();
    RefList components;
    PipelineNetwork signalProjection;
    try {
      PCA pca = new PCA().setRecenter(true).setRescale(false).setEigenvaluePower(0.0);
      Tensor channelMeans = pca.getChannelMeans(baseContent.addRef());
      Tensor channelRms = pca.getChannelRms(baseContent.addRef(), contentDimensions[2], channelMeans.addRef());
      assert channelRms != null;
      double[] covariance = PCA.bandCovariance(baseContent.getPixelStream(), PCA.countPixels(baseContent.addRef()),
          channelMeans.getData(), channelRms.getData());
      channelMeans.scaleInPlace(-1);
      Tensor map = channelRms.map(x -> 1 / x);
      signalProjection = PipelineNetwork.build(1,
          new ImgBandBiasLayer(channelMeans),
          new ImgBandScaleLayer(map.getData()));
      map.freeRef();
      components = PCA.pca(covariance, pca.getEigenvaluePower());
      channelRms.freeRef();
    } catch (Throwable e) {
      log.info("Error processing PCA for dimensions " + RefArrays.toString(contentDimensions), e);
      PipelineNetwork pipelineNetwork = new PipelineNetwork(1);
      pipelineNetwork.add(new ValueLayer(new Tensor(0.0)), new DAGNode[]{}).freeRef();
      return pipelineNetwork;
    }
    int bands = Math.min(getBands(), contentDimensions[2]);

    Tensor prefixPattern = Result.getData0(signalProjection.eval(baseContent.addRef()));
    ConvolutionLayer convolutionLayer4 = new ConvolutionLayer(1, 1, contentDimensions[2], bands);
    convolutionLayer4.setPaddingXY(0, 0);
    ConvolutionLayer convolutionLayer12 = getConvolutionLayer1(convolutionLayer4, components.addRef(), bands);
    Layer explode2 = convolutionLayer12.explode();
    convolutionLayer12.freeRef();
    channelStats(Result.getData0(explode2.eval(prefixPattern.addRef())), bands);
    explode2.freeRef();
    ConvolutionLayer convolutionLayer3 = new ConvolutionLayer(1, 1, contentDimensions[2], bands);
    convolutionLayer3.setPaddingXY(0, 0);
    ConvolutionLayer convolutionLayer21 = getConvolutionLayer2(convolutionLayer3, components.addRef(), bands);
    Layer explode1 = convolutionLayer21.explode();
    convolutionLayer21.freeRef();
    channelStats(Result.getData0(explode1.eval(prefixPattern)), bands);
    explode1.freeRef();
    ConvolutionLayer convolutionLayer2 = new ConvolutionLayer(1, 1, contentDimensions[2], bands);
    convolutionLayer2.setPaddingXY(0, 0);
    ConvolutionLayer convolutionLayer11 = getConvolutionLayer1(
        convolutionLayer2, components, bands);
    signalProjection.add(convolutionLayer11.explode()).freeRef();
    convolutionLayer11.freeRef();
    Tensor spacialPattern = Result.getData0(signalProjection.eval(baseContent));
    channelStats(spacialPattern.addRef(), bands);

    DAGNode signalProjectionHead = signalProjection.getHead();
    NthPowerActivationLayer nthPowerActivationLayer = new NthPowerActivationLayer();
    nthPowerActivationLayer.setPower(-0.5);
    signalProjection.add(new ProductLayer(), signalProjectionHead, signalProjection.add(PipelineNetwork.build(1,
        new SquareActivationLayer(), new AvgReducerLayer(), nthPowerActivationLayer))).freeRef();
    Tensor tensor = spacialPattern.unit();
    spacialPattern.freeRef();
    tensor.scaleInPlace(-1);
    ConvolutionLayer convolutionLayer1 = new ConvolutionLayer(contentDimensions[0], contentDimensions[1], bands, 1);
    convolutionLayer1.setPaddingXY(0, 0);
    convolutionLayer1.set(tensor);
    Layer explode = convolutionLayer1.explode();
    convolutionLayer1.freeRef();
    signalProjection.add(explode).freeRef();
    BoundedActivationLayer boundedActivationLayer1 = new BoundedActivationLayer();
    boundedActivationLayer1.setMinValue(getMinValue());
    boundedActivationLayer1.setMaxValue(getMaxValue());
    signalProjection.add(boundedActivationLayer1).freeRef();
    final Layer nextHead = isAveraging() ? new AvgReducerLayer() : new SumReducerLayer();
    signalProjection.add(nextHead).freeRef();
    signalProjection.setName(RefString.format("PCA Match"));
    network.add(signalProjection).freeRef();
    network.freeze();
    return network;
  }

  public void channelStats(@Nonnull Tensor spacialPattern, int bands) {
    double[] means = RefIntStream.range(0, bands).mapToDouble(band -> {
      Tensor selectBand = spacialPattern.selectBand(band);
      double mean = selectBand.mean();
      selectBand.freeRef();
      return mean;
    }).toArray();
    double[] stdDevs = RefIntStream.range(0, bands).mapToDouble(band -> {
      Tensor bandPattern = spacialPattern.selectBand(band);
      double sqrt = Math.sqrt(Math.pow(bandPattern.rms(), 2) - Math.pow(bandPattern.mean(), 2));
      bandPattern.freeRef();
      return sqrt;
    }).toArray();
    spacialPattern.freeRef();
    log.info("Means: " + RefArrays.toString(means) + "; StdDev: " + RefArrays.toString(stdDevs));
  }

  @Nonnull
  public ConvolutionLayer getConvolutionLayer1(@Nonnull ConvolutionLayer convolutionLayer, @Nonnull RefList components,
                                               int stride) {
    convolutionLayer.setByCoord(c -> {
      int[] coords = c.getCoords();
      Tensor tensor = components.get(coords[2] % stride);
      double v = tensor.get(coords[2] / stride);
      tensor.freeRef();
      return v;
    });
    components.freeRef();
    return convolutionLayer;
  }

  @Nonnull
  public ConvolutionLayer getConvolutionLayer2(@Nonnull ConvolutionLayer convolutionLayer, @Nonnull RefList components,
                                               int stride) {
    convolutionLayer.setByCoord(c -> {
      int[] coords = c.getCoords();
      Tensor tensor = components.get(coords[2] / stride);
      double v = tensor.get(coords[2] % stride);
      tensor.freeRef();
      return v;
    });
    components.freeRef();
    return convolutionLayer;
  }

}