com.simiacryptus.mindseye.test.data.Caltech101 Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of mindseye-test Show documentation
Testing Tools for Neural Network Components
The newest version!
/*
 * Copyright (c) 2019 by Andrew Charneski.
 *
 * The author licenses this file to you under the
 * Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance
 * with the License.  You may obtain a copy
 * of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package com.simiacryptus.mindseye.test.data;

import com.simiacryptus.mindseye.test.TestUtil;
import com.simiacryptus.ref.wrappers.RefList;
import com.simiacryptus.ref.wrappers.RefStream;
import com.simiacryptus.util.Util;
import com.simiacryptus.util.function.WeakCachedSupplier;
import com.simiacryptus.util.io.DataLoader;
import com.simiacryptus.util.test.LabeledObject;
import org.apache.commons.io.IOUtils;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import javax.imageio.ImageIO;
import java.awt.image.BufferedImage;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

/**
 * The type Caltech 101.
 */
public class Caltech101 {

  @Nullable
  private static final DataLoader>> training = new DataLoader>>() {
    public @SuppressWarnings("unused")
    void _free() {
      super._free();
    }

    @Override
    protected void read(@Nonnull final RefList>> queue) {
      try {
        @Nullable
        InputStream stream = null;
        try {
          // Repackaging as a zip is needed - the tar format classes dont work here
          stream = Util.cacheStream(TestUtil.S3_ROOT.resolve("101_ObjectCategories.zip"));
        } catch (@Nonnull NoSuchAlgorithmException | KeyManagementException e) {
          throw Util.throwException(e);
        }
        final boolean continueLoop = true;
        @Nullable final ZipInputStream tar = new ZipInputStream(stream);
        while (true) {
          if (Thread.interrupted()) {
            break;
          }
          final ZipEntry entry = tar.getNextEntry();
          if (null == entry) {
            System.err.println("Null Entry");
            break;
          }
          if (0 == entry.getSize()) {
            continue;
          }
          final String category = entry.getName().split("/")[1];
          //com.simiacryptus.ref.wrappers.System.err.println(String.format("%s -> %s (%s)", entry.getName(), category, entry.getImageSize()));
          final byte[] data = IOUtils.toByteArray(tar, entry.getSize());
          if (!entry.getName().toLowerCase().endsWith(".jpg")) {
            continue;
          }
          queue.add(new LabeledObject<>(new WeakCachedSupplier<>(() -> {
            try {
              return ImageIO.read(new ByteArrayInputStream(data));
            } catch (@Nonnull final IOException e) {
              throw Util.throwException(e);
            }
          }), category));
        }
      } catch (@Nonnull final IOException e) {
        e.printStackTrace();
        throw Util.throwException(e);
      }
      queue.freeRef();
    }
  };

  /**
   * Halt.
   */
  public static void halt() {
    assert Caltech101.training != null;
    Caltech101.training.stop();
  }

  /**
   * Training data stream ref stream.
   *
   * @return the ref stream
   */
  @Nonnull
  public static RefStream>> trainingDataStream() {
    assert Caltech101.training != null;
    return Caltech101.training.stream();
  }

}