eu.stratosphere.core.testing.TestRecords Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of stratosphere-testplan Show documentation
The newest version!
/***********************************************************************************************************************
 *
 * Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
 * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations under the License.
 *
 **********************************************************************************************************************/

package eu.stratosphere.core.testing;

import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.ints.IntList;

import java.util.ArrayList;
import java.util.List;

import eu.stratosphere.api.common.typeutils.TypeComparator;
import eu.stratosphere.api.common.typeutils.TypeComparatorFactory;
import eu.stratosphere.configuration.Configuration;
import eu.stratosphere.core.testing.fuzzy.DoubleValueSimilarity;
import eu.stratosphere.core.testing.fuzzy.FuzzyValueMatcher;
import eu.stratosphere.core.testing.fuzzy.NaiveFuzzyValueMatcher;
import eu.stratosphere.core.testing.fuzzy.RecordDistance;
import eu.stratosphere.pact.runtime.plugable.pactrecord.RecordComparator;
import eu.stratosphere.pact.runtime.plugable.pactrecord.RecordPairComparatorFactory;
import eu.stratosphere.pact.runtime.plugable.pactrecord.RecordSerializerFactory;
import eu.stratosphere.types.Key;
import eu.stratosphere.types.Record;
import eu.stratosphere.types.Value;

/**
 * Represents the input or output values of a {@link TestPlan}. The class is
 * especially important when setting the expected values in the TestPlan.

 * 

 * There are two ways to specify the values:
 * 
 * From a file: with {@link #load(Class, Configuration)} and {@link #load(Class, String, Configuration)} the
 * location, format, and configuration of the data can be specified. The file is lazily loaded and thus can be
 * comparable large.
 * Ad-hoc: key/value records can be added with {@link #add(Record...)}, {@link #add(Value...)}, and
 * {@link #add(Iterable)}. Please note that the actual amount of records is quite for a test case as the TestPlan
 * already involves a certain degree of overhead.

 * 

 * TestRecords are directly comparable with equals and hashCode based on its content. Please note that in the case of
 * large file-based TestRecords, the time needed to compute the {@link #hashCode()} or to compare two instances with
 * {@link #equals(Object)} can become quite long. Currently, the comparison result is order-dependent as TestRecords are
 * interpreted as a list.
 */
public class TestRecords extends GenericTestRecords {

	/**
	 * Initializes TestRecords with the given schema.
	 */
	public TestRecords(final Class firstEntry, final Class... remainingEntries) {
		super(getRecordConfig(firstEntry, remainingEntries));
	}

	/**
	 * Initializes TestRecords with the given schema.
	 */
	public TestRecords(final Class[] schema) {
		super(getRecordConfig(schema));
	}

	/**
	 * Initializes TestRecords with the given type config.
	 */
	public TestRecords(final TypeConfig typeConfig) {
		super(typeConfig);
	}

	/**
	 * Initializes TestRecords.
	 */
	TestRecords() {
	}

	/*
	 * (non-Javadoc)
	 * @see eu.stratosphere.core.testing.GenericTestRecords#add(eu.stratosphere.core.testing.GenericTestRecords)
	 */
	@Override
	public TestRecords add(final GenericTestRecords records) {
		return (TestRecords) super.add(records);
	}

	/*
	 * (non-Javadoc)
	 * @see eu.stratosphere.core.testing.GenericTestRecords#add(java.lang.Iterable)
	 */
	@Override
	public TestRecords add(final Iterable records) {
		return (TestRecords) super.add(records);
	}

	/*
	 * (non-Javadoc)
	 * @see eu.stratosphere.core.testing.GenericTestRecords#add(T[])
	 */
	@Override
	public TestRecords add(final Record... records) {
		return (TestRecords) super.add(records);
	}

	/**
	 * Adds input or expected values to this {@link TestRecords}.
	 */
	public TestRecords add(final Value... values) {
		final Record pactRecord = new Record();
		for (final Value value : values)
			pactRecord.addField(value);
		this.add(pactRecord);
		return this;
	}

	/**
	 * 
	 */
	@SuppressWarnings("unchecked")
	public void inferTypeConfig() {
		final List records = this.getRecords();
		if (records.isEmpty())
			this.setTypeConfig(getRecordConfig(new Class[0]));
		else {
			final Record first = records.get(0);
			final Class[] schema = new Class[first.getNumFields()];
			for (int index = 0; index < schema.length; index++)
				schema[index] = first.getField(index, Value.class).getClass();
			this.setTypeConfig(getRecordConfig(schema));
		}

	}

	/**
	 * Sets the double delta that is allowed so that {@link DoubleValue}s are treated as equal.
	 */
	public void setAllowedDoubleDelta(final double delta) {
		final TypeConfig typeConfig = this.getTypeConfig().clone();
		final FuzzyValueMatcher fuzzyValueMatcher = typeConfig.getFuzzyValueMatcher();
		final RecordDistance pactRecordDistance;
		if (fuzzyValueMatcher instanceof NaiveFuzzyValueMatcher)
			pactRecordDistance =
				(RecordDistance) ((NaiveFuzzyValueMatcher) fuzzyValueMatcher).getTypeDistance();
		else
			typeConfig.setFuzzyValueMatcher(new NaiveFuzzyValueMatcher(
				pactRecordDistance = new RecordDistance()));

		final Class[] schema = getSchema(typeConfig);
		final DoubleValueSimilarity similarity = new DoubleValueSimilarity(delta);
		final RecordKeyExtractor keyExtractor = (RecordKeyExtractor) typeConfig.getKeyExtractor();
		for (int index = 0; index < schema.length; index++)
			if (similarity.isApplicable(schema[index])) {
				pactRecordDistance.addSimilarity(index, similarity);
				keyExtractor.removeKey(index);
			}
		this.setTypeConfig(typeConfig);
	}

	/**
	 * Sets the double delta that is allowed so that {@link DoubleValue}s are treated as equal.
	 */
	public TestRecords withAllowedDoubleDelta(final double delta) {
		this.setAllowedDoubleDelta(delta);
		return this;
	}

	/**
	 * Creates the type config specific for Stratosphere {@link Record}s with the given schema.
	 * 
	 * @return the type config
	 */
	public static final TypeConfig getRecordConfig(final Class firstEntry,
			final Class... remainingEntries) {
		return getRecordConfig(SchemaUtils.combineSchema(firstEntry, remainingEntries));
	}

	/**
	 * Creates the type config specific for Stratosphere {@link Record}s with the given schema.
	 * 
	 * @return the type config
	 */
	@SuppressWarnings("unchecked")
	public static final TypeConfig getRecordConfig(final Class[] schema) {
		final IntList indices = new IntArrayList();
		final List> keyTypes = new ArrayList>();
		for (int index = 0; index < schema.length; index++)
			if (Key.class.isAssignableFrom(schema[index])) {
				indices.add(index);
				keyTypes.add((Class) schema[index]);
			}

		final RecordKeyExtractor keyExtractor =
			new RecordKeyExtractor(indices, keyTypes);
		final TypeConfig[] typeConfigReference = new TypeConfig[1];
		typeConfigReference[0] =
			new TypeConfig(new RecordComparatorFactory(typeConfigReference),
				RecordPairComparatorFactory.get(),
				RecordSerializerFactory.get(), new RecordStringifier(schema), keyExtractor,
				new RecordEqualer(schema));
		return typeConfigReference[0];
	}

	/**
	 * Returns the schema from the given config.
	 */
	public static final Class[] getSchema(final TypeConfig config) {
		return ((RecordEqualer) config.getEqualer()).getSchema();
	}

	/**
	 */
	private static final class RecordComparatorFactory implements TypeComparatorFactory {
		private final TypeConfig[] typeConfigReference;

		/**
		 * Initializes RecordComparatorFactory.
		 * 
		 * @param typeConfigReference
		 */
		public RecordComparatorFactory(final TypeConfig[] typeConfigReference) {
			this.typeConfigReference = typeConfigReference;
		}

		/*
		 * (non-Javadoc)
		 * @see eu.stratosphere.api.typeutils.TypeComparatorFactory#createComparator()
		 */
		@SuppressWarnings("unchecked")
		@Override
		public TypeComparator createComparator() {
			final TypeConfig typeConfig = this.typeConfigReference[0];
			final RecordKeyExtractor keyExtractor = (RecordKeyExtractor) typeConfig.getKeyExtractor();
			return new RecordComparator(keyExtractor.getIndices().toIntArray(),
				keyExtractor.getKeyClasses().toArray(new Class[0]));
		}

		/*
		 * (non-Javadoc)
		 * @see
		 * eu.stratosphere.api.typeutils.TypeComparatorFactory#readParametersFromConfig(eu.stratosphere.nephele
		 * .configuration.Configuration, java.lang.ClassLoader)
		 */
		@Override
		public void readParametersFromConfig(final Configuration config, final ClassLoader cl)
				throws ClassNotFoundException {
		}

		/*
		 * (non-Javadoc)
		 * @see
		 * eu.stratosphere.api.typeutils.TypeComparatorFactory#writeParametersToConfig(eu.stratosphere.nephele.
		 * configuration.Configuration)
		 */
		@Override
		public void writeParametersToConfig(final Configuration config) {
		}
	}

}