com.google.cloud.dataflow.sdk.testing.CoderProperties Maven / Gradle / Ivy
/*
* Copyright (C) 2015 Google Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
* use this file except in compliance with the License. You may obtain a copy of
* the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations under
* the License.
*/
package com.google.cloud.dataflow.sdk.testing;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.emptyIterable;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.fail;
import com.google.cloud.dataflow.sdk.coders.Coder;
import com.google.cloud.dataflow.sdk.coders.Coder.NonDeterministicException;
import com.google.cloud.dataflow.sdk.coders.CoderException;
import com.google.cloud.dataflow.sdk.util.CoderUtils;
import com.google.cloud.dataflow.sdk.util.PropertyNames;
import com.google.cloud.dataflow.sdk.util.SerializableUtils;
import com.google.cloud.dataflow.sdk.util.Serializer;
import com.google.cloud.dataflow.sdk.util.Structs;
import com.google.common.collect.Iterables;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
* Properties for use in {@link Coder} tests. These are implemented with junit assertions
* rather than as predicates for the sake of error messages.
*
* We serialize and deserialize the coder to make sure that any state information required by the
* coder is preserved. This causes tests written such that coders that lose information during
* serialization or change state during encoding/decoding will fail.
*/
public class CoderProperties {
/**
* All the contexts, for use in test cases.
*/
public static final List ALL_CONTEXTS = Arrays.asList(
Coder.Context.OUTER, Coder.Context.NESTED);
/**
* Verifies that for the given {@code Coder}, and values of
* type {@code T}, if the values are equal then the encoded bytes are equal, in any
* {@code Coder.Context}.
*/
public static void coderDeterministic(
Coder coder, T value1, T value2)
throws Exception {
for (Coder.Context context : ALL_CONTEXTS) {
coderDeterministicInContext(coder, context, value1, value2);
}
}
/**
* Verifies that for the given {@code Coder}, {@code Coder.Context}, and values of
* type {@code T}, if the values are equal then the encoded bytes are equal.
*/
public static void coderDeterministicInContext(
Coder coder, Coder.Context context, T value1, T value2)
throws Exception {
try {
coder.verifyDeterministic();
} catch (NonDeterministicException e) {
fail("Expected that the coder is deterministic");
}
assertThat("Expected that the passed in values are equal()", value1, equalTo(value2));
assertThat(
encode(coder, context, value1),
equalTo(encode(coder, context, value2)));
}
/**
* Verifies that for the given {@code Coder},
* and value of type {@code T}, encoding followed by decoding yields an
* equal value of type {@code T}, in any {@code Coder.Context}.
*/
public static void coderDecodeEncodeEqual(
Coder coder, T value)
throws Exception {
for (Coder.Context context : ALL_CONTEXTS) {
coderDecodeEncodeEqualInContext(coder, context, value);
}
}
/**
* Verifies that for the given {@code Coder}, {@code Coder.Context},
* and value of type {@code T}, encoding followed by decoding yields an
* equal value of type {@code T}.
*/
public static void coderDecodeEncodeEqualInContext(
Coder coder, Coder.Context context, T value)
throws Exception {
assertThat(decodeEncode(coder, context, value), equalTo(value));
}
/**
* Verifies that for the given {@code Coder>},
* and value of type {@code Collection}, encoding followed by decoding yields an
* equal value of type {@code Collection}, in any {@code Coder.Context}.
*/
public static > void coderDecodeEncodeContentsEqual(
Coder coder, CollectionT value)
throws Exception {
for (Coder.Context context : ALL_CONTEXTS) {
coderDecodeEncodeContentsEqualInContext(coder, context, value);
}
}
/**
* Verifies that for the given {@code Coder>},
* and value of type {@code Collection}, encoding followed by decoding yields an
* equal value of type {@code Collection}, in the given {@code Coder.Context}.
*/
@SuppressWarnings("unchecked")
public static > void coderDecodeEncodeContentsEqualInContext(
Coder coder, Coder.Context context, CollectionT value)
throws Exception {
// Matchers.containsInAnyOrder() requires at least one element
Collection result = decodeEncode(coder, context, value);
if (value.isEmpty()) {
assertThat(result, emptyIterable());
} else {
// This is the only Matchers.containInAnyOrder() overload that takes literal values
assertThat(result, containsInAnyOrder((T[]) value.toArray()));
}
}
/**
* Verifies that for the given {@code Coder>},
* and value of type {@code Collection}, encoding followed by decoding yields an
* equal value of type {@code Collection}, in any {@code Coder.Context}.
*/
public static > void coderDecodeEncodeContentsInSameOrder(
Coder coder, IterableT value)
throws Exception {
for (Coder.Context context : ALL_CONTEXTS) {
CoderProperties.coderDecodeEncodeContentsInSameOrderInContext(
coder, context, value);
}
}
/**
* Verifies that for the given {@code Coder>},
* and value of type {@code Iterable}, encoding followed by decoding yields an
* equal value of type {@code Collection}, in the given {@code Coder.Context}.
*/
@SuppressWarnings("unchecked")
public static > void
coderDecodeEncodeContentsInSameOrderInContext(
Coder coder, Coder.Context context, IterableT value)
throws Exception {
Iterable result = decodeEncode(coder, context, value);
// Matchers.contains() requires at least one element
if (Iterables.isEmpty(value)) {
assertThat(result, emptyIterable());
} else {
// This is the only Matchers.contains() overload that takes literal values
assertThat(result, contains((T[]) Iterables.toArray(value, Object.class)));
}
}
public static void coderSerializable(Coder coder) {
SerializableUtils.ensureSerializable(coder);
}
public static void coderConsistentWithEquals(
Coder coder, T value1, T value2)
throws Exception {
for (Coder.Context context : ALL_CONTEXTS) {
CoderProperties.coderConsistentWithEqualsInContext(coder, context, value1, value2);
}
}
public static void coderConsistentWithEqualsInContext(
Coder coder, Coder.Context context, T value1, T value2) throws Exception {
assertEquals(
value1.equals(value2),
Arrays.equals(
encode(coder, context, value1),
encode(coder, context, value2)));
}
public static void coderHasEncodingId(Coder coder, String encodingId) throws Exception {
assertThat(coder.getEncodingId(), equalTo(encodingId));
assertThat(Structs.getString(coder.asCloudObject(), PropertyNames.ENCODING_ID, ""),
equalTo(encodingId));
}
public static void coderAllowsEncoding(Coder coder, String encodingId) throws Exception {
assertThat(coder.getAllowedEncodings(), hasItem(encodingId));
assertThat(
String.format("Expected to find \"%s\" in property \"%s\" of %s",
encodingId, PropertyNames.ALLOWED_ENCODINGS, coder.asCloudObject()),
Structs.getStrings(
coder.asCloudObject(),
PropertyNames.ALLOWED_ENCODINGS,
Collections.emptyList()),
hasItem(encodingId));
}
public static void structuralValueConsistentWithEquals(
Coder coder, T value1, T value2)
throws Exception {
for (Coder.Context context : ALL_CONTEXTS) {
CoderProperties.structuralValueConsistentWithEqualsInContext(
coder, context, value1, value2);
}
}
public static void structuralValueConsistentWithEqualsInContext(
Coder coder, Coder.Context context, T value1, T value2) throws Exception {
assertEquals(
coder.structuralValue(value1).equals(coder.structuralValue(value2)),
Arrays.equals(
encode(coder, context, value1),
encode(coder, context, value2)));
}
private static final String DECODING_WIRE_FORMAT_MESSAGE =
"Decoded value from known wire format does not match expected value."
+ " This probably means that this Coder no longer correctly decodes"
+ " a prior wire format. Changing the wire formats this Coder can read"
+ " should be avoided, as it is likely to cause breakage."
+ " If you truly intend to change the backwards compatibility for this Coder "
+ " then you must remove any now-unsupported encodings from getAllowedEncodings().";
public static void coderDecodesBase64(Coder coder, String base64Encoding, T value)
throws Exception {
assertThat(DECODING_WIRE_FORMAT_MESSAGE, CoderUtils.decodeFromBase64(coder, base64Encoding),
equalTo(value));
}
public static void coderDecodesBase64(
Coder coder, List base64Encodings, List values) throws Exception {
assertThat("List of base64 encodings has different size than List of values",
base64Encodings.size(), equalTo(values.size()));
for (int i = 0; i < base64Encodings.size(); i++) {
coderDecodesBase64(coder, base64Encodings.get(i), values.get(i));
}
}
private static final String ENCODING_WIRE_FORMAT_MESSAGE =
"Encoded value does not match expected wire format."
+ " Changing the wire format should be avoided, as it is likely to cause breakage."
+ " If you truly intend to change the wire format for this Coder "
+ " then you must update getEncodingId() to a new value and add any supported"
+ " prior formats to getAllowedEncodings()."
+ " See com.google.cloud.dataflow.sdk.coders.PrintBase64Encoding for how to generate"
+ " new test data.";
public static void coderEncodesBase64(Coder coder, T value, String base64Encoding)
throws Exception {
assertThat(ENCODING_WIRE_FORMAT_MESSAGE, CoderUtils.encodeToBase64(coder, value),
equalTo(base64Encoding));
}
public static void coderEncodesBase64(
Coder coder, List values, List base64Encodings) throws Exception {
assertThat("List of base64 encodings has different size than List of values",
base64Encodings.size(), equalTo(values.size()));
for (int i = 0; i < base64Encodings.size(); i++) {
coderEncodesBase64(coder, values.get(i), base64Encodings.get(i));
}
}
@SuppressWarnings("unchecked")
public static > void coderDecodesBase64ContentsEqual(
Coder coder, String base64Encoding, IterableT expected) throws Exception {
IterableT result = CoderUtils.decodeFromBase64(coder, base64Encoding);
if (Iterables.isEmpty(expected)) {
assertThat(ENCODING_WIRE_FORMAT_MESSAGE, result, emptyIterable());
} else {
assertThat(ENCODING_WIRE_FORMAT_MESSAGE, result,
containsInAnyOrder((T[]) Iterables.toArray(expected, Object.class)));
}
}
public static > void coderDecodesBase64ContentsEqual(
Coder coder, List base64Encodings, List expected)
throws Exception {
assertThat("List of base64 encodings has different size than List of values",
base64Encodings.size(), equalTo(expected.size()));
for (int i = 0; i < base64Encodings.size(); i++) {
coderDecodesBase64ContentsEqual(coder, base64Encodings.get(i), expected.get(i));
}
}
//////////////////////////////////////////////////////////////////////////
private static byte[] encode(
Coder coder, Coder.Context context, T value) throws CoderException, IOException {
@SuppressWarnings("unchecked")
Coder deserializedCoder = Serializer.deserialize(coder.asCloudObject(), Coder.class);
ByteArrayOutputStream os = new ByteArrayOutputStream();
deserializedCoder.encode(value, os, context);
return os.toByteArray();
}
private static T decode(
Coder coder, Coder.Context context, byte[] bytes) throws CoderException, IOException {
@SuppressWarnings("unchecked")
Coder deserializedCoder = Serializer.deserialize(coder.asCloudObject(), Coder.class);
ByteArrayInputStream is = new ByteArrayInputStream(bytes);
return deserializedCoder.decode(is, context);
}
private static T decodeEncode(Coder coder, Coder.Context context, T value)
throws CoderException, IOException {
return decode(coder, context, encode(coder, context, value));
}
}