Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance.
Project price only 1 $
You can buy this project and download/modify it how often you want.
io.datakernel.aggregation.AggregationUtils Maven / Gradle / Ivy
/*
* Copyright (C) 2015-2018 SoftIndex LLC.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.datakernel.aggregation;
import io.datakernel.aggregation.annotation.Key;
import io.datakernel.aggregation.annotation.Measures;
import io.datakernel.aggregation.fieldtype.FieldType;
import io.datakernel.aggregation.measure.Measure;
import io.datakernel.aggregation.ot.AggregationStructure;
import io.datakernel.aggregation.util.PartitionPredicate;
import io.datakernel.codec.StructuredCodec;
import io.datakernel.codegen.*;
import io.datakernel.serializer.BinarySerializer;
import io.datakernel.serializer.SerializerBuilder;
import io.datakernel.serializer.asm.SerializerGenClass;
import io.datakernel.stream.processor.StreamReducers.Reducer;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Stream;
import static io.datakernel.codec.StructuredCodecs.ofTupleArray;
import static io.datakernel.codegen.Expressions.*;
import static io.datakernel.util.CollectionUtils.concat;
import static io.datakernel.util.CollectionUtils.keysToMap;
import static io.datakernel.util.Preconditions.checkArgument;
import static io.datakernel.util.ReflectionUtils.extractFieldNameFromGetter;
/**
* Defines a structure of an aggregation.
* It is defined by keys, fields and their types.
* Contains methods for defining dynamic classes, that are used for different operations.
* Provides serializer for records that have the defined structure.
*/
@SuppressWarnings({"rawtypes", "unchecked"})
public class AggregationUtils {
private AggregationUtils() {
}
public static Class createKeyClass(Map keys, DefiningClassLoader classLoader) {
List keyList = new ArrayList<>(keys.keySet());
return ClassBuilder.create(classLoader, Comparable.class)
.initialize(cb ->
keys.forEach((key, value) ->
cb.withField(key, value.getInternalDataType())))
.withMethod("compareTo", compareTo(keyList))
.withMethod("equals", asEquals(keyList))
.withMethod("hashCode", hashCodeOfThis(keyList))
.withMethod("toString", asString(keyList))
.build();
}
public static Comparator createKeyComparator(Class recordClass, List keys, DefiningClassLoader classLoader) {
return ClassBuilder.create(classLoader, Comparator.class)
.withMethod("compare", compare(recordClass, keys))
.buildClassAndCreateNewInstance();
}
public static Function createMapper(Class recordClass, Class resultClass,
List keys, List fields,
DefiningClassLoader classLoader) {
return ClassBuilder.create(classLoader, Function.class)
.withMethod("apply", () -> {
Expression result1 = let(constructor(resultClass));
ExpressionSequence sequence = ExpressionSequence.create();
for (String fieldName : (Iterable) Stream.concat(keys.stream(), fields.stream())::iterator) {
sequence.add(set(
property(result1, fieldName),
property(cast(arg(0), recordClass), fieldName)));
}
return sequence.add(result1);
})
.buildClassAndCreateNewInstance();
}
public static Function createKeyFunction(Class recordClass, Class keyClass,
List keys,
DefiningClassLoader classLoader) {
return ClassBuilder.create(classLoader, Function.class)
.withMethod("apply", () -> {
Expression key = let(constructor(keyClass));
ExpressionSequence sequence = ExpressionSequence.create();
for (String keyString : keys) {
sequence.add(set(
property(key, keyString),
property(cast(arg(0), recordClass), keyString)));
}
return sequence.add(key);
})
.buildClassAndCreateNewInstance();
}
public static Class createRecordClass(AggregationStructure aggregation,
Collection keys, Collection fields,
DefiningClassLoader classLoader) {
return createRecordClass(
keysToMap(keys.stream(), aggregation.getKeyTypes()::get),
keysToMap(fields.stream(), aggregation.getMeasureTypes()::get),
classLoader);
}
public static Class createRecordClass(Map keys, Map fields,
DefiningClassLoader classLoader) {
return ClassBuilder.create(classLoader, (Class) Object.class)
.initialize(cb ->
keys.forEach((key, value) ->
cb.withField(key, value.getInternalDataType())))
.initialize(cb ->
fields.forEach((key, value) ->
cb.withField(key, value.getInternalDataType())))
.withMethod("toString", asString(concat(keys.keySet(), fields.keySet())))
.build();
}
public static BinarySerializer createBinarySerializer(AggregationStructure aggregation, Class recordClass,
List keys, List fields,
DefiningClassLoader classLoader) {
return createBinarySerializer(recordClass,
keysToMap(keys.stream(), aggregation.getKeyTypes()::get),
keysToMap(fields.stream(), aggregation.getMeasureTypes()::get),
classLoader);
}
private static BinarySerializer createBinarySerializer(Class recordClass,
Map keys, Map fields,
DefiningClassLoader classLoader) {
SerializerGenClass serializerGenClass = new SerializerGenClass(recordClass);
for (String key : keys.keySet()) {
FieldType keyType = keys.get(key);
try {
Field recordClassKey = recordClass.getField(key);
serializerGenClass.addField(recordClassKey, keyType.getSerializer(), -1, -1);
} catch (NoSuchFieldException e) {
throw new RuntimeException(e);
}
}
for (String field : fields.keySet()) {
try {
Field recordClassField = recordClass.getField(field);
serializerGenClass.addField(recordClassField, fields.get(field).getSerializer(), -1, -1);
} catch (NoSuchFieldException e) {
throw new RuntimeException(e);
}
}
return SerializerBuilder.create(classLoader).build(serializerGenClass);
}
public static Reducer aggregationReducer(AggregationStructure aggregation, Class inputClass, Class outputClass,
List keys, List fields,
DefiningClassLoader classLoader) {
Expression accumulator = let(constructor(outputClass));
ExpressionSequence onFirstItem = ExpressionSequence.create();
ExpressionSequence onNextItem = ExpressionSequence.create();
for (String key : keys) {
onFirstItem.add(set(
property(accumulator, key),
property(cast(arg(2), inputClass), key)));
}
for (String field : fields) {
Measure aggregateFunction = aggregation.getMeasure(field);
onFirstItem.add(aggregateFunction.initAccumulatorWithAccumulator(
property(accumulator, field),
property(cast(arg(2), inputClass), field)
));
onNextItem.add(aggregateFunction.reduce(
property(cast(arg(3), outputClass), field),
property(cast(arg(2), inputClass), field)
));
}
onFirstItem.add(accumulator);
onNextItem.add(arg(3));
return ClassBuilder.create(classLoader, Reducer.class)
.withMethod("onFirstItem", onFirstItem)
.withMethod("onNextItem", onNextItem)
.withMethod("onComplete", call(arg(0), "accept", arg(2)))
.buildClassAndCreateNewInstance();
}
public static Aggregate createPreaggregator(AggregationStructure aggregation, Class inputClass, Class outputClass,
Map keyFields, Map measureFields,
DefiningClassLoader classLoader) {
Expression accumulator = let(constructor(outputClass));
ExpressionSequence createAccumulator = ExpressionSequence.create();
ExpressionSequence accumulate = ExpressionSequence.create();
for (String key : keyFields.keySet()) {
String inputField = keyFields.get(key);
createAccumulator.add(set(
property(accumulator, key),
property(cast(arg(0), inputClass), inputField)));
}
for (String measure : measureFields.keySet()) {
String inputFields = measureFields.get(measure);
Measure aggregateFunction = aggregation.getMeasure(measure);
createAccumulator.add(aggregateFunction.initAccumulatorWithValue(
property(accumulator, measure),
inputFields == null ? null : property(cast(arg(0), inputClass), inputFields)));
accumulate.add(aggregateFunction.accumulate(
property(cast(arg(0), outputClass), measure),
inputFields == null ? null : property(cast(arg(1), inputClass), inputFields)));
}
createAccumulator.add(accumulator);
return ClassBuilder.create(classLoader, Aggregate.class)
.withMethod("createAccumulator", createAccumulator)
.withMethod("accumulate", accumulate)
.buildClassAndCreateNewInstance();
}
private static final PartitionPredicate SINGLE_PARTITION = (t, u) -> true;
public static PartitionPredicate singlePartition() {
return SINGLE_PARTITION;
}
public static PartitionPredicate createPartitionPredicate(Class recordClass, List partitioningKey,
DefiningClassLoader classLoader) {
if (partitioningKey.isEmpty())
return singlePartition();
PredicateDefAnd predicate = PredicateDefAnd.create();
for (String keyComponent : partitioningKey) {
predicate.add(cmpEq(
property(cast(arg(0), recordClass), keyComponent),
property(cast(arg(1), recordClass), keyComponent)));
}
return ClassBuilder.create(classLoader, PartitionPredicate.class)
.withMethod("isSamePartition", predicate)
.buildClassAndCreateNewInstance();
}
public static Map scanKeyFields(Class inputClass) {
Map keyFields = new LinkedHashMap<>();
for (Field field : inputClass.getFields()) {
for (Annotation annotation : field.getAnnotations()) {
if (annotation.annotationType() == Key.class) {
String value = ((Key) annotation).value();
keyFields.put("".equals(value) ? field.getName() : value, field.getName());
}
}
}
for (Method method : inputClass.getMethods()) {
for (Annotation annotation : method.getAnnotations()) {
if (annotation.annotationType() == Key.class) {
String value = ((Key) annotation).value();
keyFields.put("".equals(value) ? method.getName() : value, method.getName());
}
}
}
checkArgument(!keyFields.isEmpty(), "Missing @Key annotations in %s", inputClass);
return keyFields;
}
public static Map scanMeasureFields(Class inputClass) {
Map measureFields = new LinkedHashMap<>();
for (Annotation annotation : inputClass.getAnnotations()) {
if (annotation.annotationType() == Measures.class) {
for (String measure : ((Measures) annotation).value()) {
measureFields.put(measure, null);
}
}
}
for (Field field : inputClass.getFields()) {
for (Annotation annotation : field.getAnnotations()) {
if (annotation.annotationType() == Measures.class) {
for (String measure : ((Measures) annotation).value()) {
measureFields.put(measure.equals("") ? field.getName() : measure, field.getName());
}
}
}
}
for (Method method : inputClass.getMethods()) {
for (Annotation annotation : method.getAnnotations()) {
if (annotation.annotationType() == Measures.class) {
for (String measure : ((Measures) annotation).value()) {
measureFields.put(measure.equals("") ? extractFieldNameFromGetter(method) : measure, method.getName());
}
}
}
}
checkArgument(!measureFields.isEmpty(), "Missing @Measure(s) annotations in %s", inputClass);
return measureFields;
}
public static StructuredCodec getPrimaryKeyCodec(AggregationStructure aggregation) {
StructuredCodec[] keyCodec = new StructuredCodec[aggregation.getKeys().size()];
for (int i = 0; i < aggregation.getKeys().size(); i++) {
String key = aggregation.getKeys().get(i);
FieldType keyType = aggregation.getKeyTypes().get(key);
keyCodec[i] = keyType.getInternalCodec();
}
return ofTupleArray(keyCodec)
.transform(PrimaryKey::ofArray, PrimaryKey::getArray);
}
}
