com.linkedin.dagli.dag.DynamicDAG Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of core Show documentation
Show all versions of core Show documentation
DAG-oriented machine learning framework for bug-resistant, readable, efficient, maintainable and trivially deployable models in Java and other JVM languages
// AUTOGENERATED CODE. DO NOT MODIFY DIRECTLY! Instead, please modify the dag/DynamicDAG.ftl file.
// See the README in the module's src/template directory for details.
package com.linkedin.dagli.dag;
import com.linkedin.dagli.annotation.equality.IgnoredByValueEquality;
import com.linkedin.dagli.annotation.equality.ValueEquality;
import com.linkedin.dagli.handle.ProducerHandle;
import com.linkedin.dagli.objectio.ObjectIterator;
import com.linkedin.dagli.objectio.ObjectReader;
import com.linkedin.dagli.placeholder.Placeholder;
import com.linkedin.dagli.preparer.AbstractBatchPreparerDynamic;
import com.linkedin.dagli.preparer.PreparerContext;
import com.linkedin.dagli.preparer.PreparerDynamic;
import com.linkedin.dagli.preparer.PreparerResultMixed;
import com.linkedin.dagli.producer.MissingInput;
import com.linkedin.dagli.producer.Producer;
import com.linkedin.dagli.reducer.Reducer;
import com.linkedin.dagli.transformer.AbstractPreparableTransformerDynamic;
import com.linkedin.dagli.transformer.AbstractPreparedTransformerDynamic;
import com.linkedin.dagli.transformer.DynamicInputs;
import com.linkedin.dagli.transformer.PreparedTransformer;
import com.linkedin.dagli.tuple.Tuple;
import com.linkedin.dagli.tuple.Tuple2;
import com.linkedin.dagli.tuple.Tuple3;
import com.linkedin.dagli.tuple.Tuple4;
import com.linkedin.dagli.tuple.Tuple5;
import com.linkedin.dagli.tuple.Tuple6;
import com.linkedin.dagli.tuple.Tuple7;
import com.linkedin.dagli.tuple.Tuple8;
import com.linkedin.dagli.tuple.Tuple9;
import com.linkedin.dagli.tuple.Tuple10;
import com.linkedin.dagli.util.invariant.Arguments;
import com.linkedin.dagli.util.collection.LinkedStack;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
/**
* A DynamicDAG is similar to DAGs of fixed arity, such as {@link DAG2x3}, but allows for any number of inputs to the
* DAG (while still limiting the number of outputs to a maximum of 10).
*
* DynamicDAGs are rarely required: usually, a DAG with a high number of inputs (more than 2 or 3) should aggregate
* their input values into @Structs or other data structure that can provide well-documented, user-friendly inputs
* to the DAG.
*
* The utility for DynamicDAGs is thus:
* (1) When you need a larger number of inputs but can't be bothered to create a @Struct (or similar); DynamicDAG
* may be preferable to the fixed-arity DAGs in these cases because it required "named", rather than ordinal,
* inputs, which greatly reduces the chances of accidentally transposing two inputs of the same type and creating
* logic bugs.
* (2) When automatically generating graphs that may have a large number of inputs (e.g. when generating subgraphs from
* an existing graph).
*/
@ValueEquality
public class DynamicDAG extends AbstractPreparableTransformerDynamic, DynamicDAG>
implements PreparableDAGTransformer, DynamicDAG> {
private static final long serialVersionUID = 1;
private DAGStructure _dag = null;
private List> _placeholders = null;
private List> _outputs = null;
// Keep track of the original output handles so users can refer to them later (the DAGStructure contains a map from
// placeholders to their indices, so we dont need to store those ourselves)
@IgnoredByValueEquality
private Object2IntOpenHashMap> _outputToIndexMap = null;
// not ignored by value equality because _dag may not have been constructed yet
private Reducer.Level _reductionLevel = Reducer.Level.NORMAL;
@IgnoredByValueEquality
private DAGExecutor _executor = new LocalDAGExecutor();
@Override
public void validate() {
super.validate();
Objects.requireNonNull(_dag, "The DAG has not yet been set by calling withOutputs(...)");
}
/**
* Returns a copy of this DAG that will use the specified {@link Producer} to provide values corresponding to the
* given {@link Placeholder}. This must be called after the DAG has been specified via {@code withOutputs(...)} and
* is only relevant when the DAG is being used as a transformer within a larger DAG.
*
* @param placeholder the placeholder corresponding to the input to set
* @param inputForPlaceholder the producer providing the values for the input
* @param the type of value required for the given input
* @return a copy of this DAG that will accept the specified input
*/
public DynamicDAG withInput(Placeholder placeholder, Producer extends T> inputForPlaceholder) {
int index = _dag.getNodeIndex(placeholder);
Arguments.check(index >= 0, "Placeholder is not among the known placeholders for this DAG");
return clone(c -> c._inputs.set(index, inputForPlaceholder));
}
/**
* Returns a copy of this DAG that will use the specified {@link Producer} to provide values corresponding to the
* given {@link Placeholder}. This must be called after the DAG has been specified via {@code withOutputs(...)} and
* is only relevant when the DAG is being used as a transformer within a larger DAG.
*
* @param placeholderAccessor the placeholder corresponding to the input to set
* @param inputForPlaceholder the producer providing the values for the input
* @param the type of value required for the given input
* @return a copy of this DAG that will accept the specified input
*/
public DynamicDAG withInput(DynamicInputs.Accessor placeholderAccessor,
Producer extends T> inputForPlaceholder) {
return clone(c -> c._inputs.set(placeholderAccessor.getIndex(), inputForPlaceholder));
}
/**
* Creates an {@link DynamicInputs.Accessor} that may be used to refer to a particular placeholder when providing
* input values to the DAG. Using accessors is faster than using {@link Producer}s directly because it avoids
* repeated hashtable lookups, but only if the accessor is used more than once.
*
* @param placeholder the placeholder whose accessor should be retrieved
* @param the type of value produced by the placeholder
* @return an accessor for the given placeholder, or null if the placeholder is not known to this DAG
*/
public DynamicInputs.Accessor getPlaceholderAccessor(Placeholder placeholder) {
int index = _dag.getNodeIndex(placeholder);
return index < 0 ? null : new DynamicInputs.Accessor<>(index);
}
@SuppressWarnings("unchecked")
private void setOutputs(List> outputs) {
_outputs = outputs;
if (_placeholders == null) {
_placeholders =
(List) Producer.subgraphProducers(outputs).map(LinkedStack::peek)
.filter(node -> node instanceof Placeholder>).distinct().collect(Collectors.toList());
}
setDAG();
}
private void setPlaceholders(List> placeholders) {
_placeholders = placeholders;
if (_outputs != null) {
setDAG();
}
}
private void setDAG() {
_outputToIndexMap = new Object2IntOpenHashMap<>(_outputs.size());
_outputs.stream().map(Producer::handle).forEach(handle -> _outputToIndexMap.put(handle, _outputToIndexMap.size()));
_dag = new DAGStructure<>(DAGReducer.reduce(new DeduplicatedDAG(_placeholders, _outputs), _reductionLevel));
this._inputs = new ArrayList<>(MissingInput.producerList(_placeholders.size()));
}
/**
* Returns a copy of this DAG that will use the specified placeholders.
*
* It is more efficient to call this method prior to {@code withOutputs(...)}.
*
* @param placeholders the placeholders to use for the DAG
* @return a copy of this DAG that will use the specified placeholders
*/
public DynamicDAG withPlaceholders(Placeholder>... placeholders) {
return withPlaceholders(Arrays.asList(placeholders));
}
/**
* Returns a copy of this DAG that will use the specified placeholders.
*
* It is more efficient to call this method prior to {@code withOutputs(...)}.
*
* @param placeholders the placeholders to use for the DAG
* @return a copy of this DAG that will use the specified placeholders
*/
public DynamicDAG withPlaceholders(List extends Placeholder>> placeholders) {
return clone(c -> c.setPlaceholders(new ArrayList<>(placeholders)));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param outputs the outputs of this DAG (at most 10)
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
public DynamicDAG> withOutputs(Producer>... outputs) {
return withOutputs(Arrays.asList(outputs));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param outputs the outputs of this DAG (at most 10)
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
public DynamicDAG> withOutputs(Collection extends Producer>> outputs) {
Arguments.check(outputs.size() <= 10, "A DAG cannot have more than 10 outputs; consider aggregating "
+ "some of the outputs into a data structure like a @Struct or List");
return clone(c -> c.setOutputs(new ArrayList<>(outputs)));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param the type of the first result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG withOutputs(Producer output1) {
return (DynamicDAG) withOutputs(Arrays.asList(output1));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param output2 the second output of the DAG
* @param the type of the first result
* @param the type of the second result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG> withOutputs(Producer output1, Producer output2) {
return (DynamicDAG>) withOutputs(Arrays.asList(output1, output2));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param output2 the second output of the DAG
* @param output3 the third output of the DAG
* @param the type of the first result
* @param the type of the second result
* @param the type of the third result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG> withOutputs(Producer output1, Producer output2,
Producer output3) {
return (DynamicDAG>) withOutputs(Arrays.asList(output1, output2, output3));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param output2 the second output of the DAG
* @param output3 the third output of the DAG
* @param output4 the fourth output of the DAG
* @param the type of the first result
* @param the type of the second result
* @param the type of the third result
* @param the type of the fourth result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG> withOutputs(Producer output1, Producer output2,
Producer output3, Producer output4) {
return (DynamicDAG>) withOutputs(Arrays.asList(output1, output2, output3, output4));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param output2 the second output of the DAG
* @param output3 the third output of the DAG
* @param output4 the fourth output of the DAG
* @param output5 the fifth output of the DAG
* @param the type of the first result
* @param the type of the second result
* @param the type of the third result
* @param the type of the fourth result
* @param the type of the fifth result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG> withOutputs(Producer output1,
Producer output2, Producer output3, Producer output4, Producer output5) {
return (DynamicDAG>) withOutputs(Arrays.asList(output1, output2, output3, output4,
output5));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param output2 the second output of the DAG
* @param output3 the third output of the DAG
* @param output4 the fourth output of the DAG
* @param output5 the fifth output of the DAG
* @param output6 the sixth output of the DAG
* @param the type of the first result
* @param the type of the second result
* @param the type of the third result
* @param the type of the fourth result
* @param the type of the fifth result
* @param the type of the sixth result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG> withOutputs(Producer output1,
Producer output2, Producer output3, Producer output4, Producer output5, Producer output6) {
return (DynamicDAG>) withOutputs(Arrays.asList(output1, output2, output3, output4,
output5, output6));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param output2 the second output of the DAG
* @param output3 the third output of the DAG
* @param output4 the fourth output of the DAG
* @param output5 the fifth output of the DAG
* @param output6 the sixth output of the DAG
* @param output7 the seventh output of the DAG
* @param the type of the first result
* @param the type of the second result
* @param the type of the third result
* @param the type of the fourth result
* @param the type of the fifth result
* @param the type of the sixth result
* @param the type of the seventh result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG> withOutputs(Producer output1,
Producer output2, Producer output3, Producer output4, Producer output5, Producer output6,
Producer output7) {
return (DynamicDAG>) withOutputs(Arrays.asList(output1, output2, output3,
output4, output5, output6, output7));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param output2 the second output of the DAG
* @param output3 the third output of the DAG
* @param output4 the fourth output of the DAG
* @param output5 the fifth output of the DAG
* @param output6 the sixth output of the DAG
* @param output7 the seventh output of the DAG
* @param output8 the eighth output of the DAG
* @param the type of the first result
* @param the type of the second result
* @param the type of the third result
* @param the type of the fourth result
* @param the type of the fifth result
* @param the type of the sixth result
* @param the type of the seventh result
* @param the type of the eighth result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG> withOutputs(
Producer output1, Producer output2, Producer output3, Producer output4, Producer output5,
Producer output6, Producer output7, Producer output8) {
return (DynamicDAG>) withOutputs(Arrays.asList(output1, output2, output3,
output4, output5, output6, output7, output8));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param output2 the second output of the DAG
* @param output3 the third output of the DAG
* @param output4 the fourth output of the DAG
* @param output5 the fifth output of the DAG
* @param output6 the sixth output of the DAG
* @param output7 the seventh output of the DAG
* @param output8 the eighth output of the DAG
* @param output9 the ninth output of the DAG
* @param the type of the first result
* @param the type of the second result
* @param the type of the third result
* @param the type of the fourth result
* @param the type of the fifth result
* @param the type of the sixth result
* @param the type of the seventh result
* @param the type of the eighth result
* @param the type of the ninth result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG> withOutputs(
Producer output1, Producer output2, Producer output3, Producer output4, Producer output5,
Producer output6, Producer output7, Producer output8, Producer output9) {
return (DynamicDAG>) withOutputs(Arrays.asList(output1, output2,
output3, output4, output5, output6, output7, output8, output9));
}
/**
* Returns a copy of this DAG that will have the given ordered sequence of outputs. The placeholders comprising the
* inputs to the new DAG will be automatically discovered unless specified via {@code withPlaceholders(...)}.
*
* @param output1 the first output of the DAG
* @param output2 the second output of the DAG
* @param output3 the third output of the DAG
* @param output4 the fourth output of the DAG
* @param output5 the fifth output of the DAG
* @param output6 the sixth output of the DAG
* @param output7 the seventh output of the DAG
* @param output8 the eighth output of the DAG
* @param output9 the ninth output of the DAG
* @param output10 the tenth output of the DAG
* @param the type of the first result
* @param the type of the second result
* @param the type of the third result
* @param the type of the fourth result
* @param the type of the fifth result
* @param the type of the sixth result
* @param the type of the seventh result
* @param the type of the eighth result
* @param the type of the ninth result
* @param the type of the tenth result
* @return a copy of this DAG that will produce the requested outputs
* @throws IllegalArgumentException if more than 10 outputs are specified
*/
@SuppressWarnings("unchecked")
public DynamicDAG> withOutputs(
Producer output1, Producer output2, Producer output3, Producer output4, Producer output5,
Producer output6, Producer output7, Producer output8, Producer output9, Producer output10) {
return (DynamicDAG>) withOutputs(Arrays.asList(output1, output2,
output3, output4, output5, output6, output7, output8, output9, output10));
}
@Override
public DynamicDAG withExecutor(DAGExecutor executor) {
return clone(r -> r._executor = Objects.requireNonNull(executor));
}
/**
* @return a copy of this DAG that, the next time the DAG is fully specified (by setting the outputs or by setting
* the placeholders when the outputs are already set), will not apply reductions to its encapsulated
* elements (note that this does not affect the reduction of this DAG within another, encapsulating DAG)
*/
public DynamicDAG withNoReduction() {
return withReduction(null);
}
@Override
public DynamicDAG withReduction(Reducer.Level level) {
if (_dag == null || Reducer.Level.compare(level, _reductionLevel) >= 0) {
return level == _reductionLevel ? this : clone(c -> c._reductionLevel = level);
}
return clone(c -> {
c._dag = new DAGStructure<>(DAGReducer.reduce(new DeduplicatedDAG(_dag), level));
c._reductionLevel = level;
});
}
@Override
protected Collection extends Reducer super DynamicDAG>> getGraphReducers() {
return Collections.singletonList(DAGTransformerReducer.INSTANCE);
}
@Override
protected boolean hasAlwaysConstantResult() {
return _dag._isAlwaysConstant;
}
@Override
public InternalAPI internalAPI() {
return new InternalAPI();
}
public class InternalAPI extends AbstractPreparableTransformerDynamic, DynamicDAG>.InternalAPI
implements PreparableDAGTransformer.InternalAPI, DynamicDAG> {
@Override
public DAGStructure getDAGStructure() {
return _dag;
}
@Override
public Reducer.Level getReductionLevel() {
return _reductionLevel;
}
@Override
public DAGExecutor getDAGExecutor() {
return _executor;
}
@Override
public DynamicDAG getInstance() {
return DynamicDAG.this;
}
}
@Override
protected boolean hasIdempotentPreparer() {
return _dag._hasIdempotentPreparer;
}
private abstract class AbstractPrepareBuilder> {
@SuppressWarnings("unchecked")
final ObjectReader