software.amazon.awssdk.services.sagemaker.model.Channel Maven / Gradle / Ivy
Show all versions of sagemaker Show documentation
/*
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with
* the License. A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
* and limitations under the License.
*/
package software.amazon.awssdk.services.sagemaker.model;
import java.io.Serializable;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Function;
import software.amazon.awssdk.annotations.Generated;
import software.amazon.awssdk.core.SdkField;
import software.amazon.awssdk.core.SdkPojo;
import software.amazon.awssdk.core.protocol.MarshallLocation;
import software.amazon.awssdk.core.protocol.MarshallingType;
import software.amazon.awssdk.core.traits.LocationTrait;
import software.amazon.awssdk.utils.ToString;
import software.amazon.awssdk.utils.builder.CopyableBuilder;
import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
/**
*
* A channel is a named input source that training algorithms can consume.
*
*/
@Generated("software.amazon.awssdk:codegen")
public final class Channel implements SdkPojo, Serializable, ToCopyableBuilder {
private static final SdkField CHANNEL_NAME_FIELD = SdkField. builder(MarshallingType.STRING)
.memberName("ChannelName").getter(getter(Channel::channelName)).setter(setter(Builder::channelName))
.traits(LocationTrait.builder().location(MarshallLocation.PAYLOAD).locationName("ChannelName").build()).build();
private static final SdkField DATA_SOURCE_FIELD = SdkField. builder(MarshallingType.SDK_POJO)
.memberName("DataSource").getter(getter(Channel::dataSource)).setter(setter(Builder::dataSource))
.constructor(DataSource::builder)
.traits(LocationTrait.builder().location(MarshallLocation.PAYLOAD).locationName("DataSource").build()).build();
private static final SdkField CONTENT_TYPE_FIELD = SdkField. builder(MarshallingType.STRING)
.memberName("ContentType").getter(getter(Channel::contentType)).setter(setter(Builder::contentType))
.traits(LocationTrait.builder().location(MarshallLocation.PAYLOAD).locationName("ContentType").build()).build();
private static final SdkField COMPRESSION_TYPE_FIELD = SdkField. builder(MarshallingType.STRING)
.memberName("CompressionType").getter(getter(Channel::compressionTypeAsString))
.setter(setter(Builder::compressionType))
.traits(LocationTrait.builder().location(MarshallLocation.PAYLOAD).locationName("CompressionType").build()).build();
private static final SdkField RECORD_WRAPPER_TYPE_FIELD = SdkField. builder(MarshallingType.STRING)
.memberName("RecordWrapperType").getter(getter(Channel::recordWrapperTypeAsString))
.setter(setter(Builder::recordWrapperType))
.traits(LocationTrait.builder().location(MarshallLocation.PAYLOAD).locationName("RecordWrapperType").build()).build();
private static final SdkField INPUT_MODE_FIELD = SdkField. builder(MarshallingType.STRING)
.memberName("InputMode").getter(getter(Channel::inputModeAsString)).setter(setter(Builder::inputMode))
.traits(LocationTrait.builder().location(MarshallLocation.PAYLOAD).locationName("InputMode").build()).build();
private static final SdkField SHUFFLE_CONFIG_FIELD = SdkField
. builder(MarshallingType.SDK_POJO).memberName("ShuffleConfig").getter(getter(Channel::shuffleConfig))
.setter(setter(Builder::shuffleConfig)).constructor(ShuffleConfig::builder)
.traits(LocationTrait.builder().location(MarshallLocation.PAYLOAD).locationName("ShuffleConfig").build()).build();
private static final List> SDK_FIELDS = Collections.unmodifiableList(Arrays.asList(CHANNEL_NAME_FIELD,
DATA_SOURCE_FIELD, CONTENT_TYPE_FIELD, COMPRESSION_TYPE_FIELD, RECORD_WRAPPER_TYPE_FIELD, INPUT_MODE_FIELD,
SHUFFLE_CONFIG_FIELD));
private static final long serialVersionUID = 1L;
private final String channelName;
private final DataSource dataSource;
private final String contentType;
private final String compressionType;
private final String recordWrapperType;
private final String inputMode;
private final ShuffleConfig shuffleConfig;
private Channel(BuilderImpl builder) {
this.channelName = builder.channelName;
this.dataSource = builder.dataSource;
this.contentType = builder.contentType;
this.compressionType = builder.compressionType;
this.recordWrapperType = builder.recordWrapperType;
this.inputMode = builder.inputMode;
this.shuffleConfig = builder.shuffleConfig;
}
/**
*
* The name of the channel.
*
*
* @return The name of the channel.
*/
public final String channelName() {
return channelName;
}
/**
*
* The location of the channel data.
*
*
* @return The location of the channel data.
*/
public final DataSource dataSource() {
return dataSource;
}
/**
*
* The MIME type of the data.
*
*
* @return The MIME type of the data.
*/
public final String contentType() {
return contentType;
}
/**
*
* If training data is compressed, the compression type. The default value is None.
* CompressionType is used only in Pipe input mode. In File mode, leave this field unset or set it to
* None.
*
*
* If the service returns an enum value that is not available in the current SDK version, {@link #compressionType}
* will return {@link CompressionType#UNKNOWN_TO_SDK_VERSION}. The raw value returned by the service is available
* from {@link #compressionTypeAsString}.
*
*
* @return If training data is compressed, the compression type. The default value is None.
* CompressionType is used only in Pipe input mode. In File mode, leave this field unset or set
* it to None.
* @see CompressionType
*/
public final CompressionType compressionType() {
return CompressionType.fromValue(compressionType);
}
/**
*
* If training data is compressed, the compression type. The default value is None.
* CompressionType is used only in Pipe input mode. In File mode, leave this field unset or set it to
* None.
*
*
* If the service returns an enum value that is not available in the current SDK version, {@link #compressionType}
* will return {@link CompressionType#UNKNOWN_TO_SDK_VERSION}. The raw value returned by the service is available
* from {@link #compressionTypeAsString}.
*
*
* @return If training data is compressed, the compression type. The default value is None.
* CompressionType is used only in Pipe input mode. In File mode, leave this field unset or set
* it to None.
* @see CompressionType
*/
public final String compressionTypeAsString() {
return compressionType;
}
/**
*
*
* Specify RecordIO as the value when input data is in raw format but the training algorithm requires the RecordIO
* format. In this case, SageMaker wraps each individual S3 object in a RecordIO record. If the input data is
* already in RecordIO format, you don't need to set this attribute. For more information, see Create a Dataset Using
* RecordIO.
*
*
* In File mode, leave this field unset or set it to None.
*
*
* If the service returns an enum value that is not available in the current SDK version, {@link #recordWrapperType}
* will return {@link RecordWrapper#UNKNOWN_TO_SDK_VERSION}. The raw value returned by the service is available from
* {@link #recordWrapperTypeAsString}.
*
*
* @return
* Specify RecordIO as the value when input data is in raw format but the training algorithm requires the
* RecordIO format. In this case, SageMaker wraps each individual S3 object in a RecordIO record. If the
* input data is already in RecordIO format, you don't need to set this attribute. For more information, see
* Create a Dataset Using
* RecordIO.
*
*
* In File mode, leave this field unset or set it to None.
* @see RecordWrapper
*/
public final RecordWrapper recordWrapperType() {
return RecordWrapper.fromValue(recordWrapperType);
}
/**
*
*
* Specify RecordIO as the value when input data is in raw format but the training algorithm requires the RecordIO
* format. In this case, SageMaker wraps each individual S3 object in a RecordIO record. If the input data is
* already in RecordIO format, you don't need to set this attribute. For more information, see Create a Dataset Using
* RecordIO.
*
*
* In File mode, leave this field unset or set it to None.
*
*
* If the service returns an enum value that is not available in the current SDK version, {@link #recordWrapperType}
* will return {@link RecordWrapper#UNKNOWN_TO_SDK_VERSION}. The raw value returned by the service is available from
* {@link #recordWrapperTypeAsString}.
*
*
* @return
* Specify RecordIO as the value when input data is in raw format but the training algorithm requires the
* RecordIO format. In this case, SageMaker wraps each individual S3 object in a RecordIO record. If the
* input data is already in RecordIO format, you don't need to set this attribute. For more information, see
* Create a Dataset Using
* RecordIO.
*
*
* In File mode, leave this field unset or set it to None.
* @see RecordWrapper
*/
public final String recordWrapperTypeAsString() {
return recordWrapperType;
}
/**
*
* (Optional) The input mode to use for the data channel in a training job. If you don't set a value for
* InputMode, SageMaker uses the value set for TrainingInputMode. Use this parameter to
* override the TrainingInputMode setting in a AlgorithmSpecification request when you have a channel that needs a different input mode from the training
* job's general setting. To download the data from Amazon Simple Storage Service (Amazon S3) to the provisioned ML
* storage volume, and mount the directory to a Docker volume, use File input mode. To stream data
* directly from Amazon S3 to the container, choose Pipe input mode.
*
*
* To use a model for incremental training, choose File input model.
*
*
* If the service returns an enum value that is not available in the current SDK version, {@link #inputMode} will
* return {@link TrainingInputMode#UNKNOWN_TO_SDK_VERSION}. The raw value returned by the service is available from
* {@link #inputModeAsString}.
*
*
* @return (Optional) The input mode to use for the data channel in a training job. If you don't set a value for
* InputMode, SageMaker uses the value set for TrainingInputMode. Use this
* parameter to override the TrainingInputMode setting in a AlgorithmSpecification request when you have a channel that needs a different input mode from the
* training job's general setting. To download the data from Amazon Simple Storage Service (Amazon S3) to
* the provisioned ML storage volume, and mount the directory to a Docker volume, use File
* input mode. To stream data directly from Amazon S3 to the container, choose Pipe input
* mode.
*
* To use a model for incremental training, choose File input model.
* @see TrainingInputMode
*/
public final TrainingInputMode inputMode() {
return TrainingInputMode.fromValue(inputMode);
}
/**
*
* (Optional) The input mode to use for the data channel in a training job. If you don't set a value for
* InputMode, SageMaker uses the value set for TrainingInputMode. Use this parameter to
* override the TrainingInputMode setting in a AlgorithmSpecification request when you have a channel that needs a different input mode from the training
* job's general setting. To download the data from Amazon Simple Storage Service (Amazon S3) to the provisioned ML
* storage volume, and mount the directory to a Docker volume, use File input mode. To stream data
* directly from Amazon S3 to the container, choose Pipe input mode.
*
*
* To use a model for incremental training, choose File input model.
*
*
* If the service returns an enum value that is not available in the current SDK version, {@link #inputMode} will
* return {@link TrainingInputMode#UNKNOWN_TO_SDK_VERSION}. The raw value returned by the service is available from
* {@link #inputModeAsString}.
*
*
* @return (Optional) The input mode to use for the data channel in a training job. If you don't set a value for
* InputMode, SageMaker uses the value set for TrainingInputMode. Use this
* parameter to override the TrainingInputMode setting in a AlgorithmSpecification request when you have a channel that needs a different input mode from the
* training job's general setting. To download the data from Amazon Simple Storage Service (Amazon S3) to
* the provisioned ML storage volume, and mount the directory to a Docker volume, use File
* input mode. To stream data directly from Amazon S3 to the container, choose Pipe input
* mode.
*
* To use a model for incremental training, choose File input model.
* @see TrainingInputMode
*/
public final String inputModeAsString() {
return inputMode;
}
/**
*
* A configuration for a shuffle option for input data in a channel. If you use S3Prefix for
* S3DataType, this shuffles the results of the S3 key prefix matches. If you use
* ManifestFile, the order of the S3 object references in the ManifestFile is shuffled. If
* you use AugmentedManifestFile, the order of the JSON lines in the AugmentedManifestFile
* is shuffled. The shuffling order is determined using the Seed value.
*
*
* For Pipe input mode, shuffling is done at the start of every epoch. With large datasets this ensures that the
* order of the training data is different for each epoch, it helps reduce bias and possible overfitting. In a
* multi-node training job when ShuffleConfig is combined with S3DataDistributionType of
* ShardedByS3Key, the data is shuffled across nodes so that the content sent to a particular node on
* the first epoch might be sent to a different node on the second epoch.
*
*
* @return A configuration for a shuffle option for input data in a channel. If you use S3Prefix for
* S3DataType, this shuffles the results of the S3 key prefix matches. If you use
* ManifestFile, the order of the S3 object references in the ManifestFile is
* shuffled. If you use AugmentedManifestFile, the order of the JSON lines in the
* AugmentedManifestFile is shuffled. The shuffling order is determined using the
* Seed value.
*
* For Pipe input mode, shuffling is done at the start of every epoch. With large datasets this ensures that
* the order of the training data is different for each epoch, it helps reduce bias and possible
* overfitting. In a multi-node training job when ShuffleConfig is combined with
* S3DataDistributionType of ShardedByS3Key, the data is shuffled across nodes so
* that the content sent to a particular node on the first epoch might be sent to a different node on the
* second epoch.
*/
public final ShuffleConfig shuffleConfig() {
return shuffleConfig;
}
@Override
public Builder toBuilder() {
return new BuilderImpl(this);
}
public static Builder builder() {
return new BuilderImpl();
}
public static Class serializableBuilderClass() {
return BuilderImpl.class;
}
@Override
public final int hashCode() {
int hashCode = 1;
hashCode = 31 * hashCode + Objects.hashCode(channelName());
hashCode = 31 * hashCode + Objects.hashCode(dataSource());
hashCode = 31 * hashCode + Objects.hashCode(contentType());
hashCode = 31 * hashCode + Objects.hashCode(compressionTypeAsString());
hashCode = 31 * hashCode + Objects.hashCode(recordWrapperTypeAsString());
hashCode = 31 * hashCode + Objects.hashCode(inputModeAsString());
hashCode = 31 * hashCode + Objects.hashCode(shuffleConfig());
return hashCode;
}
@Override
public final boolean equals(Object obj) {
return equalsBySdkFields(obj);
}
@Override
public final boolean equalsBySdkFields(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (!(obj instanceof Channel)) {
return false;
}
Channel other = (Channel) obj;
return Objects.equals(channelName(), other.channelName()) && Objects.equals(dataSource(), other.dataSource())
&& Objects.equals(contentType(), other.contentType())
&& Objects.equals(compressionTypeAsString(), other.compressionTypeAsString())
&& Objects.equals(recordWrapperTypeAsString(), other.recordWrapperTypeAsString())
&& Objects.equals(inputModeAsString(), other.inputModeAsString())
&& Objects.equals(shuffleConfig(), other.shuffleConfig());
}
/**
* Returns a string representation of this object. This is useful for testing and debugging. Sensitive data will be
* redacted from this string using a placeholder value.
*/
@Override
public final String toString() {
return ToString.builder("Channel").add("ChannelName", channelName()).add("DataSource", dataSource())
.add("ContentType", contentType()).add("CompressionType", compressionTypeAsString())
.add("RecordWrapperType", recordWrapperTypeAsString()).add("InputMode", inputModeAsString())
.add("ShuffleConfig", shuffleConfig()).build();
}
public final Optional getValueForField(String fieldName, Class clazz) {
switch (fieldName) {
case "ChannelName":
return Optional.ofNullable(clazz.cast(channelName()));
case "DataSource":
return Optional.ofNullable(clazz.cast(dataSource()));
case "ContentType":
return Optional.ofNullable(clazz.cast(contentType()));
case "CompressionType":
return Optional.ofNullable(clazz.cast(compressionTypeAsString()));
case "RecordWrapperType":
return Optional.ofNullable(clazz.cast(recordWrapperTypeAsString()));
case "InputMode":
return Optional.ofNullable(clazz.cast(inputModeAsString()));
case "ShuffleConfig":
return Optional.ofNullable(clazz.cast(shuffleConfig()));
default:
return Optional.empty();
}
}
@Override
public final List> sdkFields() {
return SDK_FIELDS;
}
private static Function getter(Function g) {
return obj -> g.apply((Channel) obj);
}
private static BiConsumer setter(BiConsumer s) {
return (obj, val) -> s.accept((Builder) obj, val);
}
public interface Builder extends SdkPojo, CopyableBuilder {
/**
*
* The name of the channel.
*
*
* @param channelName
* The name of the channel.
* @return Returns a reference to this object so that method calls can be chained together.
*/
Builder channelName(String channelName);
/**
*
* The location of the channel data.
*
*
* @param dataSource
* The location of the channel data.
* @return Returns a reference to this object so that method calls can be chained together.
*/
Builder dataSource(DataSource dataSource);
/**
*
* The location of the channel data.
*
* This is a convenience method that creates an instance of the {@link DataSource.Builder} avoiding the need to
* create one manually via {@link DataSource#builder()}.
*
*
* When the {@link Consumer} completes, {@link DataSource.Builder#build()} is called immediately and its result
* is passed to {@link #dataSource(DataSource)}.
*
* @param dataSource
* a consumer that will call methods on {@link DataSource.Builder}
* @return Returns a reference to this object so that method calls can be chained together.
* @see #dataSource(DataSource)
*/
default Builder dataSource(Consumer dataSource) {
return dataSource(DataSource.builder().applyMutation(dataSource).build());
}
/**
*
* The MIME type of the data.
*
*
* @param contentType
* The MIME type of the data.
* @return Returns a reference to this object so that method calls can be chained together.
*/
Builder contentType(String contentType);
/**
*
* If training data is compressed, the compression type. The default value is None.
* CompressionType is used only in Pipe input mode. In File mode, leave this field unset or set it
* to None.
*
*
* @param compressionType
* If training data is compressed, the compression type. The default value is None.
* CompressionType is used only in Pipe input mode. In File mode, leave this field unset or
* set it to None.
* @see CompressionType
* @return Returns a reference to this object so that method calls can be chained together.
* @see CompressionType
*/
Builder compressionType(String compressionType);
/**
*
* If training data is compressed, the compression type. The default value is None.
* CompressionType is used only in Pipe input mode. In File mode, leave this field unset or set it
* to None.
*
*
* @param compressionType
* If training data is compressed, the compression type. The default value is None.
* CompressionType is used only in Pipe input mode. In File mode, leave this field unset or
* set it to None.
* @see CompressionType
* @return Returns a reference to this object so that method calls can be chained together.
* @see CompressionType
*/
Builder compressionType(CompressionType compressionType);
/**
*
*
* Specify RecordIO as the value when input data is in raw format but the training algorithm requires the
* RecordIO format. In this case, SageMaker wraps each individual S3 object in a RecordIO record. If the input
* data is already in RecordIO format, you don't need to set this attribute. For more information, see Create a Dataset Using
* RecordIO.
*
*
* In File mode, leave this field unset or set it to None.
*
*
* @param recordWrapperType
*
* Specify RecordIO as the value when input data is in raw format but the training algorithm requires the
* RecordIO format. In this case, SageMaker wraps each individual S3 object in a RecordIO record. If the
* input data is already in RecordIO format, you don't need to set this attribute. For more information,
* see Create a Dataset
* Using RecordIO.
*
*
* In File mode, leave this field unset or set it to None.
* @see RecordWrapper
* @return Returns a reference to this object so that method calls can be chained together.
* @see RecordWrapper
*/
Builder recordWrapperType(String recordWrapperType);
/**
*
*
* Specify RecordIO as the value when input data is in raw format but the training algorithm requires the
* RecordIO format. In this case, SageMaker wraps each individual S3 object in a RecordIO record. If the input
* data is already in RecordIO format, you don't need to set this attribute. For more information, see Create a Dataset Using
* RecordIO.
*
*
* In File mode, leave this field unset or set it to None.
*
*
* @param recordWrapperType
*
* Specify RecordIO as the value when input data is in raw format but the training algorithm requires the
* RecordIO format. In this case, SageMaker wraps each individual S3 object in a RecordIO record. If the
* input data is already in RecordIO format, you don't need to set this attribute. For more information,
* see Create a Dataset
* Using RecordIO.
*
*
* In File mode, leave this field unset or set it to None.
* @see RecordWrapper
* @return Returns a reference to this object so that method calls can be chained together.
* @see RecordWrapper
*/
Builder recordWrapperType(RecordWrapper recordWrapperType);
/**
*
* (Optional) The input mode to use for the data channel in a training job. If you don't set a value for
* InputMode, SageMaker uses the value set for TrainingInputMode. Use this parameter
* to override the TrainingInputMode setting in a AlgorithmSpecification request when you have a channel that needs a different input mode from the
* training job's general setting. To download the data from Amazon Simple Storage Service (Amazon S3) to the
* provisioned ML storage volume, and mount the directory to a Docker volume, use File input mode.
* To stream data directly from Amazon S3 to the container, choose Pipe input mode.
*
*
* To use a model for incremental training, choose File input model.
*
*
* @param inputMode
* (Optional) The input mode to use for the data channel in a training job. If you don't set a value for
* InputMode, SageMaker uses the value set for TrainingInputMode. Use this
* parameter to override the TrainingInputMode setting in a AlgorithmSpecification request when you have a channel that needs a different input mode from the
* training job's general setting. To download the data from Amazon Simple Storage Service (Amazon S3) to
* the provisioned ML storage volume, and mount the directory to a Docker volume, use File
* input mode. To stream data directly from Amazon S3 to the container, choose Pipe input
* mode.
*
* To use a model for incremental training, choose File input model.
* @see TrainingInputMode
* @return Returns a reference to this object so that method calls can be chained together.
* @see TrainingInputMode
*/
Builder inputMode(String inputMode);
/**
*
* (Optional) The input mode to use for the data channel in a training job. If you don't set a value for
* InputMode, SageMaker uses the value set for TrainingInputMode. Use this parameter
* to override the TrainingInputMode setting in a AlgorithmSpecification request when you have a channel that needs a different input mode from the
* training job's general setting. To download the data from Amazon Simple Storage Service (Amazon S3) to the
* provisioned ML storage volume, and mount the directory to a Docker volume, use File input mode.
* To stream data directly from Amazon S3 to the container, choose Pipe input mode.
*
*
* To use a model for incremental training, choose File input model.
*
*
* @param inputMode
* (Optional) The input mode to use for the data channel in a training job. If you don't set a value for
* InputMode, SageMaker uses the value set for TrainingInputMode. Use this
* parameter to override the TrainingInputMode setting in a AlgorithmSpecification request when you have a channel that needs a different input mode from the
* training job's general setting. To download the data from Amazon Simple Storage Service (Amazon S3) to
* the provisioned ML storage volume, and mount the directory to a Docker volume, use File
* input mode. To stream data directly from Amazon S3 to the container, choose Pipe input
* mode.
*
* To use a model for incremental training, choose File input model.
* @see TrainingInputMode
* @return Returns a reference to this object so that method calls can be chained together.
* @see TrainingInputMode
*/
Builder inputMode(TrainingInputMode inputMode);
/**
*
* A configuration for a shuffle option for input data in a channel. If you use S3Prefix for
* S3DataType, this shuffles the results of the S3 key prefix matches. If you use
* ManifestFile, the order of the S3 object references in the ManifestFile is
* shuffled. If you use AugmentedManifestFile, the order of the JSON lines in the
* AugmentedManifestFile is shuffled. The shuffling order is determined using the Seed
* value.
*
*
* For Pipe input mode, shuffling is done at the start of every epoch. With large datasets this ensures that the
* order of the training data is different for each epoch, it helps reduce bias and possible overfitting. In a
* multi-node training job when ShuffleConfig is combined with S3DataDistributionType of
* ShardedByS3Key, the data is shuffled across nodes so that the content sent to a particular node
* on the first epoch might be sent to a different node on the second epoch.
*
*
* @param shuffleConfig
* A configuration for a shuffle option for input data in a channel. If you use S3Prefix for
* S3DataType, this shuffles the results of the S3 key prefix matches. If you use
* ManifestFile, the order of the S3 object references in the ManifestFile is
* shuffled. If you use AugmentedManifestFile, the order of the JSON lines in the
* AugmentedManifestFile is shuffled. The shuffling order is determined using the
* Seed value.
*
* For Pipe input mode, shuffling is done at the start of every epoch. With large datasets this ensures
* that the order of the training data is different for each epoch, it helps reduce bias and possible
* overfitting. In a multi-node training job when ShuffleConfig is combined with
* S3DataDistributionType of ShardedByS3Key, the data is shuffled across nodes
* so that the content sent to a particular node on the first epoch might be sent to a different node on
* the second epoch.
* @return Returns a reference to this object so that method calls can be chained together.
*/
Builder shuffleConfig(ShuffleConfig shuffleConfig);
/**
*
* A configuration for a shuffle option for input data in a channel. If you use S3Prefix for
* S3DataType, this shuffles the results of the S3 key prefix matches. If you use
* ManifestFile, the order of the S3 object references in the ManifestFile is
* shuffled. If you use AugmentedManifestFile, the order of the JSON lines in the
* AugmentedManifestFile is shuffled. The shuffling order is determined using the Seed
* value.
*
*
* For Pipe input mode, shuffling is done at the start of every epoch. With large datasets this ensures that the
* order of the training data is different for each epoch, it helps reduce bias and possible overfitting. In a
* multi-node training job when ShuffleConfig is combined with S3DataDistributionType of
* ShardedByS3Key, the data is shuffled across nodes so that the content sent to a particular node
* on the first epoch might be sent to a different node on the second epoch.
*
* This is a convenience method that creates an instance of the {@link ShuffleConfig.Builder} avoiding the need
* to create one manually via {@link ShuffleConfig#builder()}.
*
*
* When the {@link Consumer} completes, {@link ShuffleConfig.Builder#build()} is called immediately and its
* result is passed to {@link #shuffleConfig(ShuffleConfig)}.
*
* @param shuffleConfig
* a consumer that will call methods on {@link ShuffleConfig.Builder}
* @return Returns a reference to this object so that method calls can be chained together.
* @see #shuffleConfig(ShuffleConfig)
*/
default Builder shuffleConfig(Consumer shuffleConfig) {
return shuffleConfig(ShuffleConfig.builder().applyMutation(shuffleConfig).build());
}
}
static final class BuilderImpl implements Builder {
private String channelName;
private DataSource dataSource;
private String contentType;
private String compressionType;
private String recordWrapperType;
private String inputMode;
private ShuffleConfig shuffleConfig;
private BuilderImpl() {
}
private BuilderImpl(Channel model) {
channelName(model.channelName);
dataSource(model.dataSource);
contentType(model.contentType);
compressionType(model.compressionType);
recordWrapperType(model.recordWrapperType);
inputMode(model.inputMode);
shuffleConfig(model.shuffleConfig);
}
public final String getChannelName() {
return channelName;
}
public final void setChannelName(String channelName) {
this.channelName = channelName;
}
@Override
public final Builder channelName(String channelName) {
this.channelName = channelName;
return this;
}
public final DataSource.Builder getDataSource() {
return dataSource != null ? dataSource.toBuilder() : null;
}
public final void setDataSource(DataSource.BuilderImpl dataSource) {
this.dataSource = dataSource != null ? dataSource.build() : null;
}
@Override
public final Builder dataSource(DataSource dataSource) {
this.dataSource = dataSource;
return this;
}
public final String getContentType() {
return contentType;
}
public final void setContentType(String contentType) {
this.contentType = contentType;
}
@Override
public final Builder contentType(String contentType) {
this.contentType = contentType;
return this;
}
public final String getCompressionType() {
return compressionType;
}
public final void setCompressionType(String compressionType) {
this.compressionType = compressionType;
}
@Override
public final Builder compressionType(String compressionType) {
this.compressionType = compressionType;
return this;
}
@Override
public final Builder compressionType(CompressionType compressionType) {
this.compressionType(compressionType == null ? null : compressionType.toString());
return this;
}
public final String getRecordWrapperType() {
return recordWrapperType;
}
public final void setRecordWrapperType(String recordWrapperType) {
this.recordWrapperType = recordWrapperType;
}
@Override
public final Builder recordWrapperType(String recordWrapperType) {
this.recordWrapperType = recordWrapperType;
return this;
}
@Override
public final Builder recordWrapperType(RecordWrapper recordWrapperType) {
this.recordWrapperType(recordWrapperType == null ? null : recordWrapperType.toString());
return this;
}
public final String getInputMode() {
return inputMode;
}
public final void setInputMode(String inputMode) {
this.inputMode = inputMode;
}
@Override
public final Builder inputMode(String inputMode) {
this.inputMode = inputMode;
return this;
}
@Override
public final Builder inputMode(TrainingInputMode inputMode) {
this.inputMode(inputMode == null ? null : inputMode.toString());
return this;
}
public final ShuffleConfig.Builder getShuffleConfig() {
return shuffleConfig != null ? shuffleConfig.toBuilder() : null;
}
public final void setShuffleConfig(ShuffleConfig.BuilderImpl shuffleConfig) {
this.shuffleConfig = shuffleConfig != null ? shuffleConfig.build() : null;
}
@Override
public final Builder shuffleConfig(ShuffleConfig shuffleConfig) {
this.shuffleConfig = shuffleConfig;
return this;
}
@Override
public Channel build() {
return new Channel(this);
}
@Override
public List> sdkFields() {
return SDK_FIELDS;
}
}
}