com.datastax.spark.connector.japi.CassandraJavaUtil Maven / Gradle / Ivy
package com.datastax.spark.connector.japi;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.rdd.RDD;
import scala.Option;
import scala.Tuple1;
import scala.Tuple10;
import scala.Tuple11;
import scala.Tuple12;
import scala.Tuple13;
import scala.Tuple14;
import scala.Tuple15;
import scala.Tuple16;
import scala.Tuple17;
import scala.Tuple18;
import scala.Tuple19;
import scala.Tuple2;
import scala.Tuple20;
import scala.Tuple21;
import scala.Tuple22;
import scala.Tuple3;
import scala.Tuple4;
import scala.Tuple5;
import scala.Tuple6;
import scala.Tuple7;
import scala.Tuple8;
import scala.Tuple9;
import scala.reflect.ClassTag;
import scala.reflect.ClassTag$;
import scala.reflect.api.TypeTags;
import com.datastax.spark.connector.AllColumns$;
import com.datastax.spark.connector.BatchSize;
import com.datastax.spark.connector.BatchSize$;
import com.datastax.spark.connector.BytesInBatch$;
import com.datastax.spark.connector.ColumnName;
import com.datastax.spark.connector.ColumnName$;
import com.datastax.spark.connector.ColumnRef;
import com.datastax.spark.connector.ColumnSelector;
import com.datastax.spark.connector.RowsInBatch$;
import com.datastax.spark.connector.SomeColumns$;
import com.datastax.spark.connector.TTL;
import com.datastax.spark.connector.WriteTime;
import com.datastax.spark.connector.cql.CassandraConnector;
import com.datastax.spark.connector.mapper.ColumnMapper;
import com.datastax.spark.connector.mapper.JavaBeanColumnMapper;
import com.datastax.spark.connector.mapper.TupleColumnMapper;
import com.datastax.spark.connector.rdd.reader.ClassBasedRowReaderFactory;
import com.datastax.spark.connector.rdd.reader.RowReaderFactory;
import com.datastax.spark.connector.rdd.reader.ValueRowReaderFactory;
import com.datastax.spark.connector.types.TypeConverter;
import com.datastax.spark.connector.types.TypeConverter$;
import com.datastax.spark.connector.util.JavaApiHelper;
import com.datastax.spark.connector.util.JavaApiHelper$;
import com.datastax.spark.connector.writer.BatchGroupingKey;
import com.datastax.spark.connector.writer.RowWriterFactory;
import static com.datastax.spark.connector.util.JavaApiHelper.defaultRowWriterFactory;
import static com.datastax.spark.connector.util.JavaApiHelper.javaBeanColumnMapper;
/**
* The main entry point to Spark Cassandra Connector Java API.
*
* There are several helpful static factory methods which build useful wrappers around Spark Context, and
* RDD. There are also helper methods to build reader and writer factories which configure their
* parameters.
*/
@SuppressWarnings("UnusedDeclaration")
public class CassandraJavaUtil {
private CassandraJavaUtil() {
assert false;
}
// -------------------------------------------------------------------------
// Java API wrappers factory methods
// -------------------------------------------------------------------------
/**
* A static factory method to create a {@link SparkContextJavaFunctions} based on an existing {@link
* SparkContext} instance.
*/
public static SparkContextJavaFunctions javaFunctions(SparkContext sparkContext) {
return new SparkContextJavaFunctions(sparkContext);
}
/**
* A static factory method to create a {@link SparkContextJavaFunctions} based on an existing {@link
* JavaSparkContext} instance.
*/
public static SparkContextJavaFunctions javaFunctions(JavaSparkContext sparkContext) {
return new SparkContextJavaFunctions(JavaSparkContext.toSparkContext(sparkContext));
}
/**
* A static factory method to create a {@link RDDJavaFunctions} based on an existing {@link RDD}
* instance.
*/
public static RDDJavaFunctions javaFunctions(RDD rdd) {
return new RDDJavaFunctions<>(rdd);
}
/**
* A static factory method to create a {@link RDDJavaFunctions} based on an existing {@link JavaRDD}
* instance.
*/
public static RDDJavaFunctions javaFunctions(JavaRDD rdd) {
return new RDDJavaFunctions<>(rdd.rdd());
}
/**
* A static factory method to create a {@link PairRDDJavaFunctions} based on an existing {@link
* JavaPairRDD} instance.
*/
public static PairRDDJavaFunctions javaFunctions(JavaPairRDD rdd) {
return new PairRDDJavaFunctions<>(rdd.rdd());
}
// -------------------------------------------------------------------------
// Type tag resolvers
// -------------------------------------------------------------------------
public static ClassTag safeClassTag(Class cls) {
return JavaApiHelper$.MODULE$.getClassTag(cls);
}
public static ClassTag classTag(Class cls) {
return JavaApiHelper$.MODULE$.getClassTag2(cls);
}
@SuppressWarnings("unchecked")
public static ClassTag anyClassTag() {
return (ClassTag) ClassTag$.MODULE$.AnyRef();
}
/**
* Creates a type tag representing a given class type without any type parameters.
*/
public static TypeTags.TypeTag safeTypeTag(Class main) {
return JavaApiHelper.getTypeTag(main);
}
/**
* Creates a type tag representing a given class type without any type parameters.
*/
@SuppressWarnings("unchecked")
public static TypeTags.TypeTag typeTag(Class main) {
return JavaApiHelper.getTypeTag((Class) main);
}
/**
* Creates a type tag representing a given class type with given type parameters.
*
* This is the most generic method to create type tags. One can use it to create type tags of
* arbitrary class types with any number of type parameters and any depth of type parameter nesting.
*
* For example, the following code: {@code typeTag(Map.class, typeTag(String.class),
* typeTag(List.class, typeTag(String.class))} returns the type tag of {@code Map>}
*/
public static TypeTags.TypeTag typeTag(
Class main,
TypeTags.TypeTag typeParam1,
TypeTags.TypeTag... typeParams
) {
return JavaApiHelper.getTypeTag(main, ArrayUtils.add(typeParams, 0, typeParam1));
}
/**
* Creates a type tag representing a given class type with given type parameters.
*
* With this factory method you can create type tags of class types with any number of type
* parameters. However, the type parameters are class types without their own type parameters.
*
* For example, the following code: {@code typeTag(List.class, String.class)} returns a type tag of
* {@code List}, and the following code: {@code typeTag(Map.class, String.class,
* Integer.class)} returns a type tag of {@code Map}.
*
* This is a short-hand method for calling {@link #typeTag(Class, TypeTags.TypeTag,
* TypeTags.TypeTag[])} and providing type parameters with {@link #safeTypeTag(Class)}.
*/
public static TypeTags.TypeTag typeTag(Class main, Class typeParam1, Class... typeParams) {
TypeTags.TypeTag[] typeParamsTags = new TypeTags.TypeTag[typeParams.length + 1];
typeParamsTags[0] = typeTag(typeParam1);
for (int i = 0; i < typeParams.length; i++) {
typeParamsTags[i + 1] = typeTag(typeParams[i]);
}
return JavaApiHelper.getTypeTag(main, typeParamsTags);
}
// -------------------------------------------------------------------------
// Type converter resolvers
// -------------------------------------------------------------------------
/**
* Retrieves a type converter for the given class type.
*
* This is a short-hand method for calling {@link #typeConverter(TypeTags.TypeTag)} with an argument
* obtained by {@link #safeTypeTag(Class)}.
*/
public static TypeConverter safeTypeConverter(Class main) {
return TypeConverter$.MODULE$.forType(safeTypeTag(main));
}
/**
* Retrieves a type converter for a type represented by a given type tag.
*/
@SuppressWarnings("unchecked")
public static TypeConverter typeConverter(TypeTags.TypeTag typeTag) {
return (TypeConverter) TypeConverter$.MODULE$.forType(typeTag);
}
/**
* Retrieves a type converter for the given class type.
*
* This is a short-hand method for calling {@link #typeConverter(TypeTags.TypeTag)} with an argument
* obtained by {@link #safeTypeTag(Class)}.
*/
@SuppressWarnings("unchecked")
public static TypeConverter typeConverter(Class main) {
return TypeConverter$.MODULE$.forType(CassandraJavaUtil.typeTag(main));
}
/**
* Retrieves a type converter for the given class type with the given type parameters.
*
* This is a short-hand method for calling {@link #typeConverter(TypeTags.TypeTag)} with an argument
* obtained by {@link #typeTag(Class, Class, Class[])}.
*/
@SuppressWarnings("unchecked")
public static TypeConverter typeConverter(Class main, Class typeParam1, Class... typeParams) {
return TypeConverter$.MODULE$.forType(CassandraJavaUtil.typeTag(main, typeParam1, typeParams));
}
/**
* Retrieves a type converter for the given class type with the given type parameters.
*
* This is a short-hand method for calling {@link #typeConverter(TypeTags.TypeTag)} with an argument
* obtained by {@link #typeTag(Class, TypeTags.TypeTag, TypeTags.TypeTag[])}.
*/
@SuppressWarnings("unchecked")
public static TypeConverter typeConverter(
Class main,
TypeTags.TypeTag typeParam1,
TypeTags.TypeTag... typeParams
) {
return TypeConverter$.MODULE$.forType(CassandraJavaUtil.typeTag(main, typeParam1, typeParams));
}
// -------------------------------------------------------------------------
// Single column mappers
// -------------------------------------------------------------------------
/**
* Constructs a row reader factory which maps a single column from a row to the instance of a given
* class.
*
* It is a short-hand method for calling {@link #mapColumnTo(TypeConverter)} with the type converter
* resolved by {@link #typeConverter(Class)} method.
*/
public static RowReaderFactory mapColumnTo(Class targetClass) {
return new ValueRowReaderFactory<>(CassandraJavaUtil.typeConverter(targetClass));
}
/**
* Constructs a row reader factory which maps a single column from a row to the list of objects of a
* specified type.
*
* The method should be used when we have to deal with a column of some collection type (either a set
* or a list).
*
* For example the following code: {@code javaFunctions(sc).cassandraTable("ks", "tab",
* mapColumnToListOf(String.class))} returns an RDD of {@code List}.
*
* It is a short-hand method for calling {@link #mapColumnTo(Class, Class, Class[])} with the main
* type being {@link java.util.List} and the type parameter being the given target class.
*/
public static RowReaderFactory> mapColumnToListOf(Class targetClass) {
return new ValueRowReaderFactory<>(CassandraJavaUtil.>typeConverter(List.class, targetClass));
}
/**
* Constructs a row reader factory which maps a single column from a row to the set of objects of a
* specified type.
*
* The method should be used when we have to deal with a column of some collection type (either a set
* or a list).
*
* For example the following code: {@code javaFunctions(sc).cassandraTable("ks", "tab",
* mapColumnToSetOf(String.class))} returns an RDD of {@code Set}.
*
* It is a short-hand method for calling {@link #mapColumnTo(Class, Class, Class[])} with the main
* type being {@link java.util.Set} and the type parameter being the given target class.
*/
public static RowReaderFactory> mapColumnToSetOf(Class targetClass) {
return new ValueRowReaderFactory<>(CassandraJavaUtil.>typeConverter(Set.class, targetClass));
}
/**
* Constructs a row reader factory which maps a single column from a row to the map of pairs of
* specified key and value types.
*
* The method should be used when we have to deal with a column of a map type.
*
* For example the following code: {@code javaFunctions(sc).cassandraTable("ks", "tab",
* mapColumnToMapOf(String.class, Integer.class))} returns an RDD of {@code Map}.
*
* It is a short-hand method for calling {@link #mapColumnTo(Class, Class, Class[])} with the main
* type being {@link java.util.Map} and the type parameters being the given target key and target
* value classes.
*/
public static RowReaderFactory> mapColumnToMapOf(
Class targetKeyClass,
Class targetValueClass
) {
return new ValueRowReaderFactory<>(
CassandraJavaUtil.>typeConverter(Map.class, targetKeyClass, targetValueClass));
}
/**
* Constructs a row reader factory which maps a single column from a row to the objects of a
* specified type with given type parameters.
*
* It is a short-hand method for calling {@link #mapColumnTo(TypeConverter)} with a type converter
* resolved by calling {@link #typeConverter(Class, Class, Class[])}.
*/
@SuppressWarnings("unchecked")
public static RowReaderFactory mapColumnTo(Class targetClass, Class typeParam1, Class... typeParams) {
return new ValueRowReaderFactory<>(CassandraJavaUtil.typeConverter(targetClass, typeParam1, typeParams));
}
/**
* Constructs a row reader factory which maps a single column from a row using a specified type
* converter.
*
* Unlike {@code mapRowTo(...)} methods, this method converts the first available column from the
* result set. Therefore, it is strongly recommended to explicitly specify the projection on the RDD
* in order to decrease the amount of data to be fetched and to make it explicit which column is to
* be mapped.
*/
public static RowReaderFactory mapColumnTo(TypeConverter typeConverter) {
return new ValueRowReaderFactory<>(typeConverter);
}
// -------------------------------------------------------------------------
// Entire row mappers
// -------------------------------------------------------------------------
/**
* Constructs a row reader factory which maps an entire row to an object of a specified type.
*
* The given target class is considered to follow the Java bean style convention. It should have a
* public, no-args constructor, as well as a set of getters and setters. The names of the attributes
* must follow the names of columns in the table (however, different conventions are allowed - see
* the guide for more info). The default mapping of attributes to column names can be overridden by
* providing a custom map of attribute-column mappings for the pairs which do not follow the general
* convention.
*
* Another version of this method {@link #mapRowTo(Class, Pair[])} can be considered much more handy
* for inline invocations.
*
* The method uses {@link JavaBeanColumnMapper} as the column mapper. If another column mapper has to
* be used, see {@link #mapRowTo(Class, ColumnMapper)} method.
*/
public static RowReaderFactory mapRowTo(Class targetClass, Map columnMapping) {
TypeTags.TypeTag tt = typeTag(targetClass);
ColumnMapper mapper = javaBeanColumnMapper(safeClassTag(targetClass), columnMapping);
return new ClassBasedRowReaderFactory<>(tt, mapper);
}
/**
* Constructs a row reader factory which maps an entire row to an object of a specified type.
*
* The given target class is considered to follow the Java bean style convention. It should have a
* public, no-args constructor, as well as a set of getters and setters. The names of the attributes
* must follow the names of columns in the table (however, different conventions are allowed - see
* the guide for more info). The default mapping of attributes to column names can be changed by
* providing a custom map of attribute-column mappings for the pairs which do not follow the general
* convention.
*
* The method uses {@link JavaBeanColumnMapper} as the column mapper. If another column mapper has to
* be used, see {@link #mapRowTo(Class, ColumnMapper)} method.
*/
public static RowReaderFactory mapRowTo(Class targetClass, Pair... columnMappings) {
TypeTags.TypeTag tt = typeTag(targetClass);
ColumnMapper mapper = javaBeanColumnMapper(safeClassTag(targetClass), convertToMap(columnMappings));
return new ClassBasedRowReaderFactory<>(tt, mapper);
}
/**
* Constructs a row reader factory which maps an entire row using a specified column mapper. Follow
* the guide to find out more about column mappers.
*/
public static RowReaderFactory mapRowTo(Class targetClass, ColumnMapper columnMapper) {
TypeTags.TypeTag tt = typeTag(targetClass);
return new ClassBasedRowReaderFactory<>(tt, columnMapper);
}
// -------------------------------------------------------------------------
// Row to tuples mapper
// -------------------------------------------------------------------------
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple1.class,
typeTag(a)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple1ColumnMapper(a));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple2.class,
typeTag(a),
typeTag(b)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple2ColumnMapper(a, b));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple3.class,
typeTag(a),
typeTag(b),
typeTag(c)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple3ColumnMapper(a, b, c));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple4.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple4ColumnMapper(a, b, c, d));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple5.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple5ColumnMapper(a, b, c, d, e));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple6.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e),
typeTag(f)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple6ColumnMapper(a, b, c, d, e, f));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f,
Class g
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple7.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e),
typeTag(f),
typeTag(g)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple7ColumnMapper(a, b, c, d, e, f, g));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f,
Class g,
Class h
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple8.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e),
typeTag(f),
typeTag(g),
typeTag(h)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple8ColumnMapper(a, b, c, d, e, f, g, h));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f,
Class g,
Class h,
Class i
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple9.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e),
typeTag(f),
typeTag(g),
typeTag(h),
typeTag(i)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple9ColumnMapper(a, b, c, d, e, f, g, h, i));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f,
Class g,
Class h,
Class i,
Class j
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple10.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e),
typeTag(f),
typeTag(g),
typeTag(h),
typeTag(i),
typeTag(j)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple10ColumnMapper(a, b, c, d, e, f, g, h, i, j));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f,
Class g,
Class h,
Class i,
Class j,
Class k
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple11.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e),
typeTag(f),
typeTag(g),
typeTag(h),
typeTag(i),
typeTag(j),
typeTag(k)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple11ColumnMapper(a, b, c, d, e, f, g, h, i, j, k));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f,
Class g,
Class h,
Class i,
Class j,
Class k,
Class l
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple12.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e),
typeTag(f),
typeTag(g),
typeTag(h),
typeTag(i),
typeTag(j),
typeTag(k),
typeTag(l)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple12ColumnMapper(a, b, c, d, e, f, g, h, i, j, k, l));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f,
Class g,
Class h,
Class i,
Class j,
Class k,
Class l,
Class m
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple13.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e),
typeTag(f),
typeTag(g),
typeTag(h),
typeTag(i),
typeTag(j),
typeTag(k),
typeTag(l),
typeTag(m)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple13ColumnMapper(a, b, c, d, e, f, g, h, i, j, k, l, m));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f,
Class g,
Class h,
Class i,
Class j,
Class k,
Class l,
Class m,
Class n
) {
final TypeTags.TypeTag> tupleTT =
typeTag(Tuple14.class,
typeTag(a),
typeTag(b),
typeTag(c),
typeTag(d),
typeTag(e),
typeTag(f),
typeTag(g),
typeTag(h),
typeTag(i),
typeTag(j),
typeTag(k),
typeTag(l),
typeTag(m),
typeTag(n)
);
return new ClassBasedRowReaderFactory<>(tupleTT, tuple14ColumnMapper(a, b, c, d, e, f, g, h, i, j, k, l, m, n));
}
/**
* Creates a RowReaderFactory instance for reading tuples with given parameter types.
*/
public static RowReaderFactory> mapRowToTuple(
Class a,
Class b,
Class c,
Class d,
Class e,
Class f,
Class g,
Class h,
Class i,
Class j,
Class k,
Class l,
Class m,
Class n,
Class