com.ryft.spark.connector.japi.SparkContextJavaFunctions Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of spark-ryft-connector-java_2.11 Show documentation
spark-ryft-connector-java
The newest version!
/*
 * ============= Ryft-Customized BSD License ============
 * Copyright (c) 2015, Ryft Systems, Inc.
 * All rights reserved.
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *   this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *   this list of conditions and the following disclaimer in the documentation and/or
 *   other materials provided with the distribution.
 * 3. All advertising materials mentioning features or use of this software must display the following acknowledgement:
 *   This product includes software developed by Ryft Systems, Inc.
 * 4. Neither the name of Ryft Systems, Inc. nor the names of its contributors may be used
 *   to endorse or promote products derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY RYFT SYSTEMS, INC. ''AS IS'' AND ANY
 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL RYFT SYSTEMS, INC. BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 * ============
 */

package com.ryft.spark.connector.japi;

import com.ryft.spark.connector.*;
import com.ryft.spark.connector.domain.RyftData;
import com.ryft.spark.connector.domain.RyftFormatMode;
import com.ryft.spark.connector.domain.RyftQueryOptions;
import com.ryft.spark.connector.query.RecordQuery;
import com.ryft.spark.connector.query.RyftQuery;
import com.ryft.spark.connector.japi.rdd.RyftJavaRDD;
import com.ryft.spark.connector.japi.rdd.RyftPairJavaRDD;
import com.ryft.spark.connector.rdd.RyftPairRDD;
import com.ryft.spark.connector.rdd.RyftRDD;
import com.ryft.spark.connector.util.JavaApiHelper;
import org.apache.spark.SparkContext;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.sql.DataFrame;
import scala.*;
import scala.collection.immutable.HashMap;
import scala.collection.immutable.List;
import scala.collection.immutable.Set;
import scala.reflect.ClassTag;

import java.util.Map;

import static com.ryft.spark.connector.util.JavaApiHelper.getClassTag;

@SuppressWarnings({"UnusedDeclaration","unchecked"})
public class SparkContextJavaFunctions {
    private final SparkContext sparkContext;
    private final SparkContextFunctions sparkContextFunctions;

    SparkContextJavaFunctions(SparkContext sparkContext) {
        this.sparkContext = sparkContext;
        this.sparkContextFunctions = new SparkContextFunctions(sparkContext);
    }

    public  RyftJavaRDD toJavaRDD(RyftRDD rdd, Class targetClass) {
        final ClassTag classTag = getClassTag(targetClass);
        return new RyftJavaRDD<>(rdd, classTag);
    }

    public  RyftJavaRDD ryftRDD(RyftQuery ryftQuery,
                                      RyftQueryOptions queryOptions,
                                      Function1> choosePartitions,
                                      Function1> preferredLocations) {
        final RyftRDD ryftRDD = sparkContextFunctions.ryftRDD(JavaApiHelper.toScalaSeq(ryftQuery),
                queryOptions, choosePartitions, preferredLocations);
        return toJavaRDD(ryftRDD, RyftData.class);
    }

    public RyftJavaRDD ryftRDDStructured(RecordQuery ryftQuery,
                                         RyftQueryOptions queryOptions,
                                         Function1> choosePartitions,
                                         Function1> preferredLocations) {
        final RyftRDD ryftRDD =  sparkContextFunctions.ryftRDD(JavaApiHelper.toScalaSeq(ryftQuery),
                queryOptions, choosePartitions, preferredLocations);
        return toJavaRDD(ryftRDD, HashMap.HashTrieMap.class);
    }

    public  RyftPairJavaRDD> ryftPairJavaRDD(RyftQuery ryftQuery,
                                                                  RyftQueryOptions queryOptions,
                                                                  Function1> choosePartitions,
                                                                  Function1> preferredLocations) {
        final RyftPairRDD ryftPairRDD = sparkContextFunctions.ryftPairRDD(JavaApiHelper.toScalaSeq(ryftQuery),
                queryOptions, choosePartitions, preferredLocations);
        return toPairJavaRDD(ryftPairRDD, RyftData.class);
    }

    public  RyftPairJavaRDD> ryftPairJavaRDD(List ryftQuery,
                                                                  RyftQueryOptions queryOptions,
                                                                  Function1> choosePartitions,
                                                                  Function1> preferredLocations) {
        final RyftPairRDD ryftPairRDD = sparkContextFunctions.ryftPairRDD(ryftQuery,
                queryOptions, choosePartitions, preferredLocations);
        return toPairJavaRDD(ryftPairRDD, RyftData.class);
    }

    private  RyftPairJavaRDD toPairJavaRDD(RyftPairRDD rdd, Class targetClass) {
        final ClassTag classTag = getClassTag(targetClass);
        return new RyftPairJavaRDD<>(rdd, classTag);
    }

    public  void saveToRyft(JavaRDD rdd,
                               String path,
                               Map params) {
        final RDDFunctions rddFunctions = com.ryft.spark.connector.package$.MODULE$.toRDDFunctions(rdd.rdd());
        rddFunctions.saveToRyft(path, params);
    }

    public  void saveEachEntryToRyft(JavaPairRDD rdd,
                                        Map params) {
        final PairRDDFunctions rddFunctions = com.ryft.spark.connector.package$.MODULE$.toPairRDDFunctions(rdd.rdd());
        rddFunctions.saveEachEntryToRyft(params);
    }

    public void saveToRyft(DataFrame df,
                           String path,
                           Map params) {
        final DataFrameFunctions dfFunctions = com.ryft.spark.connector.package$.MODULE$.toDataFrameFunctions(df);
        dfFunctions.saveToRyft(path, params);
    }
}