org.apache.crunch.impl.spark.collect.PGroupedTableImpl Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of crunch-spark Show documentation
There is a newer version: 1.0.0
/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.crunch.impl.spark.collect;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.crunch.CombineFn;
import org.apache.crunch.GroupingOptions;
import org.apache.crunch.impl.dist.collect.BaseGroupedTable;
import org.apache.crunch.impl.dist.collect.PTableBase;
import org.apache.crunch.impl.spark.ByteArray;
import org.apache.crunch.impl.spark.SparkCollection;
import org.apache.crunch.impl.spark.SparkComparator;
import org.apache.crunch.impl.spark.SparkPartitioner;
import org.apache.crunch.impl.spark.SparkRuntime;
import org.apache.crunch.impl.spark.fn.CombineMapsideFunction;
import org.apache.crunch.impl.spark.fn.MapOutputFunction;
import org.apache.crunch.impl.spark.fn.PairMapFunction;
import org.apache.crunch.impl.spark.fn.PairMapIterableFunction;
import org.apache.crunch.impl.spark.fn.PartitionedMapOutputFunction;
import org.apache.crunch.impl.spark.fn.ReduceGroupingFunction;
import org.apache.crunch.impl.spark.fn.ReduceInputFunction;
import org.apache.crunch.impl.spark.serde.AvroSerDe;
import org.apache.crunch.impl.spark.serde.SerDe;
import org.apache.crunch.impl.spark.serde.WritableSerDe;
import org.apache.crunch.types.PTableType;
import org.apache.crunch.types.avro.AvroType;
import org.apache.crunch.types.writable.WritableType;
import org.apache.crunch.util.PartitionUtils;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDDLike;
import org.apache.spark.storage.StorageLevel;

import java.util.List;

public class PGroupedTableImpl extends BaseGroupedTable implements SparkCollection {

  private static final Log LOG = LogFactory.getLog(PGroupedTableImpl.class);

  private JavaRDDLike rdd;

  PGroupedTableImpl(PTableBase parent, GroupingOptions groupingOptions) {
    super(parent, groupingOptions);
  }

  public JavaRDDLike getJavaRDDLike(SparkRuntime runtime) {
    if (!runtime.isValid(rdd)) {
      rdd = getJavaRDDLikeInternal(runtime, runtime.getCombineFn());
      rdd.rdd().setName(getName());
      StorageLevel sl = runtime.getStorageLevel(this);
      if (sl != null) {
        rdd.rdd().persist(sl);
      }
    }
    return rdd;
  }

  private JavaRDDLike getJavaRDDLikeInternal(SparkRuntime runtime, CombineFn combineFn) {
    JavaPairRDD parentRDD = (JavaPairRDD) ((SparkCollection)getOnlyParent()).getJavaRDDLike(runtime);
    if (combineFn != null) {
      parentRDD = parentRDD.mapPartitions(new CombineMapsideFunction(combineFn, runtime.getRuntimeContext()));
    }
    SerDe keySerde, valueSerde;
    PTableType parentType = ptype.getTableType();
    if (parentType instanceof AvroType) {
      keySerde = new AvroSerDe((AvroType) parentType.getKeyType());
      valueSerde = new AvroSerDe((AvroType) parentType.getValueType());
    } else {
      keySerde = new WritableSerDe(((WritableType) parentType.getKeyType()).getSerializationClass());
      valueSerde = new WritableSerDe(((WritableType) parentType.getValueType()).getSerializationClass());
    }

    int numPartitions = (groupingOptions.getNumReducers() > 0) ? groupingOptions.getNumReducers() :
        PartitionUtils.getRecommendedPartitions(this, getPipeline().getConfiguration());
    if (numPartitions <= 0) {
      LOG.warn("Attempted to set a non-positive number of partitions");
      numPartitions = 1;
    }

    JavaPairRDD> groupedRDD;
    if (groupingOptions.getPartitionerClass() != null) {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .map(new PartitionedMapOutputFunction(keySerde, valueSerde, ptype, groupingOptions.getPartitionerClass(),
              numPartitions, runtime.getRuntimeContext()))
          .groupByKey(new SparkPartitioner(numPartitions));
    } else {
      groupedRDD = parentRDD
          .map(new PairMapFunction(ptype.getOutputMapFn(), runtime.getRuntimeContext()))
          .map(new MapOutputFunction(keySerde, valueSerde))
          .groupByKey(numPartitions);
    }

    if (groupingOptions.requireSortedKeys() || groupingOptions.getSortComparatorClass() != null) {
      SparkComparator scmp = new SparkComparator(groupingOptions, ptype, runtime.getRuntimeContext());
      groupedRDD = groupedRDD.sortByKey(scmp);
    }
    if (groupingOptions.getGroupingComparatorClass() != null) {
      groupedRDD = groupedRDD.mapPartitions(
          new ReduceGroupingFunction(groupingOptions, ptype, runtime.getRuntimeContext()));
    }

    return groupedRDD
        .map(new ReduceInputFunction(keySerde, valueSerde))
        .map(new PairMapIterableFunction(ptype.getInputMapFn(), runtime.getRuntimeContext()));
  }
}