com.pivotal.pxf.plugins.gemfirexd.GemFireXDFragmenter Maven / Gradle / Ivy

Go to download
Show more of this group Show more artifacts with this name
Show all versions of gemfirexd-core Show documentation
SnappyData store based off Pivotal GemFireXD
There is a newer version: 2.0-BETA
/*
 * Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License. See accompanying
 * LICENSE file.
 */
/**
 * 
 */
package com.pivotal.pxf.plugins.gemfirexd;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.lib.CombineFileSplit;

import com.pivotal.pxf.api.Fragment;
import com.pivotal.pxf.api.Fragmenter;
import com.pivotal.pxf.api.utilities.InputData;
import com.pivotal.pxf.plugins.gemfirexd.util.GemFireXDManager;
import com.pivotal.gemfirexd.hadoop.mapred.Key;
import com.pivotal.gemfirexd.hadoop.mapred.Row;

/**
 * This is very similar to HdfsDataFragmenter except the use of GemFireXDManager
 * utility.
 */
public class GemFireXDFragmenter  extends Fragmenter {

  private JobConf jobConf;

  private GemFireXDManager gfxdManager;

  /**
   * @param inputData
   * @throws IOException
   */
  public GemFireXDFragmenter(InputData inputData) throws Exception {
    this(inputData, new GemFireXDManager(inputData, true));
  }

  /**
   * @param inputData
   * @param mgr
   * @throws Exception
   */
  public GemFireXDFragmenter(InputData inputData, GemFireXDManager mgr)
      throws Exception {
    super(inputData);
    this.jobConf = new JobConf(new Configuration(), GemFireXDFragmenter.class);

    this.gfxdManager = mgr;
    this.gfxdManager.configureJob(this.jobConf, this.gfxdManager.getHomeDir());

    String msg = this.gfxdManager.verifyUserAttributes();

    if (msg != null && !msg.equals("")) {
      throw new IllegalArgumentException(msg);
    }
  }

  @Override
  public List getFragments() throws IOException {
    InputSplit[] splits;
//    try {
      splits = getSplits();
//    } finally {
//      this.gfxdManager.resetLonerSystemInUse();
//    }

    for (InputSplit split : splits) {
      CombineFileSplit cSplit = (CombineFileSplit)split;
      
      if (cSplit.getLength() > 0L) {
        String filepath = cSplit.getPath(0).toUri().getPath();
        filepath = filepath.substring(1);
        if (this.gfxdManager.getLogger().isDebugEnabled()) {
          this.gfxdManager.getLogger().debug("fragment-filepath " + filepath);
        }
        byte[] data = this.gfxdManager.populateUserData(cSplit);
        this.fragments.add(new Fragment(filepath, cSplit.getLocations(), data));
      }
    }
    return this.fragments;
  }

  private InputSplit[] getSplits() throws IOException {
    InputFormat inputFormat = this.gfxdManager.getInputFormat();
    try {
      return inputFormat.getSplits(this.jobConf, 1);
    } catch (FileNotFoundException fnfe) {
      throw new FileNotFoundException(
          "Table "
              + this.gfxdManager.getTable()
              + " not found. "
              + "The LOCATION string may contain incorrect value for one or more of the following:"
              + "1. Path to HDFSSTORE (homeDir), 2. Schema name or 3. Table name. "
              + GemFireXDManager.LOCATION_FORMAT);
    }
  }

}