org.apache.hadoop.mapred.gridmix.InputStriper Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.mapred.gridmix;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.mapreduce.lib.input.CombineFileSplit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
/**
* Given a {@link #FilePool}, obtain a set of files capable of satisfying
* a full set of splits, then iterate over each source to fill the request.
*/
class InputStriper {
public static final Log LOG = LogFactory.getLog(InputStriper.class);
int idx;
long currentStart;
FileStatus current;
final List files = new ArrayList();
final Configuration conf = new Configuration();
/**
* @param inputDir Pool from which files are requested.
* @param mapBytes Sum of all expected split requests.
*/
InputStriper(FilePool inputDir, long mapBytes)
throws IOException {
final long inputBytes = inputDir.getInputFiles(mapBytes, files);
if (mapBytes > inputBytes) {
LOG.warn("Using " + inputBytes + "/" + mapBytes + " bytes");
}
if (files.isEmpty() && mapBytes > 0) {
throw new IOException("Failed to satisfy request for " + mapBytes);
}
current = files.isEmpty() ? null : files.get(0);
}
/**
* @param inputDir Pool used to resolve block locations.
* @param bytes Target byte count
* @param nLocs Number of block locations per split.
* @return A set of files satisfying the byte count, with locations weighted
* to the dominating proportion of input bytes.
*/
CombineFileSplit splitFor(FilePool inputDir, long bytes, int nLocs)
throws IOException {
final ArrayList paths = new ArrayList();
final ArrayList start = new ArrayList();
final ArrayList length = new ArrayList();
final HashMap sb = new HashMap();
do {
paths.add(current.getPath());
start.add(currentStart);
final long fromFile = Math.min(bytes, current.getLen() - currentStart);
length.add(fromFile);
for (BlockLocation loc :
inputDir.locationsFor(current, currentStart, fromFile)) {
final double tedium = loc.getLength() / (1.0 * bytes);
for (String l : loc.getHosts()) {
Double j = sb.get(l);
if (null == j) {
sb.put(l, tedium);
} else {
sb.put(l, j.doubleValue() + tedium);
}
}
}
currentStart += fromFile;
bytes -= fromFile;
// Switch to a new file if
// - the current file is uncompressed and completely used
// - the current file is compressed
CompressionCodecFactory compressionCodecs =
new CompressionCodecFactory(conf);
CompressionCodec codec = compressionCodecs.getCodec(current.getPath());
if (current.getLen() - currentStart == 0
|| codec != null) {
current = files.get(++idx % files.size());
currentStart = 0;
}
} while (bytes > 0);
final ArrayList> sort =
new ArrayList>(sb.entrySet());
Collections.sort(sort, hostRank);
final String[] hosts = new String[Math.min(nLocs, sort.size())];
for (int i = 0; i < nLocs && i < sort.size(); ++i) {
hosts[i] = sort.get(i).getKey();
}
return new CombineFileSplit(paths.toArray(new Path[0]),
toLongArray(start), toLongArray(length), hosts);
}
private long[] toLongArray(final ArrayList sigh) {
final long[] ret = new long[sigh.size()];
for (int i = 0; i < ret.length; ++i) {
ret[i] = sigh.get(i);
}
return ret;
}
static final Comparator> hostRank =
new Comparator>() {
public int compare(Entry a, Entry b) {
final double va = a.getValue();
final double vb = b.getValue();
return va > vb ? -1 : va < vb ? 1 : 0;
}
};
}