org.cloudgraph.hbase.mapreduce.GraphInputFormat Maven / Gradle / Ivy
/**
* CloudGraph Community Edition (CE) License
*
* This is a community release of CloudGraph, a dual-license suite of
* Service Data Object (SDO) 2.1 services designed for relational and
* big-table style "cloud" databases, such as HBase and others.
* This particular copy of the software is released under the
* version 2 of the GNU General Public License. CloudGraph was developed by
* TerraMeta Software, Inc.
*
* Copyright (c) 2013, TerraMeta Software, Inc. All rights reserved.
*
* General License information can be found below.
*
* This distribution may include materials developed by third
* parties. For license and attribution notices for these
* materials, please refer to the documentation that accompanies
* this distribution (see the "Licenses for Third-Party Components"
* appendix) or view the online documentation at
*
* Data graphs are assembled within a record {@link GraphRecordReader reader}
* based on the detailed selection criteria within a given
* query, and
* may be passed to a {@link GraphRecordRecognizer recognizer} and potentially
* screened from client {@link GraphMapper mappers} potentially illuminating
* business logic dedicated to identifying specific records.
*
*
* A graph recognizer is used only when query expressions are present which
* reference properties not found in the row key model for a target
* graph.
*
*
* @see org.apache.hadoop.hbase.mapreduce.TableSplit
* @see GraphRecordReader
* @see GraphRecordRecognizer
*
* @author Scott Cinnamond
* @since 0.5.8
*/
public class GraphInputFormat extends InputFormatimplements Configurable {
static final Log log = LogFactory.getLog(GraphInputFormat.class);
/**
* Serialized query which encapsulates meta data which drives the creation
* is one or more scans against any number of tables, starting with the root.
*/
public static final String QUERY = "cloudgraph.hbase.mapreduce.query";
/**
* Serialized store mappings or configuration which links physical tables with any number of
* data graphs/mappings. This data is typically packaged into a jar in an e.g. cloudgraph-config.xml
* file, but where these mappings are dynamic and loaded at runtime, the job (spark,mapreduce) child
* VM's must have access to and load this mapping dynamically.
* The mapping must be serialized as XML and be marshalled and be unmarshallable using the
* root {@link CloudGraphConfiguration} which may have any number of table mapping elements.
*/
public static final String TABLE_MAPPINGS = "cloudgraph.hbase.mapreduce.tablemappings";
/**
* Boolean property indicating whether a graph recognizer is necessary for
* the current query
*/
public static final String RECOGNIZER = "cloudgraph.hbase.mapreduce.recognizer";
/** Internal Job parameter that specifies the scan list. */
protected static final String SCANS = "cloudgraph.hbase.mapreduce.scans";
/**
* Internal Job parameter that specifies root the input table as derived
* from a deserialized query
*/
protected static final String ROOT_TABLE = "cloudgraph.hbase.mapreduce.roottable";
/** The timestamp used to filter columns with a specific timestamp. */
protected static final String SCAN_TIMESTAMP = "cloudgraph.hbase.mapreduce.scan.timestamp";
/**
* The starting timestamp used to filter columns with a specific range of
* versions.
*/
protected static final String SCAN_TIMERANGE_START = "cloudgraph.hbase.mapreduce.scan.timerange.start";
/**
* The ending timestamp used to filter columns with a specific range of
* versions.
*/
protected static final String SCAN_TIMERANGE_END = "cloudgraph.hbase.mapreduce.scan.timerange.end";
/** The maximum number of version to return. */
protected static final String SCAN_MAXVERSIONS = "cloudgraph.hbase.mapreduce.scan.maxversions";
/** Set to false to disable server-side caching of blocks for this scan. */
public static final String SCAN_CACHEBLOCKS = "cloudgraph.hbase.mapreduce.scan.cacheblocks";
/** The number of rows for caching that will be passed to scanners. */
public static final String SCAN_CACHEDROWS = "cloudgraph.hbase.mapreduce.scan.cachedrows";
/** The configuration. */
private Configuration conf = null;
/** The {@link Admin}. */
private Admin admin;
/** The root {@link Table} to scan. */
private Table table;
/** The {@link RegionLocator} of the table. */
private RegionLocator regionLocator;
/** The reader scanning the table, can be a custom one. */
private TableRecordReader tableRecordReader = null;
/** The underlying {@link Connection} of the table. */
private Connection connection;
/** Holds the set of scans used to define the input. */
private List scans;
/** The reader scanning the table, can be a custom one. */
private GraphRecordReader graphRecordReader = null;
private HashMap reverseDNSCache = new HashMap();
@Override
public Configuration getConf() {
return this.conf;
}
@Override
public void setConf(Configuration configuration) {
this.conf = configuration;
// String tableName = conf.get(ROOT_TABLE);
TableName tableName = TableName.valueOf(conf.get(ROOT_TABLE));
try {
// table = new HTable(new Configuration(conf), tableName);
Connection con = ConnectionFactory.createConnection(new Configuration(conf));
this.table = con.getTable(tableName);
this.regionLocator = con.getRegionLocator(tableName);
this.admin = con.getAdmin();
this.connection = con;
} catch (Exception e) {
log.error(StringUtils.stringifyException(e));
}
String[] rawScans = conf.getStrings(SCANS);
if (rawScans.length <= 0) {
throw new IllegalArgumentException("There must be at least 1 scan configuration set to : " + SCANS);
}
List scans = new ArrayList();
for (int i = 0; i < rawScans.length; i++) {
try {
Scan scan = GraphMapReduceSetup.convertStringToScan(rawScans[i]);
setConf(scan, configuration);
scans.add(scan);
} catch (IOException e) {
throw new RuntimeException("Failed to convert Scan : " + rawScans[i] + " to string", e);
}
}
this.setScans(scans);
}
private void setConf(Scan scan, Configuration configuration) throws NumberFormatException, IOException {
if (conf.get(SCAN_TIMESTAMP) != null) {
scan.setTimeStamp(Long.parseLong(conf.get(SCAN_TIMESTAMP)));
}
if (conf.get(SCAN_TIMERANGE_START) != null && conf.get(SCAN_TIMERANGE_END) != null) {
scan.setTimeRange(Long.parseLong(conf.get(SCAN_TIMERANGE_START)),
Long.parseLong(conf.get(SCAN_TIMERANGE_END)));
}
if (conf.get(SCAN_MAXVERSIONS) != null) {
scan.setMaxVersions(Integer.parseInt(conf.get(SCAN_MAXVERSIONS)));
}
if (conf.get(SCAN_CACHEDROWS) != null) {
scan.setCaching(Integer.parseInt(conf.get(SCAN_CACHEDROWS)));
}
// false by default, full table scans generate too much GC churn
scan.setCacheBlocks((conf.getBoolean(SCAN_CACHEBLOCKS, false)));
}
@Override
public List getSplits(JobContext context) throws IOException, InterruptedException {
if (scans.isEmpty()) {
throw new IOException("No scans were provided.");
}
try {
RegionSizeCalculator sizeCalculator = new RegionSizeCalculator(regionLocator, admin);
Pair keys = getStartEndKeys();
// if potentially a single split (or table is empty?)
if (keys == null || keys.getFirst() == null || keys.getFirst().length == 0) {
HRegionLocation regLoc = regionLocator.getRegionLocation(HConstants.EMPTY_BYTE_ARRAY, false);
if (null == regLoc) {
throw new IOException("Expecting at least one region.");
}
List splits = new ArrayList(1);
long regionSize = sizeCalculator.getRegionSize(regLoc.getRegionInfo().getRegionName());
Scan scan = scans.get(0);
if (scans.size() > 1)
log.warn("single split with multiple scans - ignoring other than first scan");
TableSplit split = new TableSplit(table.getName(), scan, HConstants.EMPTY_BYTE_ARRAY,
HConstants.EMPTY_BYTE_ARRAY,
regLoc.getHostnamePort().split(Addressing.HOSTNAME_PORT_SEPARATOR)[0], regionSize);
splits.add(split);
return splits;
}
List splits = new ArrayList(keys.getFirst().length);
for (Scan scan : scans) {
for (int i = 0; i < keys.getFirst().length; i++) {
if (!includeRegionInSplit(keys.getFirst()[i], keys.getSecond()[i])) {
continue;
}
HRegionLocation location = regionLocator.getRegionLocation(keys.getFirst()[i], false);
// The below InetSocketAddress creation does a name
// resolution.
InetSocketAddress isa = new InetSocketAddress(location.getHostname(), location.getPort());
if (isa.isUnresolved()) {
log.warn("Failed resolve " + isa);
}
InetAddress regionAddress = isa.getAddress();
String regionLocation;
try {
regionLocation = reverseDNS(regionAddress);
} catch (NamingException e) {
log.warn("Cannot resolve the host name for " + regionAddress + " because of " + e);
regionLocation = location.getHostname();
}
byte[] startRow = scan.getStartRow();
byte[] stopRow = scan.getStopRow();
// determine if the given start an stop key fall into the
// region.
if ((startRow.length == 0 || keys.getSecond()[i].length == 0
|| Bytes.compareTo(startRow, keys.getSecond()[i]) < 0)
&& (stopRow.length == 0 || Bytes.compareTo(stopRow, keys.getFirst()[i]) > 0)) {
byte[] splitStart = startRow.length == 0 || Bytes.compareTo(keys.getFirst()[i], startRow) >= 0
? keys.getFirst()[i] : startRow;
byte[] splitStop = (stopRow.length == 0 || Bytes.compareTo(keys.getSecond()[i], stopRow) <= 0)
&& keys.getSecond()[i].length > 0 ? keys.getSecond()[i] : stopRow;
byte[] regionName = location.getRegionInfo().getRegionName();
long regionSize = sizeCalculator.getRegionSize(regionName);
TableSplit split = new TableSplit(table.getName(),
scan, // must include the scan as it may have various filters
splitStart, splitStop, regionLocation,
regionSize);
splits.add(split);
if (log.isDebugEnabled()) {
log.debug("getSplits: split -> " + i + " -> " + split);
}
}
}
}
return splits;
} finally {
closeAll();
}
}
/**
* Uses {@link InetAddress} in case of {@link DNS} lookup failure.
* @param ipAddr the address
* @return the reverse address
* @throws NamingException
* @throws UnknownHostException
*/
public String reverseDNS(InetAddress ipAddr) throws NamingException, UnknownHostException {
String hostName = this.reverseDNSCache.get(ipAddr);
if (hostName == null) {
String ipAddressString = null;
try {
ipAddressString = DNS.reverseDns(ipAddr, null);
} catch (Exception e) {
ipAddressString = InetAddress.getByName(ipAddr.getHostAddress()).getHostName();
}
if (ipAddressString == null)
throw new UnknownHostException("No host found for " + ipAddr);
hostName = Strings.domainNamePointerToHostName(ipAddressString);
this.reverseDNSCache.put(ipAddr, hostName);
}
return hostName;
}
private void closeAll() throws IOException {
close(admin, table, regionLocator, connection);
admin = null;
table = null;
regionLocator = null;
connection = null;
}
private void close(Closeable... closables) throws IOException {
for (Closeable c : closables) {
if (c != null) {
c.close();
}
}
}
protected Pair getStartEndKeys() throws IOException {
return this.regionLocator.getStartEndKeys();
}
@Override
public RecordReader createRecordReader(InputSplit split,
TaskAttemptContext context) throws IOException, InterruptedException {
if (table == null) {
throw new IOException("Cannot create a record reader because of a"
+ " previous error. Please look at the previous logs lines from"
+ " the task's full log for more details.");
}
TableSplit tSplit = (TableSplit) split;
GraphRecordReader reader = this.graphRecordReader;
// if no record reader was provided use default
if (reader == null) {
reader = new GraphRecordReader();
}
Scan sc = tSplit.getScan();
log.debug("SCAN: " + sc.toString());
if (sc.getFilter() != null) {
log.info("SPLIT FILTER: " + FilterUtil.printFilterTree(sc.getFilter()));
}
else {
log.info("split scan has no filter");
}
sc.setStartRow(tSplit.getStartRow());
sc.setStopRow(tSplit.getEndRow());
reader.setScan(sc);
reader.setTable(table);
try {
reader.initialize(tSplit, context);
} catch (InterruptedException e) {
throw new InterruptedIOException(e.getMessage());
}
return reader;
}
/**
* Test if the given region is to be included in the InputSplit while
* splitting the regions of a table.
*
* This optimization is effective when there is a specific reasoning to
* exclude an entire region from the M-R job, (and hence, not contributing
* to the InputSplit), given the start and end keys of the same.
* Useful when we need to remember the last-processed top record and revisit
* the [last, current) interval for M-R processing, continuously. In
* addition to reducing InputSplits, reduces the load on the region server
* as well, due to the ordering of the keys.
*
* Note: It is possible that endKey.length() == 0 , for the
* last (recent) region.
* Override this method, if you want to bulk exclude regions altogether from
* M-R. By default, no region is excluded( i.e. all regions are included).
*
* @param startKey
* Start key of the region
* @param endKey
* End key of the region
* @return true, if this region needs to be included as part of the input
* (default).
*/
protected boolean includeRegionInSplit(final byte[] startKey, final byte[] endKey) {
return true;
}
/**
* Allows subclasses to get the list of {@link Scan} objects.
*/
protected List getScans() {
return this.scans;
}
/**
* Allows subclasses to set the list of {@link Scan} objects.
*
* @param scans
* The list of {@link Scan} used to define the input
*/
protected void setScans(List scans) {
this.scans = scans;
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy