org.apache.hadoop.hive.metastore.txn.CompactionTxnHandler Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.metastore.txn;
import com.facebook.presto.hive.$internal.org.apache.commons.logging.Log;
import com.facebook.presto.hive.$internal.org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.*;
import org.apache.hadoop.util.StringUtils;
import java.sql.*;
import java.util.*;
/**
* Extends the transaction handler with methods needed only by the compactor threads. These
* methods are not available through the thrift interface.
*/
public class CompactionTxnHandler extends TxnHandler {
static final private String CLASS_NAME = CompactionTxnHandler.class.getName();
static final private Log LOG = LogFactory.getLog(CLASS_NAME);
// Always access COMPACTION_QUEUE before COMPLETED_TXN_COMPONENTS
// See TxnHandler for notes on how to deal with deadlocks. Follow those notes.
public CompactionTxnHandler(HiveConf conf) {
super(conf);
}
/**
* This will look through the completed_txn_components table and look for partitions or tables
* that may be ready for compaction. Also, look through txns and txn_components tables for
* aborted transactions that we should add to the list.
* @param maxAborted Maximum number of aborted queries to allow before marking this as a
* potential compaction.
* @return list of CompactionInfo structs. These will not have id, type,
* or runAs set since these are only potential compactions not actual ones.
*/
public Set findPotentialCompactions(int maxAborted) throws MetaException {
Connection dbConn = null;
Set response = new HashSet();
Statement stmt = null;
try {
try {
dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
stmt = dbConn.createStatement();
// Check for completed transactions
String s = "select distinct ctc_database, ctc_table, " +
"ctc_partition from COMPLETED_TXN_COMPONENTS";
LOG.debug("Going to execute query <" + s + ">");
ResultSet rs = stmt.executeQuery(s);
while (rs.next()) {
CompactionInfo info = new CompactionInfo();
info.dbname = rs.getString(1);
info.tableName = rs.getString(2);
info.partName = rs.getString(3);
response.add(info);
}
// Check for aborted txns
s = "select tc_database, tc_table, tc_partition " +
"from TXNS, TXN_COMPONENTS " +
"where txn_id = tc_txnid and txn_state = '" + TXN_ABORTED + "' " +
"group by tc_database, tc_table, tc_partition " +
"having count(*) > " + maxAborted;
LOG.debug("Going to execute query <" + s + ">");
rs = stmt.executeQuery(s);
while (rs.next()) {
CompactionInfo info = new CompactionInfo();
info.dbname = rs.getString(1);
info.tableName = rs.getString(2);
info.partName = rs.getString(3);
info.tooManyAborts = true;
response.add(info);
}
LOG.debug("Going to rollback");
dbConn.rollback();
} catch (SQLException e) {
LOG.error("Unable to connect to transaction database " + e.getMessage());
checkRetryable(dbConn, e, "findPotentialCompactions");
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
}
return response;
}
catch (RetryException e) {
return findPotentialCompactions(maxAborted);
}
}
/**
* Sets the user to run as. This is for the case
* where the request was generated by the user and so the worker must set this value later.
* @param cq_id id of this entry in the queue
* @param user user to run the jobs as
*/
public void setRunAs(long cq_id, String user) throws MetaException {
try {
Connection dbConn = null;
Statement stmt = null;
try {
dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE);
stmt = dbConn.createStatement();
String s = "update COMPACTION_QUEUE set cq_run_as = '" + user + "' where cq_id = " + cq_id;
LOG.debug("Going to execute update <" + s + ">");
if (stmt.executeUpdate(s) != 1) {
LOG.error("Unable to update compaction record");
LOG.debug("Going to rollback");
dbConn.rollback();
}
LOG.debug("Going to commit");
dbConn.commit();
} catch (SQLException e) {
LOG.error("Unable to update compaction queue, " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
checkRetryable(dbConn, e, "setRunAs");
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
}
} catch (RetryException e) {
setRunAs(cq_id, user);
}
}
/**
* This will grab the next compaction request off of
* the queue, and assign it to the worker.
* @param workerId id of the worker calling this, will be recorded in the db
* @return an info element for this compaction request, or null if there is no work to do now.
*/
public CompactionInfo findNextToCompact(String workerId) throws MetaException {
try {
Connection dbConn = null;
CompactionInfo info = new CompactionInfo();
Statement stmt = null;
try {
dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE);
stmt = dbConn.createStatement();
String s = "select cq_id, cq_database, cq_table, cq_partition, " +
"cq_type from COMPACTION_QUEUE where cq_state = '" + INITIATED_STATE + "'";
LOG.debug("Going to execute query <" + s + ">");
ResultSet rs = stmt.executeQuery(s);
if (!rs.next()) {
LOG.debug("No compactions found ready to compact");
dbConn.rollback();
return null;
}
info.id = rs.getLong(1);
info.dbname = rs.getString(2);
info.tableName = rs.getString(3);
info.partName = rs.getString(4);
switch (rs.getString(5).charAt(0)) {
case MAJOR_TYPE: info.type = CompactionType.MAJOR; break;
case MINOR_TYPE: info.type = CompactionType.MINOR; break;
default: throw new MetaException("Unexpected compaction type " + rs.getString(5));
}
// Now, update this record as being worked on by this worker.
long now = getDbTime(dbConn);
s = "update COMPACTION_QUEUE set cq_worker_id = '" + workerId + "', " +
"cq_start = " + now + ", cq_state = '" + WORKING_STATE + "' where cq_id = " + info.id;
LOG.debug("Going to execute update <" + s + ">");
if (stmt.executeUpdate(s) != 1) {
LOG.error("Unable to update compaction record");
LOG.debug("Going to rollback");
dbConn.rollback();
}
LOG.debug("Going to commit");
dbConn.commit();
return info;
} catch (SQLException e) {
LOG.error("Unable to select next element for compaction, " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
checkRetryable(dbConn, e, "findNextToCompact");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
}
} catch (RetryException e) {
return findNextToCompact(workerId);
}
}
/**
* This will mark an entry in the queue as compacted
* and put it in the ready to clean state.
* @param info info on the compaction entry to mark as compacted.
*/
public void markCompacted(CompactionInfo info) throws MetaException {
try {
Connection dbConn = null;
Statement stmt = null;
try {
dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE);
stmt = dbConn.createStatement();
String s = "update COMPACTION_QUEUE set cq_state = '" + READY_FOR_CLEANING + "', " +
"cq_worker_id = null where cq_id = " + info.id;
LOG.debug("Going to execute update <" + s + ">");
if (stmt.executeUpdate(s) != 1) {
LOG.error("Unable to update compaction record");
LOG.debug("Going to rollback");
dbConn.rollback();
}
LOG.debug("Going to commit");
dbConn.commit();
} catch (SQLException e) {
LOG.error("Unable to update compaction queue " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
checkRetryable(dbConn, e, "markCompacted");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
}
} catch (RetryException e) {
markCompacted(info);
}
}
/**
* Find entries in the queue that are ready to
* be cleaned.
* @return information on the entry in the queue.
*/
public List findReadyToClean() throws MetaException {
Connection dbConn = null;
List rc = new ArrayList();
Statement stmt = null;
try {
try {
dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
stmt = dbConn.createStatement();
String s = "select cq_id, cq_database, cq_table, cq_partition, " +
"cq_type, cq_run_as from COMPACTION_QUEUE where cq_state = '" + READY_FOR_CLEANING + "'";
LOG.debug("Going to execute query <" + s + ">");
ResultSet rs = stmt.executeQuery(s);
while (rs.next()) {
CompactionInfo info = new CompactionInfo();
info.id = rs.getLong(1);
info.dbname = rs.getString(2);
info.tableName = rs.getString(3);
info.partName = rs.getString(4);
switch (rs.getString(5).charAt(0)) {
case MAJOR_TYPE: info.type = CompactionType.MAJOR; break;
case MINOR_TYPE: info.type = CompactionType.MINOR; break;
default: throw new MetaException("Unexpected compaction type " + rs.getString(5));
}
info.runAs = rs.getString(6);
rc.add(info);
}
LOG.debug("Going to rollback");
dbConn.rollback();
return rc;
} catch (SQLException e) {
LOG.error("Unable to select next element for cleaning, " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
checkRetryable(dbConn, e, "findReadyToClean");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
}
} catch (RetryException e) {
return findReadyToClean();
}
}
/**
* This will remove an entry from the queue after
* it has been compacted.
* @param info info on the compaction entry to remove
*/
public void markCleaned(CompactionInfo info) throws MetaException {
try {
Connection dbConn = null;
Statement stmt = null;
try {
dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE);
stmt = dbConn.createStatement();
String s = "delete from COMPACTION_QUEUE where cq_id = " + info.id;
LOG.debug("Going to execute update <" + s + ">");
if (stmt.executeUpdate(s) != 1) {
LOG.error("Unable to delete compaction record");
LOG.debug("Going to rollback");
dbConn.rollback();
}
// Remove entries from completed_txn_components as well, so we don't start looking there
// again.
s = "delete from COMPLETED_TXN_COMPONENTS where ctc_database = '" + info.dbname + "' and " +
"ctc_table = '" + info.tableName + "'";
if (info.partName != null) {
s += " and ctc_partition = '" + info.partName + "'";
}
LOG.debug("Going to execute update <" + s + ">");
if (stmt.executeUpdate(s) < 1) {
LOG.error("Expected to remove at least one row from completed_txn_components when " +
"marking compaction entry as clean!");
}
s = "select txn_id from TXNS, TXN_COMPONENTS where txn_id = tc_txnid and txn_state = '" +
TXN_ABORTED + "' and tc_database = '" + info.dbname + "' and tc_table = '" +
info.tableName + "'";
if (info.partName != null) s += " and tc_partition = '" + info.partName + "'";
LOG.debug("Going to execute update <" + s + ">");
ResultSet rs = stmt.executeQuery(s);
Set txnids = new HashSet();
while (rs.next()) txnids.add(rs.getLong(1));
if (txnids.size() > 0) {
// Remove entries from txn_components, as there may be aborted txn components
StringBuffer buf = new StringBuffer();
buf.append("delete from TXN_COMPONENTS where tc_txnid in (");
boolean first = true;
for (long id : txnids) {
if (first) first = false;
else buf.append(", ");
buf.append(id);
}
buf.append(") and tc_database = '");
buf.append(info.dbname);
buf.append("' and tc_table = '");
buf.append(info.tableName);
buf.append("'");
if (info.partName != null) {
buf.append(" and tc_partition = '");
buf.append(info.partName);
buf.append("'");
}
LOG.debug("Going to execute update <" + buf.toString() + ">");
int rc = stmt.executeUpdate(buf.toString());
LOG.debug("Removed " + rc + " records from txn_components");
// Don't bother cleaning from the txns table. A separate call will do that. We don't
// know here which txns still have components from other tables or partitions in the
// table, so we don't know which ones we can and cannot clean.
}
LOG.debug("Going to commit");
dbConn.commit();
} catch (SQLException e) {
LOG.error("Unable to delete from compaction queue " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
checkRetryable(dbConn, e, "markCleaned");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
}
} catch (RetryException e) {
markCleaned(info);
}
}
/**
* Clean up aborted transactions from txns that have no components in txn_components.
*/
public void cleanEmptyAbortedTxns() throws MetaException {
try {
Connection dbConn = null;
Statement stmt = null;
try {
dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE);
stmt = dbConn.createStatement();
String s = "select txn_id from TXNS where " +
"txn_id not in (select tc_txnid from TXN_COMPONENTS) and " +
"txn_state = '" + TXN_ABORTED + "'";
LOG.debug("Going to execute query <" + s + ">");
ResultSet rs = stmt.executeQuery(s);
Set txnids = new HashSet();
while (rs.next()) txnids.add(rs.getLong(1));
if (txnids.size() > 0) {
StringBuffer buf = new StringBuffer("delete from TXNS where txn_id in (");
boolean first = true;
for (long tid : txnids) {
if (first) first = false;
else buf.append(", ");
buf.append(tid);
}
buf.append(")");
LOG.debug("Going to execute update <" + buf.toString() + ">");
int rc = stmt.executeUpdate(buf.toString());
LOG.debug("Removed " + rc + " records from txns");
LOG.debug("Going to commit");
dbConn.commit();
}
} catch (SQLException e) {
LOG.error("Unable to delete from txns table " + e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
checkRetryable(dbConn, e, "cleanEmptyAbortedTxns");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
}
} catch (RetryException e) {
cleanEmptyAbortedTxns();
}
}
/**
* This will take all entries assigned to workers
* on a host return them to INITIATED state. The initiator should use this at start up to
* clean entries from any workers that were in the middle of compacting when the metastore
* shutdown. It does not reset entries from worker threads on other hosts as those may still
* be working.
* @param hostname Name of this host. It is assumed this prefixes the thread's worker id,
* so that like hostname% will match the worker id.
*/
public void revokeFromLocalWorkers(String hostname) throws MetaException {
try {
Connection dbConn = null;
Statement stmt = null;
try {
dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE);
stmt = dbConn.createStatement();
String s = "update COMPACTION_QUEUE set cq_worker_id = null, cq_start = null, cq_state = '"
+ INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_worker_id like '"
+ hostname + "%'";
LOG.debug("Going to execute update <" + s + ">");
// It isn't an error if the following returns no rows, as the local workers could have died
// with nothing assigned to them.
stmt.executeUpdate(s);
LOG.debug("Going to commit");
dbConn.commit();
} catch (SQLException e) {
LOG.error("Unable to change dead worker's records back to initiated state " +
e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
checkRetryable(dbConn, e, "revokeFromLocalWorkers");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
}
} catch (RetryException e) {
revokeFromLocalWorkers(hostname);
}
}
/**
* This call will return all compaction queue
* entries assigned to a worker but over the timeout back to the initiated state.
* This should be called by the initiator on start up and occasionally when running to clean up
* after dead threads. At start up {@link #revokeFromLocalWorkers(String)} should be called
* first.
* @param timeout number of milliseconds since start time that should elapse before a worker is
* declared dead.
*/
public void revokeTimedoutWorkers(long timeout) throws MetaException {
try {
Connection dbConn = null;
Statement stmt = null;
try {
dbConn = getDbConn(Connection.TRANSACTION_SERIALIZABLE);
long latestValidStart = getDbTime(dbConn) - timeout;
stmt = dbConn.createStatement();
String s = "update COMPACTION_QUEUE set cq_worker_id = null, cq_start = null, cq_state = '"
+ INITIATED_STATE+ "' where cq_state = '" + WORKING_STATE + "' and cq_start < "
+ latestValidStart;
LOG.debug("Going to execute update <" + s + ">");
// It isn't an error if the following returns no rows, as the local workers could have died
// with nothing assigned to them.
stmt.executeUpdate(s);
LOG.debug("Going to commit");
dbConn.commit();
} catch (SQLException e) {
LOG.error("Unable to change dead worker's records back to initiated state " +
e.getMessage());
LOG.debug("Going to rollback");
rollbackDBConn(dbConn);
checkRetryable(dbConn, e, "revokeTimedoutWorkers");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
closeDbConn(dbConn);
closeStmt(stmt);
}
} catch (RetryException e) {
revokeTimedoutWorkers(timeout);
}
}
/**
* Queries metastore DB directly to find columns in the table which have statistics information.
* If {@code ci} includes partition info then per partition stats info is examined, otherwise
* table level stats are examined.
* @throws MetaException
*/
public List findColumnsWithStats(CompactionInfo ci) throws MetaException {
Connection dbConn = null;
Statement stmt = null;
ResultSet rs = null;
try {
try {
dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED);
String quote = getIdentifierQuoteString(dbConn);
stmt = dbConn.createStatement();
StringBuilder bldr = new StringBuilder();
bldr.append("SELECT ").append(quote).append("COLUMN_NAME").append(quote)
.append(" FROM ")
.append(quote).append((ci.partName == null ? "TAB_COL_STATS" : "PART_COL_STATS"))
.append(quote)
.append(" WHERE ")
.append(quote).append("DB_NAME").append(quote).append(" = '").append(ci.dbname)
.append("' AND ").append(quote).append("TABLE_NAME").append(quote)
.append(" = '").append(ci.tableName).append("'");
if (ci.partName != null) {
bldr.append(" AND ").append(quote).append("PARTITION_NAME").append(quote).append(" = '")
.append(ci.partName).append("'");
}
String s = bldr.toString();
/*String s = "SELECT COLUMN_NAME FROM " + (ci.partName == null ? "TAB_COL_STATS" :
"PART_COL_STATS")
+ " WHERE DB_NAME='" + ci.dbname + "' AND TABLE_NAME='" + ci.tableName + "'"
+ (ci.partName == null ? "" : " AND PARTITION_NAME='" + ci.partName + "'");*/
LOG.debug("Going to execute <" + s + ">");
rs = stmt.executeQuery(s);
List columns = new ArrayList();
while (rs.next()) {
columns.add(rs.getString(1));
}
LOG.debug("Found columns to update stats: " + columns + " on " + ci.tableName +
(ci.partName == null ? "" : "/" + ci.partName));
dbConn.commit();
return columns;
} catch (SQLException e) {
LOG.error("Failed to find columns to analyze stats on for " + ci.tableName +
(ci.partName == null ? "" : "/" + ci.partName), e);
rollbackDBConn(dbConn);
checkRetryable(dbConn, e, "findColumnsWithStats");
throw new MetaException("Unable to connect to transaction database " +
StringUtils.stringifyException(e));
} finally {
close(rs, stmt, dbConn);
}
} catch (RetryException ex) {
return findColumnsWithStats(ci);
}
}
/**
* Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse} to a
* {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to
* compact the files, and thus treats only open transactions as invalid.
* @param txns txn list from the metastore
* @return a valid txn list.
*/
public static ValidTxnList createValidCompactTxnList(GetOpenTxnsInfoResponse txns) {
long highWater = txns.getTxn_high_water_mark();
long minOpenTxn = Long.MAX_VALUE;
long[] exceptions = new long[txns.getOpen_txnsSize()];
int i = 0;
for (TxnInfo txn : txns.getOpen_txns()) {
if (txn.getState() == TxnState.OPEN) minOpenTxn = Math.min(minOpenTxn, txn.getId());
exceptions[i++] = txn.getId();
}
return new ValidCompactorTxnList(exceptions, minOpenTxn, highWater);
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy