org.apache.hadoop.hive.metastore.txn.TxnDbUtil Maven / Gradle / Ivy
The newest version!
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hive.metastore.txn;
import java.sql.Connection;
import java.sql.Driver;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.SQLTransactionRollbackException;
import java.sql.Statement;
import java.util.Properties;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.shims.ShimLoader;
/**
* Utility methods for creating and destroying txn database/schema.
* Placed here in a separate class so it can be shared across unit tests.
*/
public final class TxnDbUtil {
static final private Log LOG = LogFactory.getLog(TxnDbUtil.class.getName());
private static final String TXN_MANAGER = "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager";
private static int deadlockCnt = 0;
private TxnDbUtil() {
throw new UnsupportedOperationException("Can't initialize class");
}
/**
* Set up the configuration so it will use the DbTxnManager, concurrency will be set to true,
* and the JDBC configs will be set for putting the transaction and lock info in the embedded
* metastore.
*
* @param conf HiveConf to add these values to
*/
public static void setConfValues(HiveConf conf) {
conf.setVar(HiveConf.ConfVars.HIVE_TXN_MANAGER, TXN_MANAGER);
conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, true);
}
public static void prepDb() throws Exception {
// This is a bogus hack because it copies the contents of the SQL file
// intended for creating derby databases, and thus will inexorably get
// out of date with it. I'm open to any suggestions on how to make this
// read the file in a build friendly way.
Connection conn = null;
Statement stmt = null;
try {
conn = getConnection();
stmt = conn.createStatement();
stmt.execute("CREATE TABLE TXNS (" +
" TXN_ID bigint PRIMARY KEY," +
" TXN_STATE char(1) NOT NULL," +
" TXN_STARTED bigint NOT NULL," +
" TXN_LAST_HEARTBEAT bigint NOT NULL," +
" TXN_USER varchar(128) NOT NULL," +
" TXN_HOST varchar(128) NOT NULL)");
stmt.execute("CREATE TABLE TXN_COMPONENTS (" +
" TC_TXNID bigint REFERENCES TXNS (TXN_ID)," +
" TC_DATABASE varchar(128) NOT NULL," +
" TC_TABLE varchar(128)," +
" TC_PARTITION varchar(767))");
stmt.execute("CREATE TABLE COMPLETED_TXN_COMPONENTS (" +
" CTC_TXNID bigint," +
" CTC_DATABASE varchar(128) NOT NULL," +
" CTC_TABLE varchar(128)," +
" CTC_PARTITION varchar(767))");
stmt.execute("CREATE TABLE NEXT_TXN_ID (" + " NTXN_NEXT bigint NOT NULL)");
stmt.execute("INSERT INTO NEXT_TXN_ID VALUES(1)");
stmt.execute("CREATE TABLE HIVE_LOCKS (" +
" HL_LOCK_EXT_ID bigint NOT NULL," +
" HL_LOCK_INT_ID bigint NOT NULL," +
" HL_TXNID bigint," +
" HL_DB varchar(128) NOT NULL," +
" HL_TABLE varchar(128)," +
" HL_PARTITION varchar(767)," +
" HL_LOCK_STATE char(1) NOT NULL," +
" HL_LOCK_TYPE char(1) NOT NULL," +
" HL_LAST_HEARTBEAT bigint NOT NULL," +
" HL_ACQUIRED_AT bigint," +
" HL_USER varchar(128) NOT NULL," +
" HL_HOST varchar(128) NOT NULL," +
" PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID))");
stmt.execute("CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID)");
stmt.execute("CREATE TABLE NEXT_LOCK_ID (" + " NL_NEXT bigint NOT NULL)");
stmt.execute("INSERT INTO NEXT_LOCK_ID VALUES(1)");
stmt.execute("CREATE TABLE COMPACTION_QUEUE (" +
" CQ_ID bigint PRIMARY KEY," +
" CQ_DATABASE varchar(128) NOT NULL," +
" CQ_TABLE varchar(128) NOT NULL," +
" CQ_PARTITION varchar(767)," +
" CQ_STATE char(1) NOT NULL," +
" CQ_TYPE char(1) NOT NULL," +
" CQ_WORKER_ID varchar(128)," +
" CQ_START bigint," +
" CQ_RUN_AS varchar(128))");
stmt.execute("CREATE TABLE NEXT_COMPACTION_QUEUE_ID (NCQ_NEXT bigint NOT NULL)");
stmt.execute("INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1)");
conn.commit();
} catch (SQLException e) {
// This might be a deadlock, if so, let's retry
conn.rollback();
if (e instanceof SQLTransactionRollbackException && deadlockCnt++ < 5) {
LOG.warn("Caught deadlock, retrying db creation");
prepDb();
} else {
throw e;
}
} finally {
deadlockCnt = 0;
closeResources(conn, stmt, null);
}
}
public static void cleanDb() throws Exception {
Connection conn = null;
Statement stmt = null;
try {
conn = getConnection();
stmt = conn.createStatement();
// We want to try these, whether they succeed or fail.
try {
stmt.execute("DROP INDEX HL_TXNID_INDEX");
} catch (Exception e) {
System.err.println("Unable to drop index HL_TXNID_INDEX " + e.getMessage());
}
dropTable(stmt, "TXN_COMPONENTS");
dropTable(stmt, "COMPLETED_TXN_COMPONENTS");
dropTable(stmt, "TXNS");
dropTable(stmt, "NEXT_TXN_ID");
dropTable(stmt, "HIVE_LOCKS");
dropTable(stmt, "NEXT_LOCK_ID");
dropTable(stmt, "COMPACTION_QUEUE");
dropTable(stmt, "NEXT_COMPACTION_QUEUE_ID");
conn.commit();
} finally {
closeResources(conn, stmt, null);
}
}
private static void dropTable(Statement stmt, String name) {
try {
stmt.execute("DROP TABLE " + name);
} catch (Exception e) {
System.err.println("Unable to drop table " + name + ": " + e.getMessage());
}
}
/**
* A tool to count the number of partitions, tables,
* and databases locked by a particular lockId.
*
* @param lockId lock id to look for lock components
*
* @return number of components, or 0 if there is no lock
*/
public static int countLockComponents(long lockId) throws Exception {
Connection conn = null;
PreparedStatement stmt = null;
ResultSet rs = null;
try {
conn = getConnection();
stmt = conn.prepareStatement("SELECT count(*) FROM hive_locks WHERE hl_lock_ext_id = ?");
stmt.setLong(1, lockId);
rs = stmt.executeQuery();
if (!rs.next()) {
return 0;
}
return rs.getInt(1);
} finally {
closeResources(conn, stmt, rs);
}
}
public static int findNumCurrentLocks() throws Exception {
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
try {
conn = getConnection();
stmt = conn.createStatement();
rs = stmt.executeQuery("select count(*) from hive_locks");
if (!rs.next()) {
return 0;
}
return rs.getInt(1);
} finally {
closeResources(conn, stmt, rs);
}
}
private static Connection getConnection() throws Exception {
HiveConf conf = new HiveConf();
String jdbcDriver = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER);
Driver driver = (Driver) Class.forName(jdbcDriver).newInstance();
Properties prop = new Properties();
String driverUrl = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORECONNECTURLKEY);
String user = HiveConf.getVar(conf, HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME);
String passwd =
ShimLoader.getHadoopShims().getPassword(conf, HiveConf.ConfVars.METASTOREPWD.varname);
prop.setProperty("user", user);
prop.setProperty("password", passwd);
return driver.connect(driverUrl, prop);
}
private static void closeResources(Connection conn, Statement stmt, ResultSet rs) {
if (rs != null) {
try {
rs.close();
} catch (SQLException e) {
System.err.println("Error closing ResultSet: " + e.getMessage());
}
}
if (stmt != null) {
try {
stmt.close();
} catch (SQLException e) {
System.err.println("Error closing Statement: " + e.getMessage());
}
}
if (conn != null) {
try {
conn.rollback();
} catch (SQLException e) {
System.err.println("Error rolling back: " + e.getMessage());
}
try {
conn.close();
} catch (SQLException e) {
System.err.println("Error closing Connection: " + e.getMessage());
}
}
}
}