org.apache.hadoop.hbase.backup.impl.BackupAdminImpl Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.backup.impl;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.BackupAdmin;
import org.apache.hadoop.hbase.backup.BackupClientFactory;
import org.apache.hadoop.hbase.backup.BackupInfo;
import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
import org.apache.hadoop.hbase.backup.BackupMergeJob;
import org.apache.hadoop.hbase.backup.BackupRequest;
import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.backup.BackupRestoreFactory;
import org.apache.hadoop.hbase.backup.BackupType;
import org.apache.hadoop.hbase.backup.HBackupFileSystem;
import org.apache.hadoop.hbase.backup.RestoreRequest;
import org.apache.hadoop.hbase.backup.util.BackupSet;
import org.apache.hadoop.hbase.backup.util.BackupUtils;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
@InterfaceAudience.Private
public class BackupAdminImpl implements BackupAdmin {
public final static String CHECK_OK = "Checking backup images: OK";
public final static String CHECK_FAILED =
"Checking backup images: Failed. Some dependencies are missing for restore";
private static final Logger LOG = LoggerFactory.getLogger(BackupAdminImpl.class);
private final Connection conn;
public BackupAdminImpl(Connection conn) {
this.conn = conn;
}
@Override
public void close() {
}
@Override
public BackupInfo getBackupInfo(String backupId) throws IOException {
BackupInfo backupInfo;
try (final BackupSystemTable table = new BackupSystemTable(conn)) {
if (backupId == null) {
ArrayList recentSessions = table.getBackupInfos(BackupState.RUNNING);
if (recentSessions.isEmpty()) {
LOG.warn("No ongoing sessions found.");
return null;
}
// else show status for ongoing session
// must be one maximum
return recentSessions.get(0);
} else {
backupInfo = table.readBackupInfo(backupId);
return backupInfo;
}
}
}
@Override
public int deleteBackups(String[] backupIds) throws IOException {
int totalDeleted = 0;
Map> allTablesMap = new HashMap<>();
boolean deleteSessionStarted;
boolean snapshotDone;
try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
// Step 1: Make sure there is no active session
// is running by using startBackupSession API
// If there is an active session in progress, exception will be thrown
try {
sysTable.startBackupExclusiveOperation();
deleteSessionStarted = true;
} catch (IOException e) {
LOG.warn("You can not run delete command while active backup session is in progress. \n"
+ "If there is no active backup session running, run backup repair utility to "
+ "restore \nbackup system integrity.");
return -1;
}
// Step 2: Make sure there is no failed session
List list = sysTable.getBackupInfos(BackupState.RUNNING);
if (list.size() != 0) {
// ailed sessions found
LOG.warn("Failed backup session found. Run backup repair tool first.");
return -1;
}
// Step 3: Record delete session
sysTable.startDeleteOperation(backupIds);
// Step 4: Snapshot backup system table
if (!BackupSystemTable.snapshotExists(conn)) {
BackupSystemTable.snapshot(conn);
} else {
LOG.warn("Backup system table snapshot exists");
}
snapshotDone = true;
try {
for (int i = 0; i < backupIds.length; i++) {
BackupInfo info = sysTable.readBackupInfo(backupIds[i]);
if (info != null) {
String rootDir = info.getBackupRootDir();
HashSet allTables = allTablesMap.get(rootDir);
if (allTables == null) {
allTables = new HashSet<>();
allTablesMap.put(rootDir, allTables);
}
allTables.addAll(info.getTableNames());
totalDeleted += deleteBackup(backupIds[i], sysTable);
}
}
finalizeDelete(allTablesMap, sysTable);
// Finish
sysTable.finishDeleteOperation();
// delete snapshot
BackupSystemTable.deleteSnapshot(conn);
} catch (IOException e) {
// Fail delete operation
// Step 1
if (snapshotDone) {
if (BackupSystemTable.snapshotExists(conn)) {
BackupSystemTable.restoreFromSnapshot(conn);
// delete snapshot
BackupSystemTable.deleteSnapshot(conn);
// We still have record with unfinished delete operation
LOG.error("Delete operation failed, please run backup repair utility to restore "
+ "backup system integrity", e);
throw e;
} else {
LOG.warn("Delete operation succeeded, there were some errors: ", e);
}
}
} finally {
if (deleteSessionStarted) {
sysTable.finishBackupExclusiveOperation();
}
}
}
return totalDeleted;
}
/**
* Updates incremental backup set for every backupRoot
* @param tablesMap map [backupRoot: {@code Set}]
* @param table backup system table
* @throws IOException if a table operation fails
*/
private void finalizeDelete(Map> tablesMap, BackupSystemTable table)
throws IOException {
for (String backupRoot : tablesMap.keySet()) {
Set incrTableSet = table.getIncrementalBackupTableSet(backupRoot);
Map> tableMap =
table.getBackupHistoryForTableSet(incrTableSet, backupRoot);
for (Map.Entry> entry : tableMap.entrySet()) {
if (entry.getValue() == null) {
// No more backups for a table
incrTableSet.remove(entry.getKey());
}
}
if (!incrTableSet.isEmpty()) {
table.addIncrementalBackupTableSet(incrTableSet, backupRoot);
} else { // empty
table.deleteIncrementalBackupTableSet(backupRoot);
}
}
}
/**
* Delete single backup and all related backups
* Algorithm:
* Backup type: FULL or INCREMENTAL
* Is this last backup session for table T: YES or NO
* For every table T from table list 'tables':
* if(FULL, YES) deletes only physical data (PD)
* if(FULL, NO), deletes PD, scans all newer backups and removes T from backupInfo,
* until we either reach the most recent backup for T in the system or FULL backup
* which includes T
* if(INCREMENTAL, YES) deletes only physical data (PD) if(INCREMENTAL, NO) deletes physical data
* and for table T scans all backup images between last
* FULL backup, which is older than the backup being deleted and the next FULL backup (if exists)
*
* or last one for a particular table T and removes T from list of backup tables.
* @param backupId backup id
* @param sysTable backup system table
* @return total number of deleted backup images
* @throws IOException if deleting the backup fails
*/
private int deleteBackup(String backupId, BackupSystemTable sysTable) throws IOException {
BackupInfo backupInfo = sysTable.readBackupInfo(backupId);
int totalDeleted = 0;
if (backupInfo != null) {
LOG.info("Deleting backup " + backupInfo.getBackupId() + " ...");
// Step 1: clean up data for backup session (idempotent)
BackupUtils.cleanupBackupData(backupInfo, conn.getConfiguration());
// List of tables in this backup;
List tables = backupInfo.getTableNames();
long startTime = backupInfo.getStartTs();
for (TableName tn : tables) {
boolean isLastBackupSession = isLastBackupSession(sysTable, tn, startTime);
if (isLastBackupSession) {
continue;
}
// else
List affectedBackups = getAffectedBackupSessions(backupInfo, tn, sysTable);
for (BackupInfo info : affectedBackups) {
if (info.equals(backupInfo)) {
continue;
}
removeTableFromBackupImage(info, tn, sysTable);
}
}
Map map = sysTable.readBulkLoadedFiles(backupId);
FileSystem fs = FileSystem.get(conn.getConfiguration());
boolean success = true;
int numDeleted = 0;
for (String f : map.values()) {
Path p = new Path(f);
try {
LOG.debug("Delete backup info " + p + " for " + backupInfo.getBackupId());
if (!fs.delete(p)) {
if (fs.exists(p)) {
LOG.warn(f + " was not deleted");
success = false;
}
} else {
numDeleted++;
}
} catch (IOException ioe) {
LOG.warn(f + " was not deleted", ioe);
success = false;
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(numDeleted + " bulk loaded files out of " + map.size() + " were deleted");
}
if (success) {
sysTable.deleteBulkLoadedRows(new ArrayList<>(map.keySet()));
}
sysTable.deleteBackupInfo(backupInfo.getBackupId());
LOG.info("Delete backup " + backupInfo.getBackupId() + " completed.");
totalDeleted++;
} else {
LOG.warn("Delete backup failed: no information found for backupID=" + backupId);
}
return totalDeleted;
}
private void removeTableFromBackupImage(BackupInfo info, TableName tn, BackupSystemTable sysTable)
throws IOException {
List tables = info.getTableNames();
LOG.debug("Remove " + tn + " from " + info.getBackupId() + " tables="
+ info.getTableListAsString());
if (tables.contains(tn)) {
tables.remove(tn);
if (tables.isEmpty()) {
LOG.debug("Delete backup info " + info.getBackupId());
sysTable.deleteBackupInfo(info.getBackupId());
// Idempotent operation
BackupUtils.cleanupBackupData(info, conn.getConfiguration());
} else {
info.setTables(tables);
sysTable.updateBackupInfo(info);
// Now, clean up directory for table (idempotent)
cleanupBackupDir(info, tn, conn.getConfiguration());
}
}
}
private List getAffectedBackupSessions(BackupInfo backupInfo, TableName tn,
BackupSystemTable table) throws IOException {
LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn);
long ts = backupInfo.getStartTs();
List list = new ArrayList<>();
List history = table.getBackupHistory(backupInfo.getBackupRootDir());
// Scan from most recent to backupInfo
// break when backupInfo reached
for (BackupInfo info : history) {
if (info.getStartTs() == ts) {
break;
}
List tables = info.getTableNames();
if (tables.contains(tn)) {
BackupType bt = info.getType();
if (bt == BackupType.FULL) {
// Clear list if we encounter FULL backup
list.clear();
} else {
LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn
+ " added " + info.getBackupId() + " tables=" + info.getTableListAsString());
list.add(info);
}
}
}
return list;
}
/**
* Clean up the data at target directory
* @throws IOException if cleaning up the backup directory fails
*/
private void cleanupBackupDir(BackupInfo backupInfo, TableName table, Configuration conf)
throws IOException {
try {
// clean up the data at target directory
String targetDir = backupInfo.getBackupRootDir();
if (targetDir == null) {
LOG.warn("No target directory specified for " + backupInfo.getBackupId());
return;
}
FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf);
Path targetDirPath =
new Path(BackupUtils.getTableBackupDir(backupInfo.getBackupRootDir(),
backupInfo.getBackupId(), table));
if (outputFs.delete(targetDirPath, true)) {
LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done.");
} else {
LOG.info("No data has been found in " + targetDirPath.toString() + ".");
}
} catch (IOException e1) {
LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " for table " + table
+ "at " + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + ".");
throw e1;
}
}
private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long startTime)
throws IOException {
List history = table.getBackupHistory();
for (BackupInfo info : history) {
List tables = info.getTableNames();
if (!tables.contains(tn)) {
continue;
}
return info.getStartTs() <= startTime;
}
return false;
}
@Override
public List getHistory(int n) throws IOException {
try (final BackupSystemTable table = new BackupSystemTable(conn)) {
List history = table.getBackupHistory();
if (history.size() <= n) {
return history;
}
List list = new ArrayList<>();
for (int i = 0; i < n; i++) {
list.add(history.get(i));
}
return list;
}
}
@Override
public List getHistory(int n, BackupInfo.Filter... filters) throws IOException {
if (filters.length == 0) {
return getHistory(n);
}
try (final BackupSystemTable table = new BackupSystemTable(conn)) {
List history = table.getBackupHistory();
List result = new ArrayList<>();
for (BackupInfo bi : history) {
if (result.size() == n) {
break;
}
boolean passed = true;
for (int i = 0; i < filters.length; i++) {
if (!filters[i].apply(bi)) {
passed = false;
break;
}
}
if (passed) {
result.add(bi);
}
}
return result;
}
}
@Override
public List listBackupSets() throws IOException {
try (final BackupSystemTable table = new BackupSystemTable(conn)) {
List list = table.listBackupSets();
List bslist = new ArrayList<>();
for (String s : list) {
List tables = table.describeBackupSet(s);
if (tables != null) {
bslist.add(new BackupSet(s, tables));
}
}
return bslist;
}
}
@Override
public BackupSet getBackupSet(String name) throws IOException {
try (final BackupSystemTable table = new BackupSystemTable(conn)) {
List list = table.describeBackupSet(name);
if (list == null) {
return null;
}
return new BackupSet(name, list);
}
}
@Override
public boolean deleteBackupSet(String name) throws IOException {
try (final BackupSystemTable table = new BackupSystemTable(conn)) {
if (table.describeBackupSet(name) == null) {
return false;
}
table.deleteBackupSet(name);
return true;
}
}
@Override
public void addToBackupSet(String name, TableName[] tables) throws IOException {
String[] tableNames = new String[tables.length];
try (final BackupSystemTable table = new BackupSystemTable(conn);
final Admin admin = conn.getAdmin()) {
for (int i = 0; i < tables.length; i++) {
tableNames[i] = tables[i].getNameAsString();
if (!admin.tableExists(TableName.valueOf(tableNames[i]))) {
throw new IOException("Cannot add " + tableNames[i] + " because it doesn't exist");
}
}
table.addToBackupSet(name, tableNames);
LOG.info("Added tables [" + StringUtils.join(tableNames, " ") + "] to '" + name
+ "' backup set");
}
}
@Override
public void removeFromBackupSet(String name, TableName[] tables) throws IOException {
LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name + "'");
try (final BackupSystemTable table = new BackupSystemTable(conn)) {
table.removeFromBackupSet(name, toStringArray(tables));
LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name
+ "' completed.");
}
}
private String[] toStringArray(TableName[] list) {
String[] arr = new String[list.length];
for (int i = 0; i < list.length; i++) {
arr[i] = list[i].toString();
}
return arr;
}
@Override
public void restore(RestoreRequest request) throws IOException {
if (request.isCheck()) {
HashMap backupManifestMap = new HashMap<>();
// check and load backup image manifest for the tables
Path rootPath = new Path(request.getBackupRootDir());
String backupId = request.getBackupId();
TableName[] sTableArray = request.getFromTables();
HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray,
conn.getConfiguration(), rootPath, backupId);
// Check and validate the backup image and its dependencies
if (BackupUtils.validate(backupManifestMap, conn.getConfiguration())) {
LOG.info(CHECK_OK);
} else {
LOG.error(CHECK_FAILED);
}
return;
}
// Execute restore request
new RestoreTablesClient(conn, request).execute();
}
@Override
public String backupTables(BackupRequest request) throws IOException {
BackupType type = request.getBackupType();
String targetRootDir = request.getTargetRootDir();
List tableList = request.getTableList();
String backupId = BackupRestoreConstants.BACKUPID_PREFIX + EnvironmentEdgeManager.currentTime();
if (type == BackupType.INCREMENTAL) {
Set incrTableSet;
try (BackupSystemTable table = new BackupSystemTable(conn)) {
incrTableSet = table.getIncrementalBackupTableSet(targetRootDir);
}
if (incrTableSet.isEmpty()) {
String msg = "Incremental backup table set contains no tables. "
+ "You need to run full backup first "
+ (tableList != null ? "on " + StringUtils.join(tableList, ",") : "");
throw new IOException(msg);
}
if (tableList != null) {
tableList.removeAll(incrTableSet);
if (!tableList.isEmpty()) {
String extraTables = StringUtils.join(tableList, ",");
String msg = "Some tables (" + extraTables + ") haven't gone through full backup. "
+ "Perform full backup on " + extraTables + " first, " + "then retry the command";
throw new IOException(msg);
}
}
tableList = Lists.newArrayList(incrTableSet);
}
if (tableList != null && !tableList.isEmpty()) {
for (TableName table : tableList) {
String targetTableBackupDir =
HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table);
Path targetTableBackupDirPath = new Path(targetTableBackupDir);
FileSystem outputFs =
FileSystem.get(targetTableBackupDirPath.toUri(), conn.getConfiguration());
if (outputFs.exists(targetTableBackupDirPath)) {
throw new IOException("Target backup directory " + targetTableBackupDir
+ " exists already.");
}
outputFs.mkdirs(targetTableBackupDirPath);
}
ArrayList nonExistingTableList = null;
try (Admin admin = conn.getAdmin()) {
for (TableName tableName : tableList) {
if (!admin.tableExists(tableName)) {
if (nonExistingTableList == null) {
nonExistingTableList = new ArrayList<>();
}
nonExistingTableList.add(tableName);
}
}
}
if (nonExistingTableList != null) {
if (type == BackupType.INCREMENTAL) {
// Update incremental backup set
tableList = excludeNonExistingTables(tableList, nonExistingTableList);
} else {
// Throw exception only in full mode - we try to backup non-existing table
throw new IOException("Non-existing tables found in the table list: "
+ nonExistingTableList);
}
}
}
// update table list
BackupRequest.Builder builder = new BackupRequest.Builder();
request = builder.withBackupType(request.getBackupType()).withTableList(tableList)
.withTargetRootDir(request.getTargetRootDir())
.withBackupSetName(request.getBackupSetName()).withTotalTasks(request.getTotalTasks())
.withBandwidthPerTasks((int) request.getBandwidth()).build();
TableBackupClient client;
try {
client = BackupClientFactory.create(conn, backupId, request);
} catch (IOException e) {
LOG.error("There is an active session already running");
throw e;
}
client.execute();
return backupId;
}
private List excludeNonExistingTables(List tableList,
List nonExistingTableList) {
for (TableName table : nonExistingTableList) {
tableList.remove(table);
}
return tableList;
}
@Override
public void mergeBackups(String[] backupIds) throws IOException {
try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
checkIfValidForMerge(backupIds, sysTable);
//TODO run job on remote cluster
BackupMergeJob job = BackupRestoreFactory.getBackupMergeJob(conn.getConfiguration());
job.run(backupIds);
}
}
/**
* Verifies that backup images are valid for merge.
*
*
* - All backups MUST be in the same destination
*
- No FULL backups are allowed - only INCREMENTAL
*
- All backups must be in COMPLETE state
*
- No holes in backup list are allowed
*
*
* @param backupIds list of backup ids
* @param table backup system table
* @throws IOException if the backup image is not valid for merge
*/
private void checkIfValidForMerge(String[] backupIds, BackupSystemTable table)
throws IOException {
String backupRoot = null;
final Set allTables = new HashSet<>();
final Set allBackups = new HashSet<>();
long minTime = Long.MAX_VALUE, maxTime = Long.MIN_VALUE;
for (String backupId : backupIds) {
BackupInfo bInfo = table.readBackupInfo(backupId);
if (bInfo == null) {
String msg = "Backup session " + backupId + " not found";
throw new IOException(msg);
}
if (backupRoot == null) {
backupRoot = bInfo.getBackupRootDir();
} else if (!bInfo.getBackupRootDir().equals(backupRoot)) {
throw new IOException("Found different backup destinations in a list of a backup sessions "
+ "\n1. " + backupRoot + "\n" + "2. " + bInfo.getBackupRootDir());
}
if (bInfo.getType() == BackupType.FULL) {
throw new IOException("FULL backup image can not be merged for: \n" + bInfo);
}
if (bInfo.getState() != BackupState.COMPLETE) {
throw new IOException("Backup image " + backupId
+ " can not be merged becuase of its state: " + bInfo.getState());
}
allBackups.add(backupId);
allTables.addAll(bInfo.getTableNames());
long time = bInfo.getStartTs();
if (time < minTime) {
minTime = time;
}
if (time > maxTime) {
maxTime = time;
}
}
final long startRangeTime = minTime;
final long endRangeTime = maxTime;
final String backupDest = backupRoot;
// Check we have no 'holes' in backup id list
// Filter 1 : backupRoot
// Filter 2 : time range filter
// Filter 3 : table filter
BackupInfo.Filter destinationFilter = info -> info.getBackupRootDir().equals(backupDest);
BackupInfo.Filter timeRangeFilter = info -> {
long time = info.getStartTs();
return time >= startRangeTime && time <= endRangeTime ;
};
BackupInfo.Filter tableFilter = info -> {
List tables = info.getTableNames();
return !Collections.disjoint(allTables, tables);
};
BackupInfo.Filter typeFilter = info -> info.getType() == BackupType.INCREMENTAL;
BackupInfo.Filter stateFilter = info -> info.getState() == BackupState.COMPLETE;
List allInfos = table.getBackupHistory(-1, destinationFilter,
timeRangeFilter, tableFilter, typeFilter, stateFilter);
if (allInfos.size() != allBackups.size()) {
// Yes we have at least one hole in backup image sequence
List missingIds = new ArrayList<>();
for(BackupInfo info: allInfos) {
if(allBackups.contains(info.getBackupId())) {
continue;
}
missingIds.add(info.getBackupId());
}
String errMsg =
"Sequence of backup ids has 'holes'. The following backup images must be added:" +
org.apache.hadoop.util.StringUtils.join(",", missingIds);
throw new IOException(errMsg);
}
}
}
© 2015 - 2025 Weber Informatics LLC | Privacy Policy