Many resources are needed to download a project. Please understand that we have to compensate our server costs. Thank you in advance. Project price only 1 $
You can buy this project and download/modify it how often you want.
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.snapshot;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorCompletionService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.errorhandling.ForeignExceptionSnare;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.monitoring.MonitoredTask;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
import org.apache.hadoop.hbase.regionserver.HStore;
import org.apache.hadoop.hbase.regionserver.HStoreFile;
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.hbase.util.FSTableDescriptors;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hbase.thirdparty.com.google.protobuf.CodedInputStream;
import org.apache.hbase.thirdparty.com.google.protobuf.InvalidProtocolBufferException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDataManifest;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotDescription;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
/**
* Utility class to help read/write the Snapshot Manifest.
*
* The snapshot format is transparent for the users of this class,
* once the snapshot is written, it will never be modified.
* On open() the snapshot will be loaded to the current in-memory format.
*/
@InterfaceAudience.Private
public final class SnapshotManifest {
private static final Logger LOG = LoggerFactory.getLogger(SnapshotManifest.class);
public static final String SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY = "snapshot.manifest.size.limit";
public static final String DATA_MANIFEST_NAME = "data.manifest";
private List regionManifests;
private SnapshotDescription desc;
private TableDescriptor htd;
private final ForeignExceptionSnare monitor;
private final Configuration conf;
private final Path workingDir;
private final FileSystem rootFs;
private final FileSystem workingDirFs;
private int manifestSizeLimit;
private final MonitoredTask statusTask;
/**
*
* @param conf configuration file for HBase setup
* @param rootFs root filesystem containing HFiles
* @param workingDir file path of where the manifest should be located
* @param desc description of snapshot being taken
* @param monitor monitor of foreign exceptions
* @throws IOException if the working directory file system cannot be
* determined from the config file
*/
private SnapshotManifest(final Configuration conf, final FileSystem rootFs,
final Path workingDir, final SnapshotDescription desc,
final ForeignExceptionSnare monitor, final MonitoredTask statusTask) throws IOException {
this.monitor = monitor;
this.desc = desc;
this.workingDir = workingDir;
this.conf = conf;
this.rootFs = rootFs;
this.statusTask = statusTask;
this.workingDirFs = this.workingDir.getFileSystem(this.conf);
this.manifestSizeLimit = conf.getInt(SNAPSHOT_MANIFEST_SIZE_LIMIT_CONF_KEY, 64 * 1024 * 1024);
}
/**
* Return a SnapshotManifest instance, used for writing a snapshot.
*
* There are two usage pattern:
* - The Master will create a manifest, add the descriptor, offline regions
* and consolidate the snapshot by writing all the pending stuff on-disk.
* manifest = SnapshotManifest.create(...)
* manifest.addRegion(tableDir, hri)
* manifest.consolidate()
* - The RegionServer will create a single region manifest
* manifest = SnapshotManifest.create(...)
* manifest.addRegion(region)
*/
public static SnapshotManifest create(final Configuration conf, final FileSystem fs,
final Path workingDir, final SnapshotDescription desc,
final ForeignExceptionSnare monitor) throws IOException {
return create(conf, fs, workingDir, desc, monitor, null);
}
public static SnapshotManifest create(final Configuration conf, final FileSystem fs,
final Path workingDir, final SnapshotDescription desc, final ForeignExceptionSnare monitor,
final MonitoredTask statusTask) throws IOException {
return new SnapshotManifest(conf, fs, workingDir, desc, monitor, statusTask);
}
/**
* Return a SnapshotManifest instance with the information already loaded in-memory.
* SnapshotManifest manifest = SnapshotManifest.open(...)
* TableDescriptor htd = manifest.getTableDescriptor()
* for (SnapshotRegionManifest regionManifest: manifest.getRegionManifests())
* hri = regionManifest.getRegionInfo()
* for (regionManifest.getFamilyFiles())
* ...
*/
public static SnapshotManifest open(final Configuration conf, final FileSystem fs,
final Path workingDir, final SnapshotDescription desc) throws IOException {
SnapshotManifest manifest = new SnapshotManifest(conf, fs, workingDir, desc, null, null);
manifest.load();
return manifest;
}
/**
* Add the table descriptor to the snapshot manifest
*/
public void addTableDescriptor(final TableDescriptor htd) throws IOException {
this.htd = htd;
}
interface RegionVisitor {
TRegion regionOpen(final RegionInfo regionInfo) throws IOException;
void regionClose(final TRegion region) throws IOException;
TFamily familyOpen(final TRegion region, final byte[] familyName) throws IOException;
void familyClose(final TRegion region, final TFamily family) throws IOException;
void storeFile(final TRegion region, final TFamily family, final StoreFileInfo storeFile)
throws IOException;
}
private RegionVisitor createRegionVisitor(final SnapshotDescription desc) throws IOException {
switch (getSnapshotFormat(desc)) {
case SnapshotManifestV1.DESCRIPTOR_VERSION:
return new SnapshotManifestV1.ManifestBuilder(conf, rootFs, workingDir);
case SnapshotManifestV2.DESCRIPTOR_VERSION:
return new SnapshotManifestV2.ManifestBuilder(conf, rootFs, workingDir);
default:
throw new CorruptedSnapshotException("Invalid Snapshot version: " + desc.getVersion(),
ProtobufUtil.createSnapshotDesc(desc));
}
}
public void addMobRegion(RegionInfo regionInfo) throws IOException {
// Get the ManifestBuilder/RegionVisitor
RegionVisitor visitor = createRegionVisitor(desc);
// Visit the region and add it to the manifest
addMobRegion(regionInfo, visitor);
}
protected void addMobRegion(RegionInfo regionInfo, RegionVisitor visitor) throws IOException {
// 1. dump region meta info into the snapshot directory
final String snapshotName = desc.getName();
LOG.debug("Storing mob region '" + regionInfo + "' region-info for snapshot=" + snapshotName);
Object regionData = visitor.regionOpen(regionInfo);
monitor.rethrowException();
// 2. iterate through all the stores in the region
LOG.debug("Creating references for mob files");
Path mobRegionPath = MobUtils.getMobRegionPath(conf, regionInfo.getTable());
for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
// 2.1. build the snapshot reference for the store if it's a mob store
if (!hcd.isMobEnabled()) {
continue;
}
Object familyData = visitor.familyOpen(regionData, hcd.getName());
monitor.rethrowException();
Path storePath = MobUtils.getMobFamilyPath(mobRegionPath, hcd.getNameAsString());
List storeFiles = getStoreFiles(storePath);
if (storeFiles == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("No mob files under family: " + hcd.getNameAsString());
}
continue;
}
addReferenceFiles(visitor, regionData, familyData, storeFiles, true);
visitor.familyClose(regionData, familyData);
}
visitor.regionClose(regionData);
}
/**
* Creates a 'manifest' for the specified region, by reading directly from the HRegion object.
* This is used by the "online snapshot" when the table is enabled.
*/
public void addRegion(final HRegion region) throws IOException {
// Get the ManifestBuilder/RegionVisitor
RegionVisitor visitor = createRegionVisitor(desc);
// Visit the region and add it to the manifest
addRegion(region, visitor);
}
protected void addRegion(final HRegion region, RegionVisitor visitor) throws IOException {
// 1. dump region meta info into the snapshot directory
final String snapshotName = desc.getName();
LOG.debug("Storing '" + region + "' region-info for snapshot=" + snapshotName);
Object regionData = visitor.regionOpen(region.getRegionInfo());
monitor.rethrowException();
// 2. iterate through all the stores in the region
LOG.debug("Creating references for hfiles");
for (HStore store : region.getStores()) {
// 2.1. build the snapshot reference for the store
Object familyData = visitor.familyOpen(regionData,
store.getColumnFamilyDescriptor().getName());
monitor.rethrowException();
List storeFiles = new ArrayList<>(store.getStorefiles());
if (LOG.isDebugEnabled()) {
LOG.debug("Adding snapshot references for " + storeFiles + " hfiles");
}
// 2.2. iterate through all the store's files and create "references".
for (int i = 0, sz = storeFiles.size(); i < sz; i++) {
HStoreFile storeFile = storeFiles.get(i);
monitor.rethrowException();
// create "reference" to this store file.
LOG.debug("Adding reference for file (" + (i+1) + "/" + sz + "): " + storeFile.getPath() +
" for snapshot=" + snapshotName);
visitor.storeFile(regionData, familyData, storeFile.getFileInfo());
}
visitor.familyClose(regionData, familyData);
}
visitor.regionClose(regionData);
}
/**
* Creates a 'manifest' for the specified region, by reading directly from the disk.
* This is used by the "offline snapshot" when the table is disabled.
*/
public void addRegion(final Path tableDir, final RegionInfo regionInfo) throws IOException {
// Get the ManifestBuilder/RegionVisitor
RegionVisitor visitor = createRegionVisitor(desc);
// Visit the region and add it to the manifest
addRegion(tableDir, regionInfo, visitor);
}
protected void addRegion(final Path tableDir, final RegionInfo regionInfo, RegionVisitor visitor)
throws IOException {
boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
try {
Path baseDir = tableDir;
// Open the RegionFS
if (isMobRegion) {
baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable());
}
HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs,
baseDir, regionInfo, true);
monitor.rethrowException();
// 1. dump region meta info into the snapshot directory
LOG.debug("Storing region-info for snapshot.");
Object regionData = visitor.regionOpen(regionInfo);
monitor.rethrowException();
// 2. iterate through all the stores in the region
LOG.debug("Creating references for hfiles");
// This ensures that we have an atomic view of the directory as long as we have < ls limit
// (batch size of the files in a directory) on the namenode. Otherwise, we get back the files
// in batches and may miss files being added/deleted. This could be more robust (iteratively
// checking to see if we have all the files until we are sure), but the limit is currently
// 1000 files/batch, far more than the number of store files under a single column family.
Collection familyNames = regionFs.getFamilies();
if (familyNames != null) {
for (String familyName: familyNames) {
Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
monitor.rethrowException();
Collection storeFiles = regionFs.getStoreFiles(familyName);
if (storeFiles == null) {
if (LOG.isDebugEnabled()) {
LOG.debug("No files under family: " + familyName);
}
continue;
}
// 2.1. build the snapshot reference for the store
// iterate through all the store's files and create "references".
addReferenceFiles(visitor, regionData, familyData, storeFiles, false);
visitor.familyClose(regionData, familyData);
}
}
visitor.regionClose(regionData);
} catch (IOException e) {
// the mob directory might not be created yet, so do nothing when it is a mob region
if (!isMobRegion) {
throw e;
}
}
}
private List getStoreFiles(Path storeDir) throws IOException {
FileStatus[] stats = CommonFSUtils.listStatus(rootFs, storeDir);
if (stats == null) return null;
ArrayList storeFiles = new ArrayList<>(stats.length);
for (int i = 0; i < stats.length; ++i) {
storeFiles.add(new StoreFileInfo(conf, rootFs, stats[i]));
}
return storeFiles;
}
private void addReferenceFiles(RegionVisitor visitor, Object regionData, Object familyData,
Collection storeFiles, boolean isMob) throws IOException {
final String fileType = isMob ? "mob file" : "hfile";
if (LOG.isDebugEnabled()) {
LOG.debug(String.format("Adding snapshot references for %s %ss", storeFiles, fileType));
}
int i = 0;
int sz = storeFiles.size();
for (StoreFileInfo storeFile: storeFiles) {
monitor.rethrowException();
LOG.debug(String.format("Adding reference for %s (%d/%d): %s",
fileType, ++i, sz, storeFile.getPath()));
// create "reference" to this store file.
visitor.storeFile(regionData, familyData, storeFile);
}
}
/**
* Load the information in the SnapshotManifest. Called by SnapshotManifest.open()
*
* If the format is v2 and there is no data-manifest, means that we are loading an
* in-progress snapshot. Since we support rolling-upgrades, we loook for v1 and v2
* regions format.
*/
private void load() throws IOException {
switch (getSnapshotFormat(desc)) {
case SnapshotManifestV1.DESCRIPTOR_VERSION: {
this.htd = FSTableDescriptors.getTableDescriptorFromFs(workingDirFs, workingDir);
ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
try {
this.regionManifests =
SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs, workingDir, desc);
} finally {
tpool.shutdown();
}
break;
}
case SnapshotManifestV2.DESCRIPTOR_VERSION: {
SnapshotDataManifest dataManifest = readDataManifest();
if (dataManifest != null) {
htd = ProtobufUtil.toTableDescriptor(dataManifest.getTableSchema());
regionManifests = dataManifest.getRegionManifestsList();
} else {
// Compatibility, load the v1 regions
// This happens only when the snapshot is in-progress and the cache wants to refresh.
List v1Regions, v2Regions;
ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
try {
v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, rootFs,
workingDir, desc);
v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, rootFs,
workingDir, desc, manifestSizeLimit);
} catch (InvalidProtocolBufferException e) {
throw new CorruptedSnapshotException("unable to parse region manifest " +
e.getMessage(), e);
} finally {
tpool.shutdown();
}
if (v1Regions != null && v2Regions != null) {
regionManifests = new ArrayList<>(v1Regions.size() + v2Regions.size());
regionManifests.addAll(v1Regions);
regionManifests.addAll(v2Regions);
} else if (v1Regions != null) {
regionManifests = v1Regions;
} else /* if (v2Regions != null) */ {
regionManifests = v2Regions;
}
}
break;
}
default:
throw new CorruptedSnapshotException("Invalid Snapshot version: " + desc.getVersion(),
ProtobufUtil.createSnapshotDesc(desc));
}
}
/**
* Get the current snapshot working dir
*/
public Path getSnapshotDir() {
return this.workingDir;
}
/**
* Get the SnapshotDescription
*/
public SnapshotDescription getSnapshotDescription() {
return this.desc;
}
/**
* Get the table descriptor from the Snapshot
*/
public TableDescriptor getTableDescriptor() {
return this.htd;
}
/**
* Get all the Region Manifest from the snapshot
*/
public List getRegionManifests() {
return this.regionManifests;
}
private void setStatusMsg(String msg) {
if (this.statusTask != null) {
statusTask.setStatus(msg);
}
}
/**
* Get all the Region Manifest from the snapshot.
* This is an helper to get a map with the region encoded name
*/
public Map getRegionManifestsMap() {
if (regionManifests == null || regionManifests.isEmpty()) return null;
HashMap regionsMap = new HashMap<>(regionManifests.size());
for (SnapshotRegionManifest manifest: regionManifests) {
String regionName = getRegionNameFromManifest(manifest);
regionsMap.put(regionName, manifest);
}
return regionsMap;
}
public void consolidate() throws IOException {
if (getSnapshotFormat(desc) == SnapshotManifestV1.DESCRIPTOR_VERSION) {
LOG.info("Using old Snapshot Format");
// write a copy of descriptor to the snapshot directory
FSTableDescriptors.createTableDescriptorForTableDirectory(workingDirFs, workingDir, htd,
false);
} else {
LOG.debug("Convert to Single Snapshot Manifest for {}", this.desc.getName());
convertToV2SingleManifest();
}
}
/*
* In case of rolling-upgrade, we try to read all the formats and build
* the snapshot with the latest format.
*/
private void convertToV2SingleManifest() throws IOException {
// Try to load v1 and v2 regions
List v1Regions, v2Regions;
ThreadPoolExecutor tpool = createExecutor("SnapshotManifestLoader");
setStatusMsg("Loading Region manifests for " + this.desc.getName());
try {
v1Regions = SnapshotManifestV1.loadRegionManifests(conf, tpool, workingDirFs,
workingDir, desc);
v2Regions = SnapshotManifestV2.loadRegionManifests(conf, tpool, workingDirFs,
workingDir, desc, manifestSizeLimit);
SnapshotDataManifest.Builder dataManifestBuilder = SnapshotDataManifest.newBuilder();
dataManifestBuilder.setTableSchema(ProtobufUtil.toTableSchema(htd));
if (v1Regions != null && v1Regions.size() > 0) {
dataManifestBuilder.addAllRegionManifests(v1Regions);
}
if (v2Regions != null && v2Regions.size() > 0) {
dataManifestBuilder.addAllRegionManifests(v2Regions);
}
// Write the v2 Data Manifest.
// Once the data-manifest is written, the snapshot can be considered complete.
// Currently snapshots are written in a "temporary" directory and later
// moved to the "complated" snapshot directory.
setStatusMsg("Writing data manifest for " + this.desc.getName());
SnapshotDataManifest dataManifest = dataManifestBuilder.build();
writeDataManifest(dataManifest);
this.regionManifests = dataManifest.getRegionManifestsList();
// Remove the region manifests. Everything is now in the data-manifest.
// The delete operation is "relaxed", unless we get an exception we keep going.
// The extra files in the snapshot directory will not give any problem,
// since they have the same content as the data manifest, and even by re-reading
// them we will get the same information.
int totalDeletes = 0;
ExecutorCompletionService completionService = new ExecutorCompletionService<>(tpool);
if (v1Regions != null) {
for (SnapshotRegionManifest regionManifest: v1Regions) {
++totalDeletes;
completionService.submit(() -> {
SnapshotManifestV1.deleteRegionManifest(workingDirFs, workingDir, regionManifest);
return null;
});
}
}
if (v2Regions != null) {
for (SnapshotRegionManifest regionManifest: v2Regions) {
++totalDeletes;
completionService.submit(() -> {
SnapshotManifestV2.deleteRegionManifest(workingDirFs, workingDir, regionManifest);
return null;
});
}
}
// Wait for the deletes to finish.
for (int i = 0; i < totalDeletes; i++) {
try {
completionService.take().get();
} catch (InterruptedException ie) {
throw new InterruptedIOException(ie.getMessage());
} catch (ExecutionException e) {
throw new IOException("Error deleting region manifests", e.getCause());
}
}
} finally {
tpool.shutdown();
}
}
/*
* Write the SnapshotDataManifest file
*/
private void writeDataManifest(final SnapshotDataManifest manifest)
throws IOException {
try (FSDataOutputStream stream = workingDirFs.create(new Path(workingDir, DATA_MANIFEST_NAME))) {
manifest.writeTo(stream);
}
}
/*
* Read the SnapshotDataManifest file
*/
private SnapshotDataManifest readDataManifest() throws IOException {
try (FSDataInputStream in = workingDirFs.open(new Path(workingDir, DATA_MANIFEST_NAME))) {
CodedInputStream cin = CodedInputStream.newInstance(in);
cin.setSizeLimit(manifestSizeLimit);
return SnapshotDataManifest.parseFrom(cin);
} catch (FileNotFoundException e) {
return null;
} catch (InvalidProtocolBufferException e) {
throw new CorruptedSnapshotException("unable to parse data manifest " + e.getMessage(), e);
}
}
private ThreadPoolExecutor createExecutor(final String name) {
return createExecutor(conf, name);
}
public static ThreadPoolExecutor createExecutor(final Configuration conf, final String name) {
int maxThreads = conf.getInt("hbase.snapshot.thread.pool.max", 8);
return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
new ThreadFactoryBuilder().setNameFormat(name + "-pool-%d").setDaemon(true)
.setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
}
/**
* Extract the region encoded name from the region manifest
*/
static String getRegionNameFromManifest(final SnapshotRegionManifest manifest) {
byte[] regionName = RegionInfo.createRegionName(
ProtobufUtil.toTableName(manifest.getRegionInfo().getTableName()),
manifest.getRegionInfo().getStartKey().toByteArray(),
manifest.getRegionInfo().getRegionId(), true);
return RegionInfo.encodeRegionName(regionName);
}
/*
* Return the snapshot format
*/
private static int getSnapshotFormat(final SnapshotDescription desc) {
return desc.hasVersion() ? desc.getVersion() : SnapshotManifestV1.DESCRIPTOR_VERSION;
}
}
