All Downloads are FREE. Search and download functionalities are using the official Maven repository.

org.apache.hadoop.hbase.snapshot.SnapshotInfo Maven / Gradle / Ivy

There is a newer version: 3.0.0-beta-1
Show newest version
/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.hadoop.hbase.snapshot;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.SnapshotDescription;
import org.apache.hadoop.hbase.io.HFileLink;
import org.apache.hadoop.hbase.io.WALLink;
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
import org.apache.hadoop.hbase.util.CommonFSUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
import org.apache.hbase.thirdparty.org.apache.commons.cli.DefaultParser;
import org.apache.hbase.thirdparty.org.apache.commons.cli.Option;
import org.apache.hbase.thirdparty.org.apache.commons.cli.Options;
import org.apache.hbase.thirdparty.org.apache.commons.cli.ParseException;

import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;

/**
 * Tool for dumping snapshot information.
 * 
    *
  1. Table Descriptor *
  2. Snapshot creation time, type, format version, ... *
  3. List of hfiles and wals *
  4. Stats about hfiles and logs sizes, percentage of shared with the source table, ... *
*/ @InterfaceAudience.Public public final class SnapshotInfo extends AbstractHBaseTool { private static final Logger LOG = LoggerFactory.getLogger(SnapshotInfo.class); static final class Options { static final Option SNAPSHOT = new Option(null, "snapshot", true, "The name of the snapshot to be detailed."); static final Option REMOTE_DIR = new Option(null, "remote-dir", true, "A custom root directory where snapshots are stored. " + "Use it together with the --snapshot option."); static final Option LIST_SNAPSHOTS = new Option(null, "list-snapshots", false, "List all the available snapshots and exit."); static final Option FILES = new Option(null, "files", false, "The list of files retained by the specified snapshot. " + "Use it together with the --snapshot option."); static final Option STATS = new Option(null, "stats", false, "Additional information about the specified snapshot. " + "Use it together with the --snapshot option."); static final Option SCHEMA = new Option(null, "schema", false, "Show the descriptor of the table for the specified snapshot. " + "Use it together with the --snapshot option."); static final Option SIZE_IN_BYTES = new Option(null, "size-in-bytes", false, "Print the size of the files in bytes. " + "Use it together with the --snapshot and --files options."); } /** * Statistics about the snapshot *
    *
  1. How many store files and logs are in the archive *
  2. How many store files and logs are shared with the table *
  3. Total store files and logs size and shared amount *
*/ public static class SnapshotStats { /** Information about the file referenced by the snapshot */ static class FileInfo { private final boolean corrupted; private final boolean inArchive; private final long size; FileInfo(final boolean inArchive, final long size, final boolean corrupted) { this.corrupted = corrupted; this.inArchive = inArchive; this.size = size; } /** Returns true if the file is in the archive */ public boolean inArchive() { return this.inArchive; } /** Returns true if the file is corrupted */ public boolean isCorrupted() { return this.corrupted; } /** Returns true if the file is missing */ public boolean isMissing() { return this.size < 0; } /** Returns the file size */ public long getSize() { return this.size; } String getStateToString() { if (isCorrupted()) return "CORRUPTED"; if (isMissing()) return "NOT FOUND"; if (inArchive()) return "archive"; return null; } } private AtomicInteger hfilesArchiveCount = new AtomicInteger(); private AtomicInteger hfilesCorrupted = new AtomicInteger(); private AtomicInteger hfilesMissing = new AtomicInteger(); private AtomicInteger hfilesCount = new AtomicInteger(); private AtomicInteger hfilesMobCount = new AtomicInteger(); private AtomicInteger logsMissing = new AtomicInteger(); private AtomicInteger logsCount = new AtomicInteger(); private AtomicLong hfilesArchiveSize = new AtomicLong(); private AtomicLong hfilesSize = new AtomicLong(); private AtomicLong hfilesMobSize = new AtomicLong(); private AtomicLong nonSharedHfilesArchiveSize = new AtomicLong(); private AtomicLong logSize = new AtomicLong(); private final SnapshotProtos.SnapshotDescription snapshot; private final TableName snapshotTable; private final Configuration conf; private final FileSystem fs; SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotDescription snapshot) { this.snapshot = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); this.snapshotTable = snapshot.getTableName(); this.conf = conf; this.fs = fs; } SnapshotStats(final Configuration conf, final FileSystem fs, final SnapshotProtos.SnapshotDescription snapshot) { this.snapshot = snapshot; this.snapshotTable = TableName.valueOf(snapshot.getTable()); this.conf = conf; this.fs = fs; } /** Returns the snapshot descriptor */ public SnapshotDescription getSnapshotDescription() { return ProtobufUtil.createSnapshotDesc(this.snapshot); } /** Returns true if the snapshot is corrupted */ public boolean isSnapshotCorrupted() { return hfilesMissing.get() > 0 || logsMissing.get() > 0 || hfilesCorrupted.get() > 0; } /** Returns the number of available store files */ public int getStoreFilesCount() { return hfilesCount.get() + hfilesArchiveCount.get() + hfilesMobCount.get(); } /** Returns the number of available store files in the archive */ public int getArchivedStoreFilesCount() { return hfilesArchiveCount.get(); } /** Returns the number of available store files in the mob dir */ public int getMobStoreFilesCount() { return hfilesMobCount.get(); } /** Returns the number of available log files */ public int getLogsCount() { return logsCount.get(); } /** Returns the number of missing store files */ public int getMissingStoreFilesCount() { return hfilesMissing.get(); } /** Returns the number of corrupted store files */ public int getCorruptedStoreFilesCount() { return hfilesCorrupted.get(); } /** Returns the number of missing log files */ public int getMissingLogsCount() { return logsMissing.get(); } /** Returns the total size of the store files referenced by the snapshot */ public long getStoreFilesSize() { return hfilesSize.get() + hfilesArchiveSize.get() + hfilesMobSize.get(); } /** Returns the total size of the store files shared */ public long getSharedStoreFilesSize() { return hfilesSize.get(); } /** Returns the total size of the store files in the archive */ public long getArchivedStoreFileSize() { return hfilesArchiveSize.get(); } /** Returns the total size of the store files in the mob store */ public long getMobStoreFilesSize() { return hfilesMobSize.get(); } /** * @return the total size of the store files in the archive which is not shared with other * snapshots and tables This is only calculated when * {@link #getSnapshotStats(Configuration, SnapshotProtos.SnapshotDescription, Map)} is * called with a non-null Map */ public long getNonSharedArchivedStoreFilesSize() { return nonSharedHfilesArchiveSize.get(); } /** Returns the percentage of the shared store files */ public float getSharedStoreFilePercentage() { return ((float) hfilesSize.get() / (getStoreFilesSize())) * 100; } /** Returns the percentage of the mob store files */ public float getMobStoreFilePercentage() { return ((float) hfilesMobSize.get() / (getStoreFilesSize())) * 100; } /** Returns the total log size */ public long getLogsSize() { return logSize.get(); } /** * Check if for a give file in archive, if there are other snapshots/tables still reference it. * @param filePath file path in archive * @param snapshotFilesMap a map for store files in snapshots about how many snapshots refer to * it. * @return true or false */ private boolean isArchivedFileStillReferenced(final Path filePath, final Map snapshotFilesMap) { Integer c = snapshotFilesMap.get(filePath); // Check if there are other snapshots or table from clone_snapshot() (via back-reference) // still reference to it. if ((c != null) && (c == 1)) { Path parentDir = filePath.getParent(); Path backRefDir = HFileLink.getBackReferencesDir(parentDir, filePath.getName()); try { if (CommonFSUtils.listStatus(fs, backRefDir) == null) { return false; } } catch (IOException e) { // For the purpose of this function, IOException is ignored and treated as // the file is still being referenced. } } return true; } /** * Add the specified store file to the stats * @param region region encoded Name * @param family family name * @param storeFile store file name * @param filesMap store files map for all snapshots, it may be null * @return the store file information */ FileInfo addStoreFile(final RegionInfo region, final String family, final SnapshotRegionManifest.StoreFile storeFile, final Map filesMap) throws IOException { HFileLink link = HFileLink.build(conf, snapshotTable, region.getEncodedName(), family, storeFile.getName()); boolean isCorrupted = false; boolean inArchive = false; long size = -1; try { if (fs.exists(link.getArchivePath())) { inArchive = true; size = fs.getFileStatus(link.getArchivePath()).getLen(); hfilesArchiveSize.addAndGet(size); hfilesArchiveCount.incrementAndGet(); // If store file is not shared with other snapshots and tables, // increase nonSharedHfilesArchiveSize if ( (filesMap != null) && !isArchivedFileStillReferenced(link.getArchivePath(), filesMap) ) { nonSharedHfilesArchiveSize.addAndGet(size); } } else if (fs.exists(link.getMobPath())) { inArchive = true; size = fs.getFileStatus(link.getMobPath()).getLen(); hfilesMobSize.addAndGet(size); hfilesMobCount.incrementAndGet(); } else { size = link.getFileStatus(fs).getLen(); hfilesSize.addAndGet(size); hfilesCount.incrementAndGet(); } isCorrupted = (storeFile.hasFileSize() && storeFile.getFileSize() != size); if (isCorrupted) hfilesCorrupted.incrementAndGet(); } catch (FileNotFoundException e) { hfilesMissing.incrementAndGet(); } return new FileInfo(inArchive, size, isCorrupted); } /** * Add the specified log file to the stats * @param server server name * @param logfile log file name * @return the log information */ FileInfo addLogFile(final String server, final String logfile) throws IOException { WALLink logLink = new WALLink(conf, server, logfile); long size = -1; try { size = logLink.getFileStatus(fs).getLen(); logSize.addAndGet(size); logsCount.incrementAndGet(); } catch (FileNotFoundException e) { logsMissing.incrementAndGet(); } return new FileInfo(false, size, false); } } private FileSystem fs; private Path rootDir; private SnapshotManifest snapshotManifest; private boolean listSnapshots = false; private String snapshotName; private Path remoteDir; private boolean showSchema = false; private boolean showFiles = false; private boolean showStats = false; private boolean printSizeInBytes = false; @Override public int doWork() throws IOException, InterruptedException { if (remoteDir != null) { URI defaultFs = remoteDir.getFileSystem(conf).getUri(); CommonFSUtils.setFsDefault(conf, new Path(defaultFs)); CommonFSUtils.setRootDir(conf, remoteDir); } // List Available Snapshots if (listSnapshots) { SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); System.out.printf("%-20s | %-20s | %-20s | %s%n", "SNAPSHOT", "CREATION TIME", "TTL IN SEC", "TABLE NAME"); for (SnapshotDescription desc : getSnapshotList(conf)) { System.out.printf("%-20s | %20s | %20s | %s%n", desc.getName(), df.format(new Date(desc.getCreationTime())), desc.getTtl(), desc.getTableNameAsString()); } return 0; } rootDir = CommonFSUtils.getRootDir(conf); fs = FileSystem.get(rootDir.toUri(), conf); LOG.debug("fs=" + fs.getUri().toString() + " root=" + rootDir); // Load snapshot information if (!loadSnapshotInfo(snapshotName)) { System.err.println("Snapshot '" + snapshotName + "' not found!"); return 1; } printInfo(); if (showSchema) { printSchema(); } printFiles(showFiles, showStats); return 0; } /** * Load snapshot info and table descriptor for the specified snapshot * @param snapshotName name of the snapshot to load * @return false if snapshot is not found */ private boolean loadSnapshotInfo(final String snapshotName) throws IOException { Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); if (!fs.exists(snapshotDir)) { LOG.warn("Snapshot '" + snapshotName + "' not found in: " + snapshotDir); return false; } SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir); snapshotManifest = SnapshotManifest.open(getConf(), fs, snapshotDir, snapshotDesc); return true; } /** * Dump the {@link SnapshotDescription} */ private void printInfo() { SnapshotProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss"); System.out.println("Snapshot Info"); System.out.println("----------------------------------------"); System.out.println(" Name: " + snapshotDesc.getName()); System.out.println(" Type: " + snapshotDesc.getType()); System.out.println(" Table: " + snapshotDesc.getTable()); System.out.println(" Format: " + snapshotDesc.getVersion()); System.out.println("Created: " + df.format(new Date(snapshotDesc.getCreationTime()))); System.out.println(" Ttl: " + snapshotDesc.getTtl()); System.out.println(" Owner: " + snapshotDesc.getOwner()); System.out.println(); } /** * Dump the {@link org.apache.hadoop.hbase.client.TableDescriptor} */ private void printSchema() { System.out.println("Table Descriptor"); System.out.println("----------------------------------------"); System.out.println(snapshotManifest.getTableDescriptor().toString()); System.out.println(); } /** * Collect the hfiles and logs statistics of the snapshot and dump the file list if requested and * the collected information. */ private void printFiles(final boolean showFiles, final boolean showStats) throws IOException { if (showFiles) { System.out.println("Snapshot Files"); System.out.println("----------------------------------------"); } // Collect information about hfiles and logs in the snapshot final SnapshotProtos.SnapshotDescription snapshotDesc = snapshotManifest.getSnapshotDescription(); final String table = snapshotDesc.getTable(); final SnapshotDescription desc = ProtobufUtil.createSnapshotDesc(snapshotDesc); final SnapshotStats stats = new SnapshotStats(this.getConf(), this.fs, desc); SnapshotReferenceUtil.concurrentVisitReferencedFiles(getConf(), fs, snapshotManifest, "SnapshotInfo", new SnapshotReferenceUtil.SnapshotVisitor() { @Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (storeFile.hasReference()) return; SnapshotStats.FileInfo info = stats.addStoreFile(regionInfo, family, storeFile, null); if (showFiles) { String state = info.getStateToString(); System.out.printf("%8s %s/%s/%s/%s %s%n", (info.isMissing() ? "-" : fileSizeToString(info.getSize())), table, regionInfo.getEncodedName(), family, storeFile.getName(), state == null ? "" : "(" + state + ")"); } } }); // Dump the stats System.out.println(); if (stats.isSnapshotCorrupted()) { System.out.println("**************************************************************"); System.out.printf("BAD SNAPSHOT: %d hfile(s) and %d log(s) missing.%n", stats.getMissingStoreFilesCount(), stats.getMissingLogsCount()); System.out.printf(" %d hfile(s) corrupted.%n", stats.getCorruptedStoreFilesCount()); System.out.println("**************************************************************"); } if (showStats) { System.out.printf( "%d HFiles (%d in archive, %d in mob storage), total size %s " + "(%.2f%% %s shared with the source table, %.2f%% %s in mob dir)%n", stats.getStoreFilesCount(), stats.getArchivedStoreFilesCount(), stats.getMobStoreFilesCount(), fileSizeToString(stats.getStoreFilesSize()), stats.getSharedStoreFilePercentage(), fileSizeToString(stats.getSharedStoreFilesSize()), stats.getMobStoreFilePercentage(), fileSizeToString(stats.getMobStoreFilesSize())); System.out.printf("%d Logs, total size %s%n", stats.getLogsCount(), fileSizeToString(stats.getLogsSize())); System.out.println(); } } private String fileSizeToString(long size) { return printSizeInBytes ? Long.toString(size) : StringUtils.humanReadableInt(size); } @Override protected void addOptions() { addOption(Options.SNAPSHOT); addOption(Options.REMOTE_DIR); addOption(Options.LIST_SNAPSHOTS); addOption(Options.FILES); addOption(Options.STATS); addOption(Options.SCHEMA); addOption(Options.SIZE_IN_BYTES); } @Override protected CommandLineParser newParser() { // Commons-CLI lacks the capability to handle combinations of options, so we do it ourselves // Validate in parse() to get helpful error messages instead of exploding in processOptions() return new DefaultParser() { @Override public CommandLine parse(org.apache.hbase.thirdparty.org.apache.commons.cli.Options opts, String[] args, Properties props, boolean stop) throws ParseException { CommandLine cl = super.parse(opts, args, props, stop); if (!cmd.hasOption(Options.LIST_SNAPSHOTS) && !cmd.hasOption(Options.SNAPSHOT)) { throw new ParseException("Missing required snapshot option!"); } return cl; } }; } @Override protected void processOptions(CommandLine cmd) { snapshotName = cmd.getOptionValue(Options.SNAPSHOT.getLongOpt()); showFiles = cmd.hasOption(Options.FILES.getLongOpt()); showStats = cmd.hasOption(Options.FILES.getLongOpt()) || cmd.hasOption(Options.STATS.getLongOpt()); showSchema = cmd.hasOption(Options.SCHEMA.getLongOpt()); listSnapshots = cmd.hasOption(Options.LIST_SNAPSHOTS.getLongOpt()); printSizeInBytes = cmd.hasOption(Options.SIZE_IN_BYTES.getLongOpt()); if (cmd.hasOption(Options.REMOTE_DIR.getLongOpt())) { remoteDir = new Path(cmd.getOptionValue(Options.REMOTE_DIR.getLongOpt())); } } @Override protected void printUsage() { printUsage("hbase snapshot info [options]", "Options:", ""); System.err.println("Examples:"); System.err.println(" hbase snapshot info --snapshot MySnapshot --files"); } /** * Returns the snapshot stats * @param conf the {@link Configuration} to use * @param snapshot {@link SnapshotDescription} to get stats from * @return the snapshot stats */ public static SnapshotStats getSnapshotStats(final Configuration conf, final SnapshotDescription snapshot) throws IOException { SnapshotProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); return getSnapshotStats(conf, snapshotDesc, null); } /** * Returns the snapshot stats * @param conf the {@link Configuration} to use * @param snapshotDesc HBaseProtos.SnapshotDescription to get stats from * @param filesMap {@link Map} store files map for all snapshots, it may be null * @return the snapshot stats */ public static SnapshotStats getSnapshotStats(final Configuration conf, final SnapshotProtos.SnapshotDescription snapshotDesc, final Map filesMap) throws IOException { Path rootDir = CommonFSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); final SnapshotStats stats = new SnapshotStats(conf, fs, snapshotDesc); SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, "SnapshotsStatsAggregation", new SnapshotReferenceUtil.SnapshotVisitor() { @Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (!storeFile.hasReference()) { stats.addStoreFile(regionInfo, family, storeFile, filesMap); } } }); return stats; } /** * Returns the list of available snapshots in the specified location * @param conf the {@link Configuration} to use * @return the list of snapshots */ public static List getSnapshotList(final Configuration conf) throws IOException { Path rootDir = CommonFSUtils.getRootDir(conf); FileSystem fs = FileSystem.get(rootDir.toUri(), conf); Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir); FileStatus[] snapshots = fs.listStatus(snapshotDir, new SnapshotDescriptionUtils.CompletedSnaphotDirectoriesFilter(fs)); List snapshotLists = new ArrayList<>(snapshots.length); for (FileStatus snapshotDirStat : snapshots) { SnapshotProtos.SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDirStat.getPath()); snapshotLists.add(ProtobufUtil.createSnapshotDesc(snapshotDesc)); } return snapshotLists; } /** * Gets the store files map for snapshot * @param conf the {@link Configuration} to use * @param snapshot {@link SnapshotDescription} to get stats from * @param exec the {@link ExecutorService} to use * @param filesMap {@link Map} the map to put the mapping entries * @param uniqueHFilesArchiveSize {@link AtomicLong} the accumulated store file size in archive * @param uniqueHFilesSize {@link AtomicLong} the accumulated store file size shared * @param uniqueHFilesMobSize {@link AtomicLong} the accumulated mob store file size shared */ private static void getSnapshotFilesMap(final Configuration conf, final SnapshotDescription snapshot, final ExecutorService exec, final ConcurrentHashMap filesMap, final AtomicLong uniqueHFilesArchiveSize, final AtomicLong uniqueHFilesSize, final AtomicLong uniqueHFilesMobSize) throws IOException { SnapshotProtos.SnapshotDescription snapshotDesc = ProtobufUtil.createHBaseProtosSnapshotDesc(snapshot); Path rootDir = CommonFSUtils.getRootDir(conf); final FileSystem fs = FileSystem.get(rootDir.toUri(), conf); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotDesc, rootDir); SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc); SnapshotReferenceUtil.concurrentVisitReferencedFiles(conf, fs, manifest, exec, new SnapshotReferenceUtil.SnapshotVisitor() { @Override public void storeFile(final RegionInfo regionInfo, final String family, final SnapshotRegionManifest.StoreFile storeFile) throws IOException { if (!storeFile.hasReference()) { HFileLink link = HFileLink.build(conf, snapshot.getTableName(), regionInfo.getEncodedName(), family, storeFile.getName()); long size; Integer count; Path p; AtomicLong al; int c = 0; if (fs.exists(link.getArchivePath())) { p = link.getArchivePath(); al = uniqueHFilesArchiveSize; size = fs.getFileStatus(p).getLen(); } else if (fs.exists(link.getMobPath())) { p = link.getMobPath(); al = uniqueHFilesMobSize; size = fs.getFileStatus(p).getLen(); } else { p = link.getOriginPath(); al = uniqueHFilesSize; size = link.getFileStatus(fs).getLen(); } // If it has been counted, do not double count count = filesMap.get(p); if (count != null) { c = count.intValue(); } else { al.addAndGet(size); } filesMap.put(p, ++c); } } }); } /** * Returns the map of store files based on path for all snapshots * @param conf the {@link Configuration} to use * @param uniqueHFilesArchiveSize pass out the size for store files in archive * @param uniqueHFilesSize pass out the size for store files shared * @param uniqueHFilesMobSize pass out the size for mob store files shared * @return the map of store files */ public static Map getSnapshotsFilesMap(final Configuration conf, AtomicLong uniqueHFilesArchiveSize, AtomicLong uniqueHFilesSize, AtomicLong uniqueHFilesMobSize) throws IOException { List snapshotList = getSnapshotList(conf); if (snapshotList.isEmpty()) { return Collections.emptyMap(); } ConcurrentHashMap fileMap = new ConcurrentHashMap<>(); ExecutorService exec = SnapshotManifest.createExecutor(conf, "SnapshotsFilesMapping"); try { for (final SnapshotDescription snapshot : snapshotList) { getSnapshotFilesMap(conf, snapshot, exec, fileMap, uniqueHFilesArchiveSize, uniqueHFilesSize, uniqueHFilesMobSize); } } finally { exec.shutdown(); } return fileMap; } public static void main(String[] args) { new SnapshotInfo().doStaticMain(args); } }




© 2015 - 2024 Weber Informatics LLC | Privacy Policy