org.apache.hadoop.hdfs.client.HdfsAdmin Maven / Gradle / Ivy
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.client;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.fs.BlockStoragePolicySpi;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme;
import org.apache.hadoop.hdfs.DFSInotifyEventInputStream;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicyInfo;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator.OpenFilesType;
import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.security.AccessControlException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.Collection;
import java.util.EnumSet;
/**
* The public API for performing administrative functions on HDFS. Those writing
* applications against HDFS should prefer this interface to directly accessing
* functionality in DistributedFileSystem or DFSClient.
*
* Note that this is distinct from the similarly-named DFSAdmin, which
* is a class that provides the functionality for the CLI `hdfs dfsadmin ...'
* commands.
*/
@InterfaceAudience.Public
@InterfaceStability.Evolving
public class HdfsAdmin {
final private DistributedFileSystem dfs;
public static final FsPermission TRASH_PERMISSION = new FsPermission(
FsAction.ALL, FsAction.ALL, FsAction.ALL, true);
/**
* Create a new HdfsAdmin client.
*
* @param uri the unique URI of the HDFS file system to administer
* @param conf configuration
* @throws IOException in the event the file system could not be created
*/
public HdfsAdmin(URI uri, Configuration conf) throws IOException {
FileSystem fs = FileSystem.get(uri, conf);
if ((fs instanceof ViewFileSystemOverloadScheme)) {
fs = ((ViewFileSystemOverloadScheme) fs)
.getRawFileSystem(new Path(FileSystem.getDefaultUri(conf)), conf);
}
if (!(fs instanceof DistributedFileSystem)) {
throw new IllegalArgumentException("'" + uri + "' is not an HDFS URI.");
} else {
dfs = (DistributedFileSystem)fs;
}
}
/**
* Set the namespace quota (count of files, directories, and sym links) for a
* directory.
*
* @param src the path to set the quota for
* @param quota the value to set for the quota
* @throws IOException in the event of error
*/
public void setQuota(Path src, long quota) throws IOException {
dfs.setQuota(src, quota, HdfsConstants.QUOTA_DONT_SET);
}
/**
* Clear the namespace quota (count of files, directories and sym links) for a
* directory.
*
* @param src the path to clear the quota of
* @throws IOException in the event of error
*/
public void clearQuota(Path src) throws IOException {
dfs.setQuota(src, HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_DONT_SET);
}
/**
* Set the storage space quota (size of files) for a directory. Note that
* directories and sym links do not occupy storage space.
*
* @param src the path to set the space quota of
* @param spaceQuota the value to set for the space quota
* @throws IOException in the event of error
*/
public void setSpaceQuota(Path src, long spaceQuota) throws IOException {
dfs.setQuota(src, HdfsConstants.QUOTA_DONT_SET, spaceQuota);
}
/**
* Clear the storage space quota (size of files) for a directory. Note that
* directories and sym links do not occupy storage space.
*
* @param src the path to clear the space quota of
* @throws IOException in the event of error
*/
public void clearSpaceQuota(Path src) throws IOException {
dfs.setQuota(src, HdfsConstants.QUOTA_DONT_SET, HdfsConstants.QUOTA_RESET);
}
/**
* Set the quota by storage type for a directory. Note that
* directories and sym links do not occupy storage type quota.
*
* @param src the target directory to set the quota by storage type
* @param type the storage type to set for quota by storage type
* @param quota the value to set for quota by storage type
* @throws IOException in the event of error
*/
public void setQuotaByStorageType(Path src, StorageType type, long quota)
throws IOException {
dfs.setQuotaByStorageType(src, type, quota);
}
/**
* Clear the space quota by storage type for a directory. Note that
* directories and sym links do not occupy storage type quota.
*
* @param src the target directory to clear the quota by storage type
* @param type the storage type to clear for quota by storage type
* @throws IOException in the event of error
*/
public void clearQuotaByStorageType(Path src, StorageType type) throws IOException {
dfs.setQuotaByStorageType(src, type, HdfsConstants.QUOTA_RESET);
}
/**
* Allow snapshot on a directory.
* @param path The path of the directory where snapshots will be taken.
*/
public void allowSnapshot(Path path) throws IOException {
dfs.allowSnapshot(path);
if (dfs.isSnapshotTrashRootEnabled()) {
dfs.provisionSnapshotTrash(path, TRASH_PERMISSION);
}
}
/**
* Provision a trash directory for a given snapshottable directory.
* @param path the root of the snapshottable directory
* @return Path of the provisioned trash root
* @throws IOException if the trash directory can not be created.
*/
public Path provisionSnapshotTrash(Path path)
throws IOException {
return dfs.provisionSnapshotTrash(path, TRASH_PERMISSION);
}
/**
* Disallow snapshot on a directory.
* @param path The path of the snapshottable directory.
*/
public void disallowSnapshot(Path path) throws IOException {
dfs.disallowSnapshot(path);
}
/**
* Add a new CacheDirectiveInfo.
*
* @param info Information about a directive to add.
* @param flags {@link CacheFlag}s to use for this operation.
* @return the ID of the directive that was created.
* @throws IOException if the directive could not be added
*/
public long addCacheDirective(CacheDirectiveInfo info,
EnumSet flags) throws IOException {
return dfs.addCacheDirective(info, flags);
}
/**
* Modify a CacheDirective.
*
* @param info Information about the directive to modify. You must set the ID
* to indicate which CacheDirective you want to modify.
* @param flags {@link CacheFlag}s to use for this operation.
* @throws IOException if the directive could not be modified
*/
public void modifyCacheDirective(CacheDirectiveInfo info,
EnumSet flags) throws IOException {
dfs.modifyCacheDirective(info, flags);
}
/**
* Remove a CacheDirective.
*
* @param id identifier of the CacheDirectiveInfo to remove
* @throws IOException if the directive could not be removed
*/
public void removeCacheDirective(long id)
throws IOException {
dfs.removeCacheDirective(id);
}
/**
* List cache directives. Incrementally fetches results from the server.
*
* @param filter Filter parameters to use when listing the directives, null to
* list all directives visible to us.
* @return A RemoteIterator which returns CacheDirectiveInfo objects.
*/
public RemoteIterator listCacheDirectives(
CacheDirectiveInfo filter) throws IOException {
return dfs.listCacheDirectives(filter);
}
/**
* Add a cache pool.
*
* @param info
* The request to add a cache pool.
* @throws IOException
* If the request could not be completed.
*/
public void addCachePool(CachePoolInfo info) throws IOException {
dfs.addCachePool(info);
}
/**
* Modify an existing cache pool.
*
* @param info
* The request to modify a cache pool.
* @throws IOException
* If the request could not be completed.
*/
public void modifyCachePool(CachePoolInfo info) throws IOException {
dfs.modifyCachePool(info);
}
/**
* Remove a cache pool.
*
* @param poolName
* Name of the cache pool to remove.
* @throws IOException
* if the cache pool did not exist, or could not be removed.
*/
public void removeCachePool(String poolName) throws IOException {
dfs.removeCachePool(poolName);
}
/**
* List all cache pools.
*
* @return A remote iterator from which you can get CachePoolEntry objects.
* Requests will be made as needed.
* @throws IOException
* If there was an error listing cache pools.
*/
public RemoteIterator listCachePools() throws IOException {
return dfs.listCachePools();
}
/**
* Get KeyProvider if present.
*
* @return the key provider if encryption is enabled on HDFS.
* Otherwise, it returns null.
* @throws IOException on RPC exception to the NN.
*/
public KeyProvider getKeyProvider() throws IOException {
return dfs.getClient().getKeyProvider();
}
/**
* Create an encryption zone rooted at an empty existing directory, using the
* specified encryption key. An encryption zone has an associated encryption
* key used when reading and writing files within the zone.
*
* @param path The path of the root of the encryption zone. Must refer to
* an empty, existing directory.
* @param keyName Name of key available at the KeyProvider.
* @throws IOException if there was a general IO exception
* @throws AccessControlException if the caller does not have access to path
* @throws FileNotFoundException if the path does not exist
*/
@Deprecated
public void createEncryptionZone(Path path, String keyName)
throws IOException, AccessControlException, FileNotFoundException {
dfs.createEncryptionZone(path, keyName);
}
/**
* Create an encryption zone rooted at an empty existing directory, using the
* specified encryption key. An encryption zone has an associated encryption
* key used when reading and writing files within the zone.
*
* Additional options, such as provisioning the trash directory, can be
* specified using {@link CreateEncryptionZoneFlag} flags.
*
* @param path The path of the root of the encryption zone. Must refer to
* an empty, existing directory.
* @param keyName Name of key available at the KeyProvider.
* @param flags flags for this operation.
* @throws IOException if there was a general IO exception
* @throws AccessControlException if the caller does not have access to path
* @throws FileNotFoundException if the path does not exist
* @throws HadoopIllegalArgumentException if the flags are invalid
*/
public void createEncryptionZone(Path path, String keyName,
EnumSet flags)
throws IOException, AccessControlException, FileNotFoundException,
HadoopIllegalArgumentException{
dfs.createEncryptionZone(path, keyName);
if (flags.contains(CreateEncryptionZoneFlag.PROVISION_TRASH)) {
if (flags.contains(CreateEncryptionZoneFlag.NO_TRASH)) {
throw new HadoopIllegalArgumentException(
"can not have both PROVISION_TRASH and NO_TRASH flags");
}
dfs.provisionEZTrash(path, TRASH_PERMISSION);
}
}
/**
* Provision a trash directory for a given encryption zone.
* @param path the root of the encryption zone
* @throws IOException if the trash directory can not be created.
*/
public void provisionEncryptionZoneTrash(Path path) throws IOException {
dfs.provisionEZTrash(path, TRASH_PERMISSION);
}
/**
* Get the path of the encryption zone for a given file or directory.
*
* @param path The path to get the ez for.
* @return An EncryptionZone, or null if path does not exist or is not in an
* ez.
* @throws IOException if there was a general IO exception
* @throws AccessControlException if the caller does not have access to path
*/
public EncryptionZone getEncryptionZoneForPath(Path path)
throws IOException, AccessControlException {
return dfs.getEZForPath(path);
}
/**
* Returns a RemoteIterator which can be used to list the encryption zones
* in HDFS. For large numbers of encryption zones, the iterator will fetch
* the list of zones in a number of small batches.
*
* Since the list is fetched in batches, it does not represent a
* consistent snapshot of the entire list of encryption zones.
*
* This method can only be called by HDFS superusers.
*/
public RemoteIterator listEncryptionZones()
throws IOException {
return dfs.listEncryptionZones();
}
/**
* Performs re-encryption action for a given encryption zone.
*
* @param zone the root of the encryption zone
* @param action the re-encrypt action
* @throws IOException If any error occurs when handling re-encrypt action.
*/
public void reencryptEncryptionZone(final Path zone,
final ReencryptAction action) throws IOException {
dfs.reencryptEncryptionZone(zone, action);
}
/**
* Returns a RemoteIterator which can be used to list all re-encryption
* information. For large numbers of re-encryptions, the iterator will fetch
* the list in a number of small batches.
*
* Since the list is fetched in batches, it does not represent a
* consistent snapshot of the entire list of encryption zones.
*
* This method can only be called by HDFS superusers.
*/
public RemoteIterator listReencryptionStatus()
throws IOException {
return dfs.listReencryptionStatus();
}
/**
* Returns the FileEncryptionInfo on the HdfsFileStatus for the given path.
* The return value can be null if the path points to a directory, or a file
* that is not in an encryption zone.
*
* @throws FileNotFoundException if the path does not exist.
* @throws AccessControlException if no execute permission on parent path.
*/
public FileEncryptionInfo getFileEncryptionInfo(final Path path)
throws IOException {
return dfs.getFileEncryptionInfo(path);
}
/**
* Exposes a stream of namesystem events. Only events occurring after the
* stream is created are available.
* See {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream}
* for information on stream usage.
* See {@link org.apache.hadoop.hdfs.inotify.Event}
* for information on the available events.
*
* Inotify users may want to tune the following HDFS parameters to
* ensure that enough extra HDFS edits are saved to support inotify clients
* that fall behind the current state of the namespace while reading events.
* The default parameter values should generally be reasonable. If edits are
* deleted before their corresponding events can be read, clients will see a
* {@link org.apache.hadoop.hdfs.inotify.MissingEventsException} on
* {@link org.apache.hadoop.hdfs.DFSInotifyEventInputStream} method calls.
*
* It should generally be sufficient to tune these parameters:
* dfs.namenode.num.extra.edits.retained
* dfs.namenode.max.extra.edits.segments.retained
*
* Parameters that affect the number of created segments and the number of
* edits that are considered necessary, i.e. do not count towards the
* dfs.namenode.num.extra.edits.retained quota):
* dfs.namenode.checkpoint.period
* dfs.namenode.checkpoint.txns
* dfs.namenode.num.checkpoints.retained
* dfs.ha.log-roll.period
*
* It is recommended that local journaling be configured
* (dfs.namenode.edits.dir) for inotify (in addition to a shared journal)
* so that edit transfers from the shared journal can be avoided.
*
* @throws IOException If there was an error obtaining the stream.
*/
public DFSInotifyEventInputStream getInotifyEventStream() throws IOException {
return dfs.getInotifyEventStream();
}
/**
* A version of {@link HdfsAdmin#getInotifyEventStream()} meant for advanced
* users who are aware of HDFS edits up to lastReadTxid (e.g. because they
* have access to an FSImage inclusive of lastReadTxid) and only want to read
* events after this point.
*/
public DFSInotifyEventInputStream getInotifyEventStream(long lastReadTxid)
throws IOException {
return dfs.getInotifyEventStream(lastReadTxid);
}
/**
* Set the source path to the specified storage policy.
*
* @param src The source path referring to either a directory or a file.
* @param policyName The name of the storage policy.
*/
public void setStoragePolicy(final Path src, final String policyName)
throws IOException {
dfs.setStoragePolicy(src, policyName);
}
/**
* Unset the storage policy set for a given file or directory.
*
* @param src file or directory path.
* @throws IOException
*/
public void unsetStoragePolicy(final Path src) throws IOException {
dfs.unsetStoragePolicy(src);
}
/**
* Query the effective storage policy ID for the given file or directory.
*
* @param src file or directory path.
* @return storage policy for the given file or directory.
* @throws IOException
*/
public BlockStoragePolicySpi getStoragePolicy(final Path src)
throws IOException {
return dfs.getStoragePolicy(src);
}
/**
* Retrieve all the storage policies supported by HDFS file system.
*
* @return all storage policies supported by HDFS file system.
* @throws IOException
*/
public Collection getAllStoragePolicies()
throws IOException {
return dfs.getAllStoragePolicies();
}
/**
* Set the source path to the specified erasure coding policy.
*
* @param path The source path referring to a directory.
* @param ecPolicyName The erasure coding policy name for the directory.
*
* @throws IOException
* @throws HadoopIllegalArgumentException if the specified EC policy is not
* enabled on the cluster
*/
public void setErasureCodingPolicy(final Path path,
final String ecPolicyName) throws IOException {
dfs.setErasureCodingPolicy(path, ecPolicyName);
}
/**
* Get the erasure coding policy information for the specified path
*
* @param path
* @return Returns the policy information if file or directory on the path is
* erasure coded. Null otherwise.
* @throws IOException
*/
public ErasureCodingPolicy getErasureCodingPolicy(final Path path)
throws IOException {
return dfs.getErasureCodingPolicy(path);
}
/**
* Set the source path to the specified storage policy.
*
* @param path The source path referring to either a directory or a file.
* @throws IOException
*/
public void satisfyStoragePolicy(final Path path) throws IOException {
dfs.satisfyStoragePolicy(path);
}
/**
* Get the Erasure coding policies supported.
*
* @throws IOException
*/
public ErasureCodingPolicyInfo[] getErasureCodingPolicies()
throws IOException {
return dfs.getClient().getErasureCodingPolicies();
}
/**
* Unset erasure coding policy from the directory.
*
* @param path The source path referring to a directory.
* @throws IOException
*/
public void unsetErasureCodingPolicy(final Path path) throws IOException {
dfs.unsetErasureCodingPolicy(path);
}
/**
* Add Erasure coding policies to HDFS. For each policy input, schema and
* cellSize are musts, name and id are ignored. They will be automatically
* created and assigned by Namenode once the policy is successfully added,
* and will be returned in the response; policy states will be set to
* DISABLED automatically.
*
* @param policies The user defined ec policy list to add.
* @return Return the response list of adding operations.
* @throws IOException
*/
public AddErasureCodingPolicyResponse[] addErasureCodingPolicies(
ErasureCodingPolicy[] policies) throws IOException {
return dfs.addErasureCodingPolicies(policies);
}
/**
* Remove erasure coding policy.
*
* @param ecPolicyName The name of the policy to be removed.
* @throws IOException
*/
public void removeErasureCodingPolicy(String ecPolicyName)
throws IOException {
dfs.removeErasureCodingPolicy(ecPolicyName);
}
/**
* Enable erasure coding policy.
*
* @param ecPolicyName The name of the policy to be enabled.
* @throws IOException
*/
public void enableErasureCodingPolicy(String ecPolicyName)
throws IOException {
dfs.enableErasureCodingPolicy(ecPolicyName);
}
/**
* Disable erasure coding policy.
*
* @param ecPolicyName The name of the policy to be disabled.
* @throws IOException
*/
public void disableErasureCodingPolicy(String ecPolicyName)
throws IOException {
dfs.disableErasureCodingPolicy(ecPolicyName);
}
/**
* Returns a RemoteIterator which can be used to list all open files
* currently managed by the NameNode. For large numbers of open files,
* iterator will fetch the list in batches of configured size.
*
* Since the list is fetched in batches, it does not represent a
* consistent snapshot of the all open files.
*
* This method can only be called by HDFS superusers.
*/
@Deprecated
public RemoteIterator listOpenFiles() throws IOException {
return dfs.listOpenFiles();
}
@Deprecated
public RemoteIterator listOpenFiles(
EnumSet openFilesTypes) throws IOException {
return dfs.listOpenFiles(openFilesTypes);
}
public RemoteIterator listOpenFiles(
EnumSet openFilesTypes, String path) throws IOException {
return dfs.listOpenFiles(openFilesTypes, path);
}
}