org.apache.hadoop.fs.FsShell Maven / Gradle / Ivy
The newest version!
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.fs;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.zip.GZIPInputStream;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.shell.CommandFormat;
import org.apache.hadoop.fs.shell.Count;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.util.StringUtils;
/** Provide command line access to a FileSystem. */
@InterfaceAudience.Private
public class FsShell extends Configured implements Tool {
protected FileSystem fs;
private Trash trash;
public static final SimpleDateFormat dateForm =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
protected static final SimpleDateFormat modifFmt =
new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
static final int BORDER = 2;
static {
modifFmt.setTimeZone(TimeZone.getTimeZone("UTC"));
}
static final String SETREP_SHORT_USAGE="-setrep [-R] [-w] ";
static final String GET_SHORT_USAGE = "-get [-ignoreCrc] [-crc] ";
static final String COPYTOLOCAL_SHORT_USAGE = GET_SHORT_USAGE.replace(
"-get", "-copyToLocal");
static final String TAIL_USAGE="-tail [-f] ";
static final String DU_USAGE="-du [-s] [-h] ";
/**
*/
public FsShell() {
this(null);
}
public FsShell(Configuration conf) {
super(conf);
fs = null;
trash = null;
}
protected void init() throws IOException {
getConf().setQuietMode(true);
if (this.fs == null) {
this.fs = FileSystem.get(getConf());
}
if (this.trash == null) {
this.trash = new Trash(getConf());
}
}
/**
* Copies from stdin to the indicated file.
*/
private void copyFromStdin(Path dst, FileSystem dstFs) throws IOException {
if (dstFs.isDirectory(dst)) {
throw new IOException("When source is stdin, destination must be a file.");
}
if (dstFs.exists(dst)) {
throw new IOException("Target " + dst.toString() + " already exists.");
}
FSDataOutputStream out = dstFs.create(dst);
try {
IOUtils.copyBytes(System.in, out, getConf(), false);
}
finally {
out.close();
}
}
/**
* Print from src to stdout.
*/
private void printToStdout(InputStream in) throws IOException {
try {
IOUtils.copyBytes(in, System.out, getConf(), false);
} finally {
in.close();
}
}
/**
* Add local files to the indicated FileSystem name. src is kept.
*/
void copyFromLocal(Path[] srcs, String dstf) throws IOException {
Path dstPath = new Path(dstf);
FileSystem dstFs = dstPath.getFileSystem(getConf());
if (srcs.length == 1 && srcs[0].toString().equals("-"))
copyFromStdin(dstPath, dstFs);
else
dstFs.copyFromLocalFile(false, false, srcs, dstPath);
}
/**
* Add local files to the indicated FileSystem name. src is removed.
*/
void moveFromLocal(Path[] srcs, String dstf) throws IOException {
Path dstPath = new Path(dstf);
FileSystem dstFs = dstPath.getFileSystem(getConf());
dstFs.moveFromLocalFile(srcs, dstPath);
}
/**
* Add a local file to the indicated FileSystem name. src is removed.
*/
void moveFromLocal(Path src, String dstf) throws IOException {
moveFromLocal((new Path[]{src}), dstf);
}
/**
* Obtain the indicated files that match the file pattern srcf
* and copy them to the local name. srcf is kept.
* When copying multiple files, the destination must be a directory.
* Otherwise, IOException is thrown.
* @param argv: arguments
* @param pos: Ignore everything before argv[pos]
* @exception: IOException
* @see org.apache.hadoop.fs.FileSystem.globStatus
*/
void copyToLocal(String[]argv, int pos) throws IOException {
CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
String srcstr = null;
String dststr = null;
try {
List parameters = cf.parse(argv, pos);
srcstr = parameters.get(0);
dststr = parameters.get(1);
}
catch(IllegalArgumentException iae) {
System.err.println("Usage: java FsShell " + GET_SHORT_USAGE);
throw iae;
}
boolean copyCrc = cf.getOpt("crc");
final boolean verifyChecksum = !cf.getOpt("ignoreCrc");
if (dststr.equals("-")) {
if (copyCrc) {
System.err.println("-crc option is not valid when destination is stdout.");
}
cat(srcstr, verifyChecksum);
} else {
File dst = new File(dststr);
Path srcpath = new Path(srcstr);
FileSystem srcFS = getSrcFileSystem(srcpath, verifyChecksum);
if (copyCrc && !(srcFS instanceof ChecksumFileSystem)) {
System.err.println("-crc option is not valid when source file system " +
"does not have crc files. Automatically turn the option off.");
copyCrc = false;
}
FileStatus[] srcs = srcFS.globStatus(srcpath);
if (null == srcs) {
throw new IOException(srcpath + ": No such file or directory");
}
boolean dstIsDir = dst.isDirectory();
if (srcs.length > 1 && !dstIsDir) {
throw new IOException("When copying multiple files, "
+ "destination should be a directory.");
}
for (FileStatus status : srcs) {
Path p = status.getPath();
File f = dstIsDir? new File(dst, p.getName()): dst;
copyToLocal(srcFS, status, f, copyCrc);
}
}
}
/**
* Return the {@link FileSystem} specified by src and the conf.
* It the {@link FileSystem} supports checksum, set verifyChecksum.
*/
private FileSystem getSrcFileSystem(Path src, boolean verifyChecksum
) throws IOException {
FileSystem srcFs = src.getFileSystem(getConf());
srcFs.setVerifyChecksum(verifyChecksum);
return srcFs;
}
/**
* The prefix for the tmp file used in copyToLocal.
* It must be at least three characters long, required by
* {@link java.io.File#createTempFile(String, String, File)}.
*/
static final String COPYTOLOCAL_PREFIX = "_copyToLocal_";
/**
* Copy a source file from a given file system to local destination.
* @param srcFS source file system
* @param src source path
* @param dst destination
* @param copyCrc copy CRC files?
* @exception IOException If some IO failed
*/
private void copyToLocal(final FileSystem srcFS, final FileStatus srcStatus,
final File dst, final boolean copyCrc)
throws IOException {
/* Keep the structure similar to ChecksumFileSystem.copyToLocal().
* Ideal these two should just invoke FileUtil.copy() and not repeat
* recursion here. Of course, copy() should support two more options :
* copyCrc and useTmpFile (may be useTmpFile need not be an option).
*/
Path src = srcStatus.getPath();
if (srcStatus.isFile()) {
if (dst.exists()) {
// match the error message in FileUtil.checkDest():
throw new IOException("Target " + dst + " already exists");
}
// use absolute name so that tmp file is always created under dest dir
File tmp = FileUtil.createLocalTempFile(dst.getAbsoluteFile(),
COPYTOLOCAL_PREFIX, true);
if (!FileUtil.copy(srcFS, src, tmp, false, srcFS.getConf())) {
throw new IOException("Failed to copy " + src + " to " + dst);
}
if (!tmp.renameTo(dst)) {
throw new IOException("Failed to rename tmp file " + tmp +
" to local destination \"" + dst + "\".");
}
if (copyCrc) {
if (!(srcFS instanceof ChecksumFileSystem)) {
throw new IOException("Source file system does not have crc files");
}
ChecksumFileSystem csfs = (ChecksumFileSystem) srcFS;
File dstcs = FileSystem.getLocal(srcFS.getConf())
.pathToFile(csfs.getChecksumFile(new Path(dst.getCanonicalPath())));
FileSystem fs = csfs.getRawFileSystem();
FileStatus status = csfs.getFileStatus(csfs.getChecksumFile(src));
copyToLocal(fs, status, dstcs, false);
}
} else if (srcStatus.isSymlink()) {
throw new AssertionError("Symlinks unsupported");
} else {
// once FileUtil.copy() supports tmp file, we don't need to mkdirs().
if (!dst.mkdirs()) {
throw new IOException("Failed to create local destination \"" +
dst + "\".");
}
for(FileStatus status : srcFS.listStatus(src)) {
copyToLocal(srcFS, status,
new File(dst, status.getPath().getName()), copyCrc);
}
}
}
/**
* Get all the files in the directories that match the source file
* pattern and merge and sort them to only one file on local fs
* srcf is kept.
* @param srcf: a file pattern specifying source files
* @param dstf: a destination local file/directory
* @exception: IOException
* @see org.apache.hadoop.fs.FileSystem.globStatus
*/
void copyMergeToLocal(String srcf, Path dst) throws IOException {
copyMergeToLocal(srcf, dst, false);
}
/**
* Get all the files in the directories that match the source file pattern
* and merge and sort them to only one file on local fs
* srcf is kept.
*
* Also adds a string between the files (useful for adding \n
* to a text file)
* @param srcf: a file pattern specifying source files
* @param dstf: a destination local file/directory
* @param endline: if an end of line character is added to a text file
* @exception: IOException
* @see org.apache.hadoop.fs.FileSystem.globStatus
*/
void copyMergeToLocal(String srcf, Path dst, boolean endline) throws IOException {
Path srcPath = new Path(srcf);
FileSystem srcFs = srcPath.getFileSystem(getConf());
Path [] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath),
srcPath);
for(int i=0; isrcf and display
* their content on stdout.
* @param srcf: a file pattern specifying source files
* @exception: IOException
* @see org.apache.hadoop.fs.FileSystem.globStatus
*/
void cat(String src, boolean verifyChecksum) throws IOException {
//cat behavior in Linux
// [~/1207]$ ls ?.txt
// x.txt z.txt
// [~/1207]$ cat x.txt y.txt z.txt
// xxx
// cat: y.txt: No such file or directory
// zzz
Path srcPattern = new Path(src);
new DelayedExceptionThrowing() {
@Override
void process(Path p, FileSystem srcFs) throws IOException {
printToStdout(srcFs.open(p));
}
}.globAndProcess(srcPattern, getSrcFileSystem(srcPattern, verifyChecksum));
}
private class TextRecordInputStream extends InputStream {
SequenceFile.Reader r;
WritableComparable key;
Writable val;
DataInputBuffer inbuf;
DataOutputBuffer outbuf;
public TextRecordInputStream(FileStatus f) throws IOException {
final Path fpath = f.getPath();
final Configuration lconf = getConf();
r = new SequenceFile.Reader(lconf,
SequenceFile.Reader.file(fpath));
key = ReflectionUtils.newInstance(
r.getKeyClass().asSubclass(WritableComparable.class), lconf);
val = ReflectionUtils.newInstance(
r.getValueClass().asSubclass(Writable.class), lconf);
inbuf = new DataInputBuffer();
outbuf = new DataOutputBuffer();
}
public int read() throws IOException {
int ret;
if (null == inbuf || -1 == (ret = inbuf.read())) {
if (!r.next(key, val)) {
return -1;
}
byte[] tmp = key.toString().getBytes();
outbuf.write(tmp, 0, tmp.length);
outbuf.write('\t');
tmp = val.toString().getBytes();
outbuf.write(tmp, 0, tmp.length);
outbuf.write('\n');
inbuf.reset(outbuf.getData(), outbuf.getLength());
outbuf.reset();
ret = inbuf.read();
}
return ret;
}
public void close() throws IOException {
r.close();
super.close();
}
}
private InputStream forMagic(Path p, FileSystem srcFs) throws IOException {
FSDataInputStream i = srcFs.open(p);
// check codecs
CompressionCodecFactory cf = new CompressionCodecFactory(getConf());
CompressionCodec codec = cf.getCodec(p);
if (codec != null) {
return codec.createInputStream(i);
}
switch(i.readShort()) {
case 0x1f8b: // RFC 1952
i.seek(0);
return new GZIPInputStream(i);
case 0x5345: // 'S' 'E'
if (i.readByte() == 'Q') {
i.close();
return new TextRecordInputStream(srcFs.getFileStatus(p));
}
break;
}
i.seek(0);
return i;
}
void text(String srcf) throws IOException {
Path srcPattern = new Path(srcf);
new DelayedExceptionThrowing() {
@Override
void process(Path p, FileSystem srcFs) throws IOException {
if (srcFs.isDirectory(p)) {
throw new IOException("Source must be a file.");
}
printToStdout(forMagic(p, srcFs));
}
}.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf()));
}
/**
* Parse the incoming command string
* @param cmd
* @param pos ignore anything before this pos in cmd
* @throws IOException
*/
private void setReplication(String[] cmd, int pos) throws IOException {
CommandFormat c = new CommandFormat("setrep", 2, 2, "R", "w");
String dst = null;
short rep = 0;
try {
List parameters = c.parse(cmd, pos);
rep = Short.parseShort(parameters.get(0));
dst = parameters.get(1);
} catch (NumberFormatException nfe) {
System.err.println("Illegal replication, a positive integer expected");
throw nfe;
}
catch(IllegalArgumentException iae) {
System.err.println("Usage: java FsShell " + SETREP_SHORT_USAGE);
throw iae;
}
if (rep < 1) {
System.err.println("Cannot set replication to: " + rep);
throw new IllegalArgumentException("replication must be >= 1");
}
List waitList = c.getOpt("w")? new ArrayList(): null;
setReplication(rep, dst, c.getOpt("R"), waitList);
if (waitList != null) {
waitForReplication(waitList, rep);
}
}
/**
* Wait for all files in waitList to have replication number equal to rep.
* @param waitList The files are waited for.
* @param rep The new replication number.
* @throws IOException IOException
*/
void waitForReplication(List waitList, int rep) throws IOException {
for(Path f : waitList) {
System.out.print("Waiting for " + f + " ...");
System.out.flush();
boolean printWarning = false;
FileStatus status = fs.getFileStatus(f);
long len = status.getLen();
for(boolean done = false; !done; ) {
BlockLocation[] locations = fs.getFileBlockLocations(status, 0, len);
int i = 0;
for(; i < locations.length &&
locations[i].getHosts().length == rep; i++)
if (!printWarning && locations[i].getHosts().length > rep) {
System.out.println("\nWARNING: the waiting time may be long for "
+ "DECREASING the number of replication.");
printWarning = true;
}
done = i == locations.length;
if (!done) {
System.out.print(".");
System.out.flush();
try {Thread.sleep(10000);} catch (InterruptedException e) {}
}
}
System.out.println(" done");
}
}
/**
* Set the replication for files that match file pattern srcf
* if it's a directory and recursive is true,
* set replication for all the subdirs and those files too.
* @param newRep new replication factor
* @param srcf a file pattern specifying source files
* @param recursive if need to set replication factor for files in subdirs
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
void setReplication(short newRep, String srcf, boolean recursive,
List waitingList)
throws IOException {
Path srcPath = new Path(srcf);
FileSystem srcFs = srcPath.getFileSystem(getConf());
Path[] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath),
srcPath);
for(int i=0; i waitingList)
throws IOException {
if (srcFs.getFileStatus(src).isFile()) {
setFileReplication(src, srcFs, newRep, waitingList);
return;
}
FileStatus items[];
try {
items = srcFs.listStatus(src);
} catch (FileNotFoundException fnfe) {
throw new IOException("Could not get listing for " + src);
}
for (int i = 0; i < items.length; i++) {
if (items[i].isFile()) {
setFileReplication(items[i].getPath(), srcFs, newRep, waitingList);
} else if (items[i].isSymlink()) {
throw new AssertionError("Symlinks unsupported");
} else if (recursive) {
setReplication(newRep, srcFs, items[i].getPath(), recursive,
waitingList);
}
}
}
/**
* Actually set the replication for this file
* If it fails either throw IOException or print an error msg
* @param file: a file/directory
* @param newRep: new replication factor
* @throws IOException
*/
private void setFileReplication(Path file, FileSystem srcFs, short newRep, List waitList)
throws IOException {
if (srcFs.setReplication(file, newRep)) {
if (waitList != null) {
waitList.add(file);
}
System.out.println("Replication " + newRep + " set: " + file);
} else {
System.err.println("Could not set replication for: " + file);
}
}
/**
* Get a listing of all files in that match the file pattern srcf.
* @param srcf a file pattern specifying source files
* @param recursive if need to list files in subdirs
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
private int ls(String srcf, boolean recursive) throws IOException {
Path srcPath = new Path(srcf);
FileSystem srcFs = srcPath.getFileSystem(this.getConf());
FileStatus[] srcs = srcFs.globStatus(srcPath);
if (srcs==null || srcs.length==0) {
throw new FileNotFoundException("Cannot access " + srcf +
": No such file or directory.");
}
boolean printHeader = (srcs.length == 1) ? true: false;
int numOfErrors = 0;
for(int i=0; isrc
* ideally we should provide "-l" option, that lists like "ls -l".
*/
private int ls(FileStatus src, FileSystem srcFs, boolean recursive,
boolean printHeader) throws IOException {
final String cmd = recursive? "lsr": "ls";
final FileStatus[] items = shellListStatus(cmd, srcFs, src);
if (items == null) {
return 1;
} else {
int numOfErrors = 0;
if (!recursive && printHeader) {
if (items.length != 0) {
System.out.println("Found " + items.length + " items");
}
}
int maxReplication = 3, maxLen = 10, maxOwner = 0,maxGroup = 0;
for(int i = 0; i < items.length; i++) {
FileStatus stat = items[i];
int replication = String.valueOf(stat.getReplication()).length();
int len = String.valueOf(stat.getLen()).length();
int owner = String.valueOf(stat.getOwner()).length();
int group = String.valueOf(stat.getGroup()).length();
if (replication > maxReplication) maxReplication = replication;
if (len > maxLen) maxLen = len;
if (owner > maxOwner) maxOwner = owner;
if (group > maxGroup) maxGroup = group;
}
for (int i = 0; i < items.length; i++) {
FileStatus stat = items[i];
Path cur = stat.getPath();
String mdate = dateForm.format(new Date(stat.getModificationTime()));
System.out.print((stat.isDirectory() ? "d" : "-") +
stat.getPermission() + " ");
System.out.printf("%"+ maxReplication +
"s ", (stat.isFile() ? stat.getReplication() : "-"));
if (maxOwner > 0)
System.out.printf("%-"+ maxOwner + "s ", stat.getOwner());
if (maxGroup > 0)
System.out.printf("%-"+ maxGroup + "s ", stat.getGroup());
System.out.printf("%"+ maxLen + "d ", stat.getLen());
System.out.print(mdate + " ");
System.out.println(cur.toUri().getPath());
if (recursive && stat.isDirectory()) {
numOfErrors += ls(stat,srcFs, recursive, printHeader);
}
}
return numOfErrors;
}
}
/**
* Show the size of a partition in the filesystem that contains
* the specified path.
* @param path a path specifying the source partition. null means /.
* @throws IOException
*/
void df(String path) throws IOException {
if (path == null) path = "/";
final Path srcPath = new Path(path);
final FileSystem srcFs = srcPath.getFileSystem(getConf());
if (! srcFs.exists(srcPath)) {
throw new FileNotFoundException("Cannot access "+srcPath.toString());
}
final FsStatus stats = srcFs.getStatus(srcPath);
final int PercentUsed = (int)(100.0f * (float)stats.getUsed() / (float)stats.getCapacity());
System.out.println("Filesystem\t\tSize\tUsed\tAvail\tUse%");
System.out.printf("%s\t\t%d\t%d\t%d\t%d%%\n",
path,
stats.getCapacity(), stats.getUsed(), stats.getRemaining(),
PercentUsed);
}
/**
* Show the size of all files that match the file pattern src
* @param cmd
* @param pos ignore anything before this pos in cmd
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
void du(String[] cmd, int pos) throws IOException {
CommandFormat c = new CommandFormat(
"du", 0, Integer.MAX_VALUE, "h", "s");
List params;
try {
params = c.parse(cmd, pos);
} catch (IllegalArgumentException iae) {
System.err.println("Usage: java FsShell " + DU_USAGE);
throw iae;
}
boolean humanReadable = c.getOpt("h");
boolean summary = c.getOpt("s");
// Default to cwd
if (params.isEmpty()) {
params.add(".");
}
List usages = new ArrayList();
for (String src : params) {
Path srcPath = new Path(src);
FileSystem srcFs = srcPath.getFileSystem(getConf());
FileStatus globStatus[] = srcFs.globStatus(srcPath);
FileStatus statusToPrint[];
if (summary) {
statusToPrint = globStatus;
} else {
Path statPaths[] = FileUtil.stat2Paths(globStatus, srcPath);
try {
statusToPrint = srcFs.listStatus(statPaths);
} catch(FileNotFoundException fnfe) {
statusToPrint = null;
}
}
if ((statusToPrint == null) || ((statusToPrint.length == 0) &&
(!srcFs.exists(srcPath)))){
throw new FileNotFoundException("Cannot access " + src
+ ": No such file or directory.");
}
if (!summary) {
System.out.println("Found " + statusToPrint.length + " items");
}
for (FileStatus stat : statusToPrint) {
long length;
if (summary || stat.isDirectory()) {
length = srcFs.getContentSummary(stat.getPath()).getLength();
} else {
length = stat.getLen();
}
usages.add(new UsagePair(String.valueOf(stat.getPath()), length));
}
}
printUsageSummary(usages, humanReadable);
}
/**
* Show the summary disk usage of each dir/file
* that matches the file pattern src
* @param cmd
* @param pos ignore anything before this pos in cmd
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
void dus(String[] cmd, int pos) throws IOException {
String newcmd[] = new String[cmd.length + 1];
System.arraycopy(cmd, 0, newcmd, 0, cmd.length);
newcmd[cmd.length] = "-s";
du(newcmd, pos);
}
private void printUsageSummary(List usages,
boolean humanReadable) {
int maxColumnWidth = 0;
for (UsagePair usage : usages) {
String toPrint = humanReadable ?
StringUtils.humanReadableInt(usage.bytes) : String.valueOf(usage.bytes);
if (toPrint.length() > maxColumnWidth) {
maxColumnWidth = toPrint.length();
}
}
for (UsagePair usage : usages) {
String toPrint = humanReadable ?
StringUtils.humanReadableInt(usage.bytes) : String.valueOf(usage.bytes);
System.out.printf("%-"+ (maxColumnWidth + BORDER) +"s", toPrint);
System.out.println(usage.path);
}
}
/**
* Create the given dir
*/
void mkdir(String src) throws IOException {
Path f = new Path(src);
FileSystem srcFs = f.getFileSystem(getConf());
FileStatus fstatus = null;
try {
fstatus = srcFs.getFileStatus(f);
if (fstatus.isDirectory()) {
throw new IOException("cannot create directory "
+ src + ": File exists");
}
else {
throw new IOException(src + " exists but " +
"is not a directory");
}
} catch(FileNotFoundException e) {
if (!srcFs.mkdirs(f)) {
throw new IOException("failed to create " + src);
}
}
}
/**
* (Re)create zero-length file at the specified path.
* This will be replaced by a more UNIX-like touch when files may be
* modified.
*/
void touchz(String src) throws IOException {
Path f = new Path(src);
FileSystem srcFs = f.getFileSystem(getConf());
FileStatus st;
if (srcFs.exists(f)) {
st = srcFs.getFileStatus(f);
if (st.isDirectory()) {
// TODO: handle this
throw new IOException(src + " is a directory");
} else if (st.getLen() != 0)
throw new IOException(src + " must be a zero-length file");
}
FSDataOutputStream out = srcFs.create(f);
out.close();
}
/**
* Check file types.
*/
int test(String argv[], int i) throws IOException {
if (!argv[i].startsWith("-") || argv[i].length() > 2)
throw new IOException("Not a flag: " + argv[i]);
char flag = argv[i].toCharArray()[1];
Path f = new Path(argv[++i]);
FileSystem srcFs = f.getFileSystem(getConf());
switch(flag) {
case 'e':
return srcFs.exists(f) ? 0 : 1;
case 'z':
return srcFs.getFileStatus(f).getLen() == 0 ? 0 : 1;
case 'd':
return srcFs.getFileStatus(f).isDirectory() ? 0 : 1;
default:
throw new IOException("Unknown flag: " + flag);
}
}
/**
* Print statistics about path in specified format.
* Format sequences:
* %b: Size of file in blocks
* %n: Filename
* %o: Block size
* %r: replication
* %y: UTC date as "yyyy-MM-dd HH:mm:ss"
* %Y: Milliseconds since January 1, 1970 UTC
*/
void stat(char[] fmt, String src) throws IOException {
Path srcPath = new Path(src);
FileSystem srcFs = srcPath.getFileSystem(getConf());
FileStatus glob[] = srcFs.globStatus(srcPath);
if (null == glob)
throw new IOException("cannot stat `" + src + "': No such file or directory");
for (FileStatus f : glob) {
StringBuilder buf = new StringBuilder();
for (int i = 0; i < fmt.length; ++i) {
if (fmt[i] != '%') {
buf.append(fmt[i]);
} else {
if (i + 1 == fmt.length) break;
switch(fmt[++i]) {
case 'b':
buf.append(f.getLen());
break;
case 'F':
buf.append(f.isDirectory() ? "directory"
: (f.isFile() ? "regular file" : "symlink"));
break;
case 'n':
buf.append(f.getPath().getName());
break;
case 'o':
buf.append(f.getBlockSize());
break;
case 'r':
buf.append(f.getReplication());
break;
case 'y':
buf.append(modifFmt.format(new Date(f.getModificationTime())));
break;
case 'Y':
buf.append(f.getModificationTime());
break;
default:
buf.append(fmt[i]);
break;
}
}
}
System.out.println(buf.toString());
}
}
/**
* Move files that match the file pattern srcf
* to a destination file.
* When moving mutiple files, the destination must be a directory.
* Otherwise, IOException is thrown.
* @param srcf a file pattern specifying source files
* @param dstf a destination local file/directory
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
void rename(String srcf, String dstf) throws IOException {
Path srcPath = new Path(srcf);
Path dstPath = new Path(dstf);
FileSystem fs = srcPath.getFileSystem(getConf());
URI srcURI = fs.getUri();
URI dstURI = dstPath.getFileSystem(getConf()).getUri();
if (srcURI.compareTo(dstURI) != 0) {
throw new IOException("src and destination filesystems do not match.");
}
Path[] srcs = FileUtil.stat2Paths(fs.globStatus(srcPath), srcPath);
Path dst = new Path(dstf);
if (srcs.length > 1 && !fs.isDirectory(dst)) {
throw new IOException("When moving multiple files, "
+ "destination should be a directory.");
}
for(int i=0; i 3) {
Path dst = new Path(dest);
FileSystem dstFs = dst.getFileSystem(getConf());
if (!dstFs.isDirectory(dst)) {
throw new IOException("When moving multiple files, "
+ "destination " + dest + " should be a directory.");
}
}
//
// for each source file, issue the rename
//
for (; i < argv.length - 1; i++) {
try {
//
// issue the rename to the fs
//
rename(argv[i], dest);
} catch (RemoteException e) {
//
// This is a error returned by hadoop server. Print
// out the first line of the error mesage.
//
exitCode = -1;
try {
String[] content;
content = e.getLocalizedMessage().split("\n");
System.err.println(cmd.substring(1) + ": " + content[0]);
} catch (Exception ex) {
System.err.println(cmd.substring(1) + ": " +
ex.getLocalizedMessage());
}
} catch (IOException e) {
//
// IO exception encountered locally.
//
exitCode = -1;
System.err.println(cmd.substring(1) + ": " +
e.getLocalizedMessage());
}
}
return exitCode;
}
/**
* Copy files that match the file pattern srcf
* to a destination file.
* When copying mutiple files, the destination must be a directory.
* Otherwise, IOException is thrown.
* @param srcf a file pattern specifying source files
* @param dstf a destination local file/directory
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
void copy(String srcf, String dstf, Configuration conf) throws IOException {
Path srcPath = new Path(srcf);
FileSystem srcFs = srcPath.getFileSystem(getConf());
Path dstPath = new Path(dstf);
FileSystem dstFs = dstPath.getFileSystem(getConf());
Path [] srcs = FileUtil.stat2Paths(srcFs.globStatus(srcPath), srcPath);
if (srcs.length > 1 && !dstFs.isDirectory(dstPath)) {
throw new IOException("When copying multiple files, "
+ "destination should be a directory.");
}
for(int i=0; i 3) {
Path dst = new Path(dest);
if (!fs.isDirectory(dst)) {
throw new IOException("When copying multiple files, "
+ "destination " + dest + " should be a directory.");
}
}
//
// for each source file, issue the copy
//
for (; i < argv.length - 1; i++) {
try {
//
// issue the copy to the fs
//
copy(argv[i], dest, conf);
} catch (RemoteException e) {
//
// This is a error returned by hadoop server. Print
// out the first line of the error mesage.
//
exitCode = -1;
try {
String[] content;
content = e.getLocalizedMessage().split("\n");
System.err.println(cmd.substring(1) + ": " +
content[0]);
} catch (Exception ex) {
System.err.println(cmd.substring(1) + ": " +
ex.getLocalizedMessage());
}
} catch (IOException e) {
//
// IO exception encountered locally.
//
exitCode = -1;
System.err.println(cmd.substring(1) + ": " +
e.getLocalizedMessage());
}
}
return exitCode;
}
/**
* Delete all files that match the file pattern srcf.
* @param srcf a file pattern specifying source files
* @param recursive if need to delete subdirs
* @param skipTrash Should we skip the trash, if it's enabled?
* @throws IOException
* @see org.apache.hadoop.fs.FileSystem#globStatus(Path)
*/
void delete(String srcf, final boolean recursive, final boolean skipTrash)
throws IOException {
//rm behavior in Linux
// [~/1207]$ ls ?.txt
// x.txt z.txt
// [~/1207]$ rm x.txt y.txt z.txt
// rm: cannot remove `y.txt': No such file or directory
Path srcPattern = new Path(srcf);
new DelayedExceptionThrowing() {
@Override
void process(Path p, FileSystem srcFs) throws IOException {
delete(p, srcFs, recursive, skipTrash);
}
}.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf()));
}
/* delete a file */
private void delete(Path src, FileSystem srcFs, boolean recursive,
boolean skipTrash) throws IOException {
FileStatus fs = null;
try {
fs = srcFs.getFileStatus(src);
} catch (FileNotFoundException fnfe) {
// Have to re-throw so that console output is as expected
throw new FileNotFoundException("cannot remove "
+ src + ": No such file or directory.");
}
if (fs.isDirectory() && !recursive) {
throw new IOException("Cannot remove directory \"" + src +
"\", use -rmr instead");
}
if(!skipTrash) {
try {
Trash trashTmp = new Trash(srcFs, getConf());
if (trashTmp.moveToTrash(src)) {
System.out.println("Moved to trash: " + src);
return;
}
} catch (IOException e) {
Exception cause = (Exception) e.getCause();
String msg = "";
if(cause != null) {
msg = cause.getLocalizedMessage();
}
System.err.println("Problem with Trash." + msg +". Consider using -skipTrash option");
throw e;
}
}
if (srcFs.delete(src, true)) {
System.out.println("Deleted " + src);
} else {
throw new IOException("Delete failed " + src);
}
}
private void expunge() throws IOException {
trash.expunge();
trash.checkpoint();
}
/**
* Returns the Trash object associated with this shell.
*/
public Path getCurrentTrashDir() {
return trash.getCurrentTrashDir();
}
/**
* Parse the incoming command string
* @param cmd
* @param pos ignore anything before this pos in cmd
* @throws IOException
*/
private void tail(String[] cmd, int pos) throws IOException {
CommandFormat c = new CommandFormat("tail", 1, 1, "f");
String src = null;
Path path = null;
try {
List parameters = c.parse(cmd, pos);
src = parameters.get(0);
} catch(IllegalArgumentException iae) {
System.err.println("Usage: java FsShell " + TAIL_USAGE);
throw iae;
}
boolean foption = c.getOpt("f") ? true: false;
path = new Path(src);
FileSystem srcFs = path.getFileSystem(getConf());
FileStatus fileStatus = srcFs.getFileStatus(path);
if (fileStatus.isDirectory()) {
throw new IOException("Source must be a file.");
}
long fileSize = fileStatus.getLen();
long offset = (fileSize > 1024) ? fileSize - 1024: 0;
while (true) {
FSDataInputStream in = srcFs.open(path);
try {
in.seek(offset);
IOUtils.copyBytes(in, System.out, 1024);
offset = in.getPos();
} finally {
in.close();
}
if (!foption) {
break;
}
fileSize = srcFs.getFileStatus(path).getLen();
offset = (fileSize > offset) ? offset: fileSize;
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
break;
}
}
}
/**
* This class runs a command on a given FileStatus. This can be used for
* running various commands like chmod, chown etc.
*/
static abstract class CmdHandler {
protected int errorCode = 0;
protected boolean okToContinue = true;
protected String cmdName;
int getErrorCode() { return errorCode; }
boolean okToContinue() { return okToContinue; }
String getName() { return cmdName; }
protected CmdHandler(String cmdName, FileSystem fs) {
this.cmdName = cmdName;
}
public abstract void run(FileStatus file, FileSystem fs) throws IOException;
}
/** helper returns listStatus() */
private static FileStatus[] shellListStatus(String cmd,
FileSystem srcFs,
FileStatus src) {
if (src.isFile()) {
FileStatus[] files = { src };
return files;
}
Path path = src.getPath();
try {
FileStatus[] files = srcFs.listStatus(path);
return files;
} catch(FileNotFoundException fnfe) {
System.err.println(cmd + ": could not get listing for '" + path + "'");
} catch (IOException e) {
System.err.println(cmd +
": could not get get listing for '" + path + "' : " +
e.getMessage().split("\n")[0]);
}
return null;
}
/**
* Runs the command on a given file with the command handler.
* If recursive is set, command is run recursively.
*/
private static int runCmdHandler(CmdHandler handler, FileStatus stat,
FileSystem srcFs,
boolean recursive) throws IOException {
int errors = 0;
handler.run(stat, srcFs);
if (recursive && stat.isDirectory() && handler.okToContinue()) {
FileStatus[] files = shellListStatus(handler.getName(), srcFs, stat);
if (files == null) {
return 1;
}
for(FileStatus file : files ) {
errors += runCmdHandler(handler, file, srcFs, recursive);
}
}
return errors;
}
///top level runCmdHandler
int runCmdHandler(CmdHandler handler, String[] args,
int startIndex, boolean recursive)
throws IOException {
int errors = 0;
for (int i=startIndex; i 0 || handler.getErrorCode() != 0) ? 1 : 0;
}
/**
* Return an abbreviated English-language desc of the byte length
* @deprecated Consider using {@link org.apache.hadoop.util.StringUtils#byteDesc} instead.
*/
@Deprecated
public static String byteDesc(long len) {
return StringUtils.byteDesc(len);
}
/**
* @deprecated Consider using {@link org.apache.hadoop.util.StringUtils#limitDecimalTo2} instead.
*/
@Deprecated
public static synchronized String limitDecimalTo2(double d) {
return StringUtils.limitDecimalTo2(d);
}
private void printHelp(String cmd) {
String summary = "hadoop fs is the command to execute fs commands. " +
"The full syntax is: \n\n" +
"hadoop fs [-fs ] [-conf ]\n\t" +
"[-D ] [-ls ] [-lsr ] [-df []] [-du [-s] [-h] ]\n\t" +
"[-dus ] [-mv ] [-cp ] [-rm [-skipTrash] ]\n\t" +
"[-rmr [-skipTrash] ] [-put ... ] [-copyFromLocal ... ]\n\t" +
"[-moveFromLocal ... ] [" +
GET_SHORT_USAGE + "\n\t" +
"[-getmerge [addnl]] [-cat ]\n\t" +
"[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal ]\n\t" +
"[-mkdir ] [-report] [" + SETREP_SHORT_USAGE + "]\n\t" +
"[-touchz ] [-test -[ezd] ] [-stat [format] ]\n\t" +
"[-tail [-f] ] [-text ]\n\t" +
"[" + FsShellPermissions.CHMOD_USAGE + "]\n\t" +
"[" + FsShellPermissions.CHOWN_USAGE + "]\n\t" +
"[" + FsShellPermissions.CHGRP_USAGE + "]\n\t" +
"[" + Count.USAGE + "]\n\t" +
"[-help [cmd]]\n";
String conf ="-conf : Specify an application configuration file.";
String D = "-D : Use value for given property.";
String fs = "-fs [local | ]: \tSpecify the file system to use.\n" +
"\t\tIf not specified, the current configuration is used, \n" +
"\t\ttaken from the following, in increasing precedence: \n" +
"\t\t\tcore-default.xml inside the hadoop jar file \n" +
"\t\t\tcore-site.xml in $HADOOP_CONF_DIR \n" +
"\t\t'local' means use the local file system as your DFS. \n" +
"\t\t specifies a particular file system to \n" +
"\t\tcontact. This argument is optional but if used must appear\n" +
"\t\tappear first on the command line. Exactly one additional\n" +
"\t\targument must be specified. \n";
String ls = "-ls : \tList the contents that match the specified file pattern. If\n" +
"\t\tpath is not specified, the contents of /user/\n" +
"\t\twill be listed. Directory entries are of the form \n" +
"\t\t\tdirName (full path) \n" +
"\t\tand file entries are of the form \n" +
"\t\t\tfileName(full path) size \n" +
"\t\twhere n is the number of replicas specified for the file \n" +
"\t\tand size is the size of the file, in bytes.\n";
String lsr = "-lsr : \tRecursively list the contents that match the specified\n" +
"\t\tfile pattern. Behaves very similarly to hadoop fs -ls,\n" +
"\t\texcept that the data is shown for all the entries in the\n" +
"\t\tsubtree.\n";
String df = "-df []: \tShows the capacity, free and used space of the filesystem.\n"+
"\t\tIf the filesystem has multiple partitions, and no path to a particular partition\n"+
"\t\tis specified, then the status of the root partitions will be shown.\n";
String du = "-du [-s] [-h] : \tShow the amount of space, in bytes, used by the files that \n" +
"\t\tmatch the specified file pattern. The following flags are optional:\n" +
"\t\t -s Rather than showing the size of each individual file that\n" +
"\t\t matches the pattern, shows the total (summary) size.\n" +
"\t\t -h Formats the sizes of files in a human-readable fashion\n" +
"\t\t rather than a number of bytes.\n" +
"\n" +
"\t\tNote that, even without the -s option, this only shows size summaries\n" +
"\t\tone level deep into a directory.\n" +
"\t\tThe output is in the form \n" +
"\t\t\tsize\tname(full path)\n";
String dus = "-dus : \tShow the amount of space, in bytes, used by the files that \n" +
"\t\tmatch the specified file pattern. This is equivalent to -du -s above.\n";
String mv = "-mv : Move files that match the specified file pattern \n" +
"\t\tto a destination . When moving multiple files, the \n" +
"\t\tdestination must be a directory. \n";
String cp = "-cp : Copy files that match the file pattern to a \n" +
"\t\tdestination. When copying multiple files, the destination\n" +
"\t\tmust be a directory. \n";
String rm = "-rm [-skipTrash] : \tDelete all files that match the specified file pattern.\n" +
"\t\tEquivalent to the Unix command \"rm \"\n" +
"\t\t-skipTrash option bypasses trash, if enabled, and immediately\n" +
"deletes ";
String rmr = "-rmr [-skipTrash] : \tRemove all directories which match the specified file \n" +
"\t\tpattern. Equivalent to the Unix command \"rm -rf \"\n" +
"\t\t-skipTrash option bypasses trash, if enabled, and immediately\n" +
"deletes ";
String put = "-put ... : \tCopy files " +
"from the local file system \n\t\tinto fs. \n";
String copyFromLocal = "-copyFromLocal ... :" +
" Identical to the -put command.\n";
String moveFromLocal = "-moveFromLocal ... :" +
" Same as -put, except that the source is\n\t\tdeleted after it's copied.\n";
String get = GET_SHORT_USAGE
+ ": Copy files that match the file pattern \n" +
"\t\tto the local name. is kept. When copying mutiple, \n" +
"\t\tfiles, the destination must be a directory. \n";
String getmerge = "-getmerge : Get all the files in the directories that \n" +
"\t\tmatch the source file pattern and merge and sort them to only\n" +
"\t\tone file on local fs. is kept.\n";
String cat = "-cat : \tFetch all files that match the file pattern \n" +
"\t\tand display their content on stdout.\n";
String text = "-text : \tTakes a source file and outputs the file in text format.\n" +
"\t\tThe allowed formats are zip and TextRecordInputStream.\n";
String copyToLocal = COPYTOLOCAL_SHORT_USAGE
+ ": Identical to the -get command.\n";
String moveToLocal = "-moveToLocal : Not implemented yet \n";
String mkdir = "-mkdir : \tCreate a directory in specified location. \n";
String setrep = SETREP_SHORT_USAGE
+ ": Set the replication level of a file. \n"
+ "\t\tThe -R flag requests a recursive change of replication level \n"
+ "\t\tfor an entire tree.\n";
String touchz = "-touchz : Creates a file of zero length\n"
+ "\t\t at with current time as the timestamp of that .\n"
+ "\t\t An error is returned if the file exists with non-zero length\n";
String test = "-test -[ezd] : If file { exists, has zero length, is a directory\n" +
"\t\tthen return 0, else return 1.\n";
String stat = "-stat [format] : Print statistics about the file/directory at \n" +
"\t\tin the specified format. Format accepts filesize in blocks (%b), filename (%n),\n" +
"\t\tblock size (%o), replication (%r), modification date (%y, %Y)\n";
String tail = TAIL_USAGE
+ ": Show the last 1KB of the file. \n"
+ "\t\tThe -f option shows appended data as the file grows. \n";
String chmod = FsShellPermissions.CHMOD_USAGE + "\n" +
"\t\tChanges permissions of a file.\n" +
"\t\tThis works similar to shell's chmod with a few exceptions.\n\n" +
"\t-R\tmodifies the files recursively. This is the only option\n" +
"\t\tcurrently supported.\n\n" +
"\tMODE\tMode is same as mode used for chmod shell command.\n" +
"\t\tOnly letters recognized are 'rwxXt'. E.g. +t,a+r,g-w,+rwx,o=r\n\n" +
"\tOCTALMODE Mode specifed in 3 or 4 digits. If 4 digits, the first may\n" +
"\tbe 1 or 0 to turn the sticky bit on or off, respectively. Unlike " +
"\tshell command, it is not possible to specify only part of the mode\n" +
"\t\tE.g. 754 is same as u=rwx,g=rx,o=r\n\n" +
"\t\tIf none of 'augo' is specified, 'a' is assumed and unlike\n" +
"\t\tshell command, no umask is applied.\n";
String chown = FsShellPermissions.CHOWN_USAGE + "\n" +
"\t\tChanges owner and group of a file.\n" +
"\t\tThis is similar to shell's chown with a few exceptions.\n\n" +
"\t-R\tmodifies the files recursively. This is the only option\n" +
"\t\tcurrently supported.\n\n" +
"\t\tIf only owner or group is specified then only owner or\n" +
"\t\tgroup is modified.\n\n" +
"\t\tThe owner and group names may only cosists of digits, alphabet,\n"+
"\t\tand any of '-_.@/' i.e. [-_.@/a-zA-Z0-9]. The names are case\n" +
"\t\tsensitive.\n\n" +
"\t\tWARNING: Avoid using '.' to separate user name and group though\n" +
"\t\tLinux allows it. If user names have dots in them and you are\n" +
"\t\tusing local file system, you might see surprising results since\n" +
"\t\tshell command 'chown' is used for local files.\n";
String chgrp = FsShellPermissions.CHGRP_USAGE + "\n" +
"\t\tThis is equivalent to -chown ... :GROUP ...\n";
String expunge = "-expunge: Empty the Trash.\n";
String help = "-help [cmd]: \tDisplays help for given command or all commands if none\n" +
"\t\tis specified.\n";
if ("fs".equals(cmd)) {
System.out.println(fs);
} else if ("conf".equals(cmd)) {
System.out.println(conf);
} else if ("D".equals(cmd)) {
System.out.println(D);
} else if ("ls".equals(cmd)) {
System.out.println(ls);
} else if ("lsr".equals(cmd)) {
System.out.println(lsr);
} else if ("df".equals(cmd)) {
System.out.println(df);
} else if ("du".equals(cmd)) {
System.out.println(du);
} else if ("dus".equals(cmd)) {
System.out.println(dus);
} else if ("rm".equals(cmd)) {
System.out.println(rm);
} else if ("expunge".equals(cmd)) {
System.out.println(expunge);
} else if ("rmr".equals(cmd)) {
System.out.println(rmr);
} else if ("mkdir".equals(cmd)) {
System.out.println(mkdir);
} else if ("mv".equals(cmd)) {
System.out.println(mv);
} else if ("cp".equals(cmd)) {
System.out.println(cp);
} else if ("put".equals(cmd)) {
System.out.println(put);
} else if ("copyFromLocal".equals(cmd)) {
System.out.println(copyFromLocal);
} else if ("moveFromLocal".equals(cmd)) {
System.out.println(moveFromLocal);
} else if ("get".equals(cmd)) {
System.out.println(get);
} else if ("getmerge".equals(cmd)) {
System.out.println(getmerge);
} else if ("copyToLocal".equals(cmd)) {
System.out.println(copyToLocal);
} else if ("moveToLocal".equals(cmd)) {
System.out.println(moveToLocal);
} else if ("cat".equals(cmd)) {
System.out.println(cat);
} else if ("get".equals(cmd)) {
System.out.println(get);
} else if ("setrep".equals(cmd)) {
System.out.println(setrep);
} else if ("touchz".equals(cmd)) {
System.out.println(touchz);
} else if ("test".equals(cmd)) {
System.out.println(test);
} else if ("text".equals(cmd)) {
System.out.println(text);
} else if ("stat".equals(cmd)) {
System.out.println(stat);
} else if ("tail".equals(cmd)) {
System.out.println(tail);
} else if ("chmod".equals(cmd)) {
System.out.println(chmod);
} else if ("chown".equals(cmd)) {
System.out.println(chown);
} else if ("chgrp".equals(cmd)) {
System.out.println(chgrp);
} else if (Count.NAME.equals(cmd)) {
System.out.println(Count.DESCRIPTION);
} else if ("help".equals(cmd)) {
System.out.println(help);
} else {
System.out.println(summary);
System.out.println(fs);
System.out.println(ls);
System.out.println(lsr);
System.out.println(df);
System.out.println(du);
System.out.println(dus);
System.out.println(mv);
System.out.println(cp);
System.out.println(rm);
System.out.println(rmr);
System.out.println(put);
System.out.println(copyFromLocal);
System.out.println(moveFromLocal);
System.out.println(get);
System.out.println(getmerge);
System.out.println(cat);
System.out.println(copyToLocal);
System.out.println(moveToLocal);
System.out.println(mkdir);
System.out.println(setrep);
System.out.println(tail);
System.out.println(touchz);
System.out.println(test);
System.out.println(text);
System.out.println(stat);
System.out.println(chmod);
System.out.println(chown);
System.out.println(chgrp);
System.out.println(Count.DESCRIPTION);
System.out.println(help);
}
}
/**
* Apply operation specified by 'cmd' on all parameters
* starting from argv[startindex].
*/
private int doall(String cmd, String argv[], int startindex) {
int exitCode = 0;
int i = startindex;
boolean rmSkipTrash = false;
// Check for -skipTrash option in rm/rmr
if(("-rm".equals(cmd) || "-rmr".equals(cmd))
&& "-skipTrash".equals(argv[i])) {
rmSkipTrash = true;
i++;
}
//
// for each source file, issue the command
//
for (; i < argv.length; i++) {
try {
//
// issue the command to the fs
//
if ("-cat".equals(cmd)) {
cat(argv[i], true);
} else if ("-mkdir".equals(cmd)) {
mkdir(argv[i]);
} else if ("-rm".equals(cmd)) {
delete(argv[i], false, rmSkipTrash);
} else if ("-rmr".equals(cmd)) {
delete(argv[i], true, rmSkipTrash);
} else if ("-df".equals(cmd)) {
df(argv[i]);
} else if (Count.matches(cmd)) {
new Count(argv, i, getConf()).runAll();
} else if ("-ls".equals(cmd)) {
exitCode = ls(argv[i], false);
} else if ("-lsr".equals(cmd)) {
exitCode = ls(argv[i], true);
} else if ("-touchz".equals(cmd)) {
touchz(argv[i]);
} else if ("-text".equals(cmd)) {
text(argv[i]);
}
} catch (RemoteException e) {
//
// This is a error returned by hadoop server. Print
// out the first line of the error message.
//
exitCode = -1;
try {
String[] content;
content = e.getLocalizedMessage().split("\n");
System.err.println(cmd.substring(1) + ": " +
content[0]);
} catch (Exception ex) {
System.err.println(cmd.substring(1) + ": " +
ex.getLocalizedMessage());
}
} catch (IOException e) {
//
// IO exception encountered locally.
//
exitCode = -1;
String content = e.getLocalizedMessage();
if (content != null) {
content = content.split("\n")[0];
}
System.err.println(cmd.substring(1) + ": " +
content);
}
}
return exitCode;
}
/**
* Displays format of commands.
*
*/
private static void printUsage(String cmd) {
String prefix = "Usage: java " + FsShell.class.getSimpleName();
if ("-fs".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [-fs ]");
} else if ("-conf".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [-conf ]");
} else if ("-D".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [-D <[property=value>]");
} else if ("-ls".equals(cmd) || "-lsr".equals(cmd) ||
"-du".equals(cmd) || "-dus".equals(cmd) ||
"-touchz".equals(cmd) || "-mkdir".equals(cmd) ||
"-text".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " ]");
} else if ("-df".equals(cmd) ) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " []]");
} else if (Count.matches(cmd)) {
System.err.println(prefix + " [" + Count.USAGE + "]");
} else if ("-rm".equals(cmd) || "-rmr".equals(cmd)) {
System.err.println("Usage: java FsShell [" + cmd +
" [-skipTrash] ]");
} else if ("-mv".equals(cmd) || "-cp".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " ]");
} else if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd) ||
"-moveFromLocal".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " ... ]");
} else if ("-get".equals(cmd)) {
System.err.println("Usage: java FsShell [" + GET_SHORT_USAGE + "]");
} else if ("-copyToLocal".equals(cmd)) {
System.err.println("Usage: java FsShell [" + COPYTOLOCAL_SHORT_USAGE+ "]");
} else if ("-moveToLocal".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " [-crc] ]");
} else if ("-cat".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [" + cmd + " ]");
} else if ("-setrep".equals(cmd)) {
System.err.println("Usage: java FsShell [" + SETREP_SHORT_USAGE + "]");
} else if ("-test".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [-test -[ezd] ]");
} else if ("-stat".equals(cmd)) {
System.err.println("Usage: java FsShell" +
" [-stat [format] ]");
} else if ("-tail".equals(cmd)) {
System.err.println("Usage: java FsShell [" + TAIL_USAGE + "]");
} else {
System.err.println("Usage: java FsShell");
System.err.println(" [-ls ]");
System.err.println(" [-lsr ]");
System.err.println(" [-df []]");
System.err.println(" [-du [-s] [-h] ]");
System.err.println(" [-dus ]");
System.err.println(" [" + Count.USAGE + "]");
System.err.println(" [-mv ]");
System.err.println(" [-cp ]");
System.err.println(" [-rm [-skipTrash] ]");
System.err.println(" [-rmr [-skipTrash] ]");
System.err.println(" [-expunge]");
System.err.println(" [-put ... ]");
System.err.println(" [-copyFromLocal ... ]");
System.err.println(" [-moveFromLocal ... ]");
System.err.println(" [" + GET_SHORT_USAGE + "]");
System.err.println(" [-getmerge [addnl]]");
System.err.println(" [-cat ]");
System.err.println(" [-text ]");
System.err.println(" [" + COPYTOLOCAL_SHORT_USAGE + "]");
System.err.println(" [-moveToLocal [-crc] ]");
System.err.println(" [-mkdir ]");
System.err.println(" [" + SETREP_SHORT_USAGE + "]");
System.err.println(" [-touchz ]");
System.err.println(" [-test -[ezd] ]");
System.err.println(" [-stat [format] ]");
System.err.println(" [" + TAIL_USAGE + "]");
System.err.println(" [" + FsShellPermissions.CHMOD_USAGE + "]");
System.err.println(" [" + FsShellPermissions.CHOWN_USAGE + "]");
System.err.println(" [" + FsShellPermissions.CHGRP_USAGE + "]");
System.err.println(" [-help [cmd]]");
System.err.println();
ToolRunner.printGenericCommandUsage(System.err);
}
}
/**
* run
*/
public int run(String argv[]) throws Exception {
if (argv.length < 1) {
printUsage("");
return -1;
}
int exitCode = -1;
int i = 0;
String cmd = argv[i++];
//
// verify that we have enough command line parameters
//
if ("-put".equals(cmd) || "-test".equals(cmd) ||
"-copyFromLocal".equals(cmd) || "-moveFromLocal".equals(cmd)) {
if (argv.length < 3) {
printUsage(cmd);
return exitCode;
}
} else if ("-get".equals(cmd) ||
"-copyToLocal".equals(cmd) || "-moveToLocal".equals(cmd)) {
if (argv.length < 3) {
printUsage(cmd);
return exitCode;
}
} else if ("-mv".equals(cmd) || "-cp".equals(cmd)) {
if (argv.length < 3) {
printUsage(cmd);
return exitCode;
}
} else if ("-rm".equals(cmd) || "-rmr".equals(cmd) ||
"-cat".equals(cmd) || "-mkdir".equals(cmd) ||
"-touchz".equals(cmd) || "-stat".equals(cmd) ||
"-text".equals(cmd)) {
if (argv.length < 2) {
printUsage(cmd);
return exitCode;
}
}
// initialize FsShell
try {
init();
} catch (RPC.VersionMismatch v) {
System.err.println("Version Mismatch between client and server" +
"... command aborted.");
return exitCode;
} catch (IOException e) {
System.err.println("Bad connection to FS. Command aborted. Exception: " +
e.getLocalizedMessage());
return exitCode;
}
exitCode = 0;
try {
if ("-put".equals(cmd) || "-copyFromLocal".equals(cmd)) {
Path[] srcs = new Path[argv.length-2];
for (int j=0 ; i < argv.length-1 ;)
srcs[j++] = new Path(argv[i++]);
copyFromLocal(srcs, argv[i++]);
} else if ("-moveFromLocal".equals(cmd)) {
Path[] srcs = new Path[argv.length-2];
for (int j=0 ; i < argv.length-1 ;)
srcs[j++] = new Path(argv[i++]);
moveFromLocal(srcs, argv[i++]);
} else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
copyToLocal(argv, i);
} else if ("-getmerge".equals(cmd)) {
if (argv.length>i+2)
copyMergeToLocal(argv[i++], new Path(argv[i++]), Boolean.parseBoolean(argv[i++]));
else
copyMergeToLocal(argv[i++], new Path(argv[i++]));
} else if ("-cat".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-text".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-moveToLocal".equals(cmd)) {
moveToLocal(argv[i++], new Path(argv[i++]));
} else if ("-setrep".equals(cmd)) {
setReplication(argv, i);
} else if ("-chmod".equals(cmd) ||
"-chown".equals(cmd) ||
"-chgrp".equals(cmd)) {
exitCode = FsShellPermissions.changePermissions(fs, cmd, argv, i, this);
} else if ("-ls".equals(cmd)) {
if (i < argv.length) {
exitCode = doall(cmd, argv, i);
} else {
exitCode = ls(Path.CUR_DIR, false);
}
} else if ("-lsr".equals(cmd)) {
if (i < argv.length) {
exitCode = doall(cmd, argv, i);
} else {
exitCode = ls(Path.CUR_DIR, true);
}
} else if ("-mv".equals(cmd)) {
exitCode = rename(argv, getConf());
} else if ("-cp".equals(cmd)) {
exitCode = copy(argv, getConf());
} else if ("-rm".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-rmr".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-expunge".equals(cmd)) {
expunge();
} else if ("-df".equals(cmd)) {
if (argv.length-1 > 0) {
exitCode = doall(cmd, argv, i);
} else {
df(null);
}
} else if ("-du".equals(cmd)) {
du(argv, i);
} else if ("-dus".equals(cmd)) {
dus(argv, i);
} else if (Count.matches(cmd)) {
exitCode = new Count(argv, i, getConf()).runAll();
} else if ("-mkdir".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-touchz".equals(cmd)) {
exitCode = doall(cmd, argv, i);
} else if ("-test".equals(cmd)) {
exitCode = test(argv, i);
} else if ("-stat".equals(cmd)) {
if (i + 1 < argv.length) {
stat(argv[i++].toCharArray(), argv[i++]);
} else {
stat("%y".toCharArray(), argv[i]);
}
} else if ("-help".equals(cmd)) {
if (i < argv.length) {
printHelp(argv[i]);
} else {
printHelp("");
}
} else if ("-tail".equals(cmd)) {
tail(argv, i);
} else {
exitCode = -1;
System.err.println(cmd.substring(1) + ": Unknown command");
printUsage("");
}
} catch (IllegalArgumentException arge) {
exitCode = -1;
System.err.println(cmd.substring(1) + ": " + arge.getLocalizedMessage());
printUsage(cmd);
} catch (RemoteException e) {
//
// This is a error returned by hadoop server. Print
// out the first line of the error mesage, ignore the stack trace.
exitCode = -1;
try {
String[] content;
content = e.getLocalizedMessage().split("\n");
System.err.println(cmd.substring(1) + ": " +
content[0]);
} catch (Exception ex) {
System.err.println(cmd.substring(1) + ": " +
ex.getLocalizedMessage());
}
} catch (IOException e) {
//
// IO exception encountered locally.
//
exitCode = -1;
System.err.println(cmd.substring(1) + ": " +
e.getLocalizedMessage());
} catch (Exception re) {
exitCode = -1;
System.err.println(cmd.substring(1) + ": " + re.getLocalizedMessage());
} finally {
}
return exitCode;
}
public void close() throws IOException {
if (fs != null) {
fs.close();
fs = null;
}
}
/**
* main() has some simple utility methods
*/
public static void main(String argv[]) throws Exception {
FsShell shell = new FsShell();
int res;
try {
res = ToolRunner.run(shell, argv);
} finally {
shell.close();
}
System.exit(res);
}
/**
* Accumulate exceptions if there is any. Throw them at last.
*/
private abstract class DelayedExceptionThrowing {
abstract void process(Path p, FileSystem srcFs) throws IOException;
final void globAndProcess(Path srcPattern, FileSystem srcFs
) throws IOException {
List exceptions = new ArrayList();
for(Path p : FileUtil.stat2Paths(srcFs.globStatus(srcPattern),
srcPattern))
try { process(p, srcFs); }
catch(IOException ioe) { exceptions.add(ioe); }
if (!exceptions.isEmpty())
if (exceptions.size() == 1)
throw exceptions.get(0);
else
throw new IOException("Multiple IOExceptions: " + exceptions);
}
}
/**
* Utility class for a line of du output
*/
private static class UsagePair {
public String path;
public long bytes;
public UsagePair(String path, long bytes) {
this.path = path;
this.bytes = bytes;
}
}
}