Skip to content

Commit e8e6415

Browse files
szetszwoszetszwo
committed
svn merge -c 1101199 from trunk for HADOOP-7238.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/yahoo-merge@1127280 13f79535-47bb-0310-9956-ffa450edef68
1 parent 344f414 commit e8e6415

File tree

5 files changed

+211
-160
lines changed

5 files changed

+211
-160
lines changed

‎CHANGES.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ Trunk (unreleased changes)
5656
HADOOP-7265. Keep track of relative paths in PathData. (Daryn Sharp
5757
via szetszwo)
5858

59+
HADOOP-7238. Refactor the cat and text commands to conform to new FsCommand
60+
class. (Daryn Sharp via szetszwo)
61+
5962
OPTIMIZATIONS
6063

6164
BUG FIXES

‎src/java/org/apache/hadoop/fs/FsShell.java‎

Lines changed: 14 additions & 159 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@
2828
import java.util.Date;
2929
import java.util.List;
3030
import java.util.TimeZone;
31-
import java.util.zip.GZIPInputStream;
3231

3332
import org.apache.commons.logging.Log;
3433
import org.apache.commons.logging.LogFactory;
@@ -39,17 +38,9 @@
3938
import org.apache.hadoop.fs.shell.CommandFactory;
4039
import org.apache.hadoop.fs.shell.CommandFormat;
4140
import org.apache.hadoop.fs.shell.FsCommand;
42-
import org.apache.hadoop.io.DataInputBuffer;
43-
import org.apache.hadoop.io.DataOutputBuffer;
4441
import org.apache.hadoop.io.IOUtils;
45-
import org.apache.hadoop.io.SequenceFile;
46-
import org.apache.hadoop.io.Writable;
47-
import org.apache.hadoop.io.WritableComparable;
48-
import org.apache.hadoop.io.compress.CompressionCodec;
49-
import org.apache.hadoop.io.compress.CompressionCodecFactory;
5042
import org.apache.hadoop.ipc.RPC;
5143
import org.apache.hadoop.ipc.RemoteException;
52-
import org.apache.hadoop.util.ReflectionUtils;
5344
import org.apache.hadoop.util.StringUtils;
5445
import org.apache.hadoop.util.Tool;
5546
import org.apache.hadoop.util.ToolRunner;
@@ -173,12 +164,12 @@ void moveFromLocal(Path src, String dstf) throws IOException {
173164
* and copy them to the local name. srcf is kept.
174165
* When copying multiple files, the destination must be a directory.
175166
* Otherwise, IOException is thrown.
176-
* @param argv arguments
177-
* @param pos Ignore everything before argv[pos]
178-
* @throws IOException on error
167+
* @param argv : arguments
168+
* @param pos : Ignore everything before argv[pos]
169+
* @throws Exception
179170
* @see org.apache.hadoop.fs.FileSystem.globStatus
180171
*/
181-
void copyToLocal(String[]argv, int pos) throws IOException {
172+
void copyToLocal(String[]argv, int pos) throws Exception {
182173
CommandFormat cf = new CommandFormat("copyToLocal", 2,2,"crc","ignoreCrc");
183174

184175
String srcstr = null;
@@ -199,7 +190,12 @@ void copyToLocal(String[]argv, int pos) throws IOException {
199190
if (copyCrc) {
200191
System.err.println("-crc option is not valid when destination is stdout.");
201192
}
202-
cat(srcstr, verifyChecksum);
193+
194+
List<String> catArgv = new ArrayList<String>();
195+
catArgv.add("-cat");
196+
if (cf.getOpt("ignoreCrc")) catArgv.add("-ignoreCrc");
197+
catArgv.add(srcstr);
198+
run(catArgv.toArray(new String[0]));
203199
} else {
204200
File dst = new File(dststr);
205201
Path srcpath = new Path(srcstr);
@@ -314,115 +310,6 @@ private void copyToLocal(final FileSystem srcFS, final FileStatus srcStatus,
314310
void moveToLocal(String srcf, Path dst) throws IOException {
315311
System.err.println("Option '-moveToLocal' is not implemented yet.");
316312
}
317-
318-
/**
319-
* Fetch all files that match the file pattern <i>srcf</i> and display
320-
* their content on stdout.
321-
* @param srcf a file pattern specifying source files
322-
* @throws IOException on error
323-
* @see org.apache.hadoop.fs.FileSystem.globStatus
324-
*/
325-
void cat(String src, boolean verifyChecksum) throws IOException {
326-
//cat behavior in Linux
327-
// [~/1207]$ ls ?.txt
328-
// x.txt z.txt
329-
// [~/1207]$ cat x.txt y.txt z.txt
330-
// xxx
331-
// cat: y.txt: No such file or directory
332-
// zzz
333-
334-
Path srcPattern = new Path(src);
335-
new DelayedExceptionThrowing() {
336-
@Override
337-
void process(Path p, FileSystem srcFs) throws IOException {
338-
printToStdout(srcFs.open(p));
339-
}
340-
}.globAndProcess(srcPattern, getSrcFileSystem(srcPattern, verifyChecksum));
341-
}
342-
343-
private class TextRecordInputStream extends InputStream {
344-
SequenceFile.Reader r;
345-
WritableComparable<?> key;
346-
Writable val;
347-
348-
DataInputBuffer inbuf;
349-
DataOutputBuffer outbuf;
350-
351-
public TextRecordInputStream(FileStatus f) throws IOException {
352-
final Path fpath = f.getPath();
353-
final Configuration lconf = getConf();
354-
r = new SequenceFile.Reader(lconf,
355-
SequenceFile.Reader.file(fpath));
356-
key = ReflectionUtils.newInstance(
357-
r.getKeyClass().asSubclass(WritableComparable.class), lconf);
358-
val = ReflectionUtils.newInstance(
359-
r.getValueClass().asSubclass(Writable.class), lconf);
360-
inbuf = new DataInputBuffer();
361-
outbuf = new DataOutputBuffer();
362-
}
363-
364-
public int read() throws IOException {
365-
int ret;
366-
if (null == inbuf || -1 == (ret = inbuf.read())) {
367-
if (!r.next(key, val)) {
368-
return -1;
369-
}
370-
byte[] tmp = key.toString().getBytes();
371-
outbuf.write(tmp, 0, tmp.length);
372-
outbuf.write('\t');
373-
tmp = val.toString().getBytes();
374-
outbuf.write(tmp, 0, tmp.length);
375-
outbuf.write('\n');
376-
inbuf.reset(outbuf.getData(), outbuf.getLength());
377-
outbuf.reset();
378-
ret = inbuf.read();
379-
}
380-
return ret;
381-
}
382-
383-
public void close() throws IOException {
384-
r.close();
385-
super.close();
386-
}
387-
}
388-
389-
private InputStream forMagic(Path p, FileSystem srcFs) throws IOException {
390-
FSDataInputStream i = srcFs.open(p);
391-
392-
// check codecs
393-
CompressionCodecFactory cf = new CompressionCodecFactory(getConf());
394-
CompressionCodec codec = cf.getCodec(p);
395-
if (codec != null) {
396-
return codec.createInputStream(i);
397-
}
398-
399-
switch(i.readShort()) {
400-
case 0x1f8b: // RFC 1952
401-
i.seek(0);
402-
return new GZIPInputStream(i);
403-
case 0x5345: // 'S' 'E'
404-
if (i.readByte() == 'Q') {
405-
i.close();
406-
return new TextRecordInputStream(srcFs.getFileStatus(p));
407-
}
408-
break;
409-
}
410-
i.seek(0);
411-
return i;
412-
}
413-
414-
void text(String srcf) throws IOException {
415-
Path srcPattern = new Path(srcf);
416-
new DelayedExceptionThrowing() {
417-
@Override
418-
void process(Path p, FileSystem srcFs) throws IOException {
419-
if (srcFs.isDirectory(p)) {
420-
throw new IOException("Source must be a file.");
421-
}
422-
printToStdout(forMagic(p, srcFs));
423-
}
424-
}.globAndProcess(srcPattern, srcPattern.getFileSystem(getConf()));
425-
}
426313

427314
/**
428315
* Show the size of a partition in the filesystem that contains
@@ -951,11 +838,9 @@ private void printHelp(String cmd) {
951838
"[-rmr [-skipTrash] <src>] [-put <localsrc> ... <dst>] [-copyFromLocal <localsrc> ... <dst>]\n\t" +
952839
"[-moveFromLocal <localsrc> ... <dst>] [" +
953840
GET_SHORT_USAGE + "\n\t" +
954-
"[-cat <src>]\n\t" +
955841
"[" + COPYTOLOCAL_SHORT_USAGE + "] [-moveToLocal <src> <localdst>]\n\t" +
956842
"[-report]\n\t" +
957-
"[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]\n\t" +
958-
"[-text <path>]";
843+
"[-touchz <path>] [-test -[ezd] <path>] [-stat [format] <path>]";
959844

960845
String conf ="-conf <configuration file>: Specify an application configuration file.";
961846

@@ -1023,14 +908,6 @@ private void printHelp(String cmd) {
1023908
"\t\tto the local name. <src> is kept. When copying mutiple, \n" +
1024909
"\t\tfiles, the destination must be a directory. \n";
1025910

1026-
String cat = "-cat <src>: \tFetch all files that match the file pattern <src> \n" +
1027-
"\t\tand display their content on stdout.\n";
1028-
1029-
1030-
String text = "-text <src>: \tTakes a source file and outputs the file in text format.\n" +
1031-
"\t\tThe allowed formats are zip and TextRecordInputStream.\n";
1032-
1033-
1034911
String copyToLocal = COPYTOLOCAL_SHORT_USAGE
1035912
+ ": Identical to the -get command.\n";
1036913

@@ -1089,16 +966,12 @@ private void printHelp(String cmd) {
1089966
System.out.println(copyToLocal);
1090967
} else if ("moveToLocal".equals(cmd)) {
1091968
System.out.println(moveToLocal);
1092-
} else if ("cat".equals(cmd)) {
1093-
System.out.println(cat);
1094969
} else if ("get".equals(cmd)) {
1095970
System.out.println(get);
1096971
} else if ("touchz".equals(cmd)) {
1097972
System.out.println(touchz);
1098973
} else if ("test".equals(cmd)) {
1099974
System.out.println(test);
1100-
} else if ("text".equals(cmd)) {
1101-
System.out.println(text);
1102975
} else if ("stat".equals(cmd)) {
1103976
System.out.println(stat);
1104977
} else if ("help".equals(cmd)) {
@@ -1123,12 +996,10 @@ private void printHelp(String cmd) {
1123996
System.out.println(copyFromLocal);
1124997
System.out.println(moveFromLocal);
1125998
System.out.println(get);
1126-
System.out.println(cat);
1127999
System.out.println(copyToLocal);
11281000
System.out.println(moveToLocal);
11291001
System.out.println(touchz);
11301002
System.out.println(test);
1131-
System.out.println(text);
11321003
System.out.println(stat);
11331004

11341005
for (String thisCmdName : commandFactory.getNames()) {
@@ -1179,18 +1050,14 @@ private int doall(String cmd, String argv[], int startindex) {
11791050
//
11801051
// issue the command to the fs
11811052
//
1182-
if ("-cat".equals(cmd)) {
1183-
cat(argv[i], true);
1184-
} else if ("-rm".equals(cmd)) {
1053+
if ("-rm".equals(cmd)) {
11851054
delete(argv[i], false, rmSkipTrash);
11861055
} else if ("-rmr".equals(cmd)) {
11871056
delete(argv[i], true, rmSkipTrash);
11881057
} else if ("-df".equals(cmd)) {
11891058
df(argv[i]);
11901059
} else if ("-touchz".equals(cmd)) {
11911060
touchz(argv[i]);
1192-
} else if ("-text".equals(cmd)) {
1193-
text(argv[i]);
11941061
}
11951062
} catch (RemoteException e) {
11961063
LOG.debug("Error", e);
@@ -1245,8 +1112,7 @@ private void printUsage(String cmd) {
12451112
System.err.println("Usage: java FsShell" +
12461113
" [-D <[property=value>]");
12471114
} else if ("-du".equals(cmd) || "-dus".equals(cmd) ||
1248-
"-touchz".equals(cmd) ||
1249-
"-text".equals(cmd)) {
1115+
"-touchz".equals(cmd)) {
12501116
System.err.println("Usage: java FsShell" +
12511117
" [" + cmd + " <path>]");
12521118
} else if ("-df".equals(cmd) ) {
@@ -1269,9 +1135,6 @@ private void printUsage(String cmd) {
12691135
} else if ("-moveToLocal".equals(cmd)) {
12701136
System.err.println("Usage: java FsShell" +
12711137
" [" + cmd + " [-crc] <src> <localdst>]");
1272-
} else if ("-cat".equals(cmd)) {
1273-
System.err.println("Usage: java FsShell" +
1274-
" [" + cmd + " <src>]");
12751138
} else if ("-test".equals(cmd)) {
12761139
System.err.println("Usage: java FsShell" +
12771140
" [-test -[ezd] <path>]");
@@ -1292,8 +1155,6 @@ private void printUsage(String cmd) {
12921155
System.err.println(" [-copyFromLocal <localsrc> ... <dst>]");
12931156
System.err.println(" [-moveFromLocal <localsrc> ... <dst>]");
12941157
System.err.println(" [" + GET_SHORT_USAGE + "]");
1295-
System.err.println(" [-cat <src>]");
1296-
System.err.println(" [-text <src>]");
12971158
System.err.println(" [" + COPYTOLOCAL_SHORT_USAGE + "]");
12981159
System.err.println(" [-moveToLocal [-crc] <src> <localdst>]");
12991160
System.err.println(" [-touchz <path>]");
@@ -1348,9 +1209,7 @@ public int run(String argv[]) throws Exception {
13481209
return exitCode;
13491210
}
13501211
} else if ("-rm".equals(cmd) || "-rmr".equals(cmd) ||
1351-
"-cat".equals(cmd) ||
1352-
"-touchz".equals(cmd) || "-stat".equals(cmd) ||
1353-
"-text".equals(cmd)) {
1212+
"-touchz".equals(cmd) || "-stat".equals(cmd)) {
13541213
if (argv.length < 2) {
13551214
printUsage(cmd);
13561215
return exitCode;
@@ -1397,10 +1256,6 @@ public int run(String argv[]) throws Exception {
13971256
moveFromLocal(srcs, argv[i++]);
13981257
} else if ("-get".equals(cmd) || "-copyToLocal".equals(cmd)) {
13991258
copyToLocal(argv, i);
1400-
} else if ("-cat".equals(cmd)) {
1401-
exitCode = doall(cmd, argv, i);
1402-
} else if ("-text".equals(cmd)) {
1403-
exitCode = doall(cmd, argv, i);
14041259
} else if ("-moveToLocal".equals(cmd)) {
14051260
moveToLocal(argv[i++], new Path(argv[i++]));
14061261
} else if ("-mv".equals(cmd)) {

0 commit comments

Comments
 (0)