DeviceCount MR Trace

Device Count MR Program Example:
================================
mrinmoy@ubuntu:~$ su root
Password:
root@ubuntu:/home/mrinmoy# ifconfig
eth3 Link encap:Ethernet HWaddr 00:0c:29:c4:74:1e
inet addr:192.168.37.156 Bcast:192.168.37.255 Mask:255.255.255.0
inet6 addr: fe80::20c:29ff:fec4:741e/64 Scope:Link
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:36 errors:0 dropped:0 overruns:0 frame:0
TX packets:137 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:4287 (4.2 KB) TX bytes:18602 (18.6 KB)
Interrupt:19 Base address:0x2000
lo Link encap:Local Loopback

inet addr:127.0.0.1 Mask:255.0.0.0
inet6 addr: ::1/128 Scope:Host
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:639 errors:0 dropped:0 overruns:0 frame:0
TX packets:639 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:125703 (125.7 KB) TX bytes:125703 (125.7 KB)
root@ubuntu:/home/mrinmoy# cd ..
root@ubuntu:/home# cd Gopal/
root@ubuntu:/home/Gopal# ll
total 1728
drwxr-xr-x 18 root root 4096 2013-05-15 09:59 ./
drwxr-xr-x 19 root root 4096 2013-05-07 09:20 ../
drwxr-xr-x 2 root root 4096 2013-03-16 22:02 Cap/
drwxr-xr-x 2 root root 4096 2013-03-04 06:14 CapGem/
-rw-r--r-- 1 root root 7545 2012-12-30 07:35 dept.java
-rw-r--r-- 1 root root 354 2013-05-03 09:13 derby.log
drwxr-xr-x 3 root root 4096 2013-05-15 10:03 Eclipse/
-rw-r--r-- 1 root root 8589 2012-12-30 06:49 emp.java
-rw-r--r-- 1 root root 8636 2012-12-30 07:17 employee.java
drwxr-xr-x 4 root root 4096 2013-02-23 20:53 eveningbatch/
-rw-r--r-- 1 root root 241 2013-03-02 07:38 exportData.txt
-rw-r--r-- 1 root root 8986 2013-03-02 07:42 exportErrorTab.java
-rw-r--r-- 1 root root 8669 2013-05-03 08:25 exportTable.java
-rw-r--r-- 1 root root 66 2012-12-14 06:39 first.txt
drwxrwxrwx 2 root root 4096 2012-12-11 04:59 HBase-Related/
drwxrwxrwx 5 root root 4096 2013-05-08 09:46 HIVE-Related/
-rwxr--r-- 1 root root 3318 2012-11-18 03:55 HIVE.zip*
-rw-r--r-- 1 root root 195 2013-04-16 06:49 input.txt
-rw-r--r-- 1 root root 228 2013-05-05 06:14 logData.txt
drwxr-xr-x 3 root root 4096 2013-01-27 07:34 MapReduce/
-rwxr--r-- 1 root root 786484 2013-02-05 07:47 mysql-connector-java-5.1.16-
bin.jar*
-rwxr--r-- 1 root root 786484 2012-11-27 10:02 mysql-connector-java-5.1.16-
binNew.jar*
drwxr-xr-x 2 root root 4096 2013-02-26 06:17 naresh/
drwxr-xr-x 3 root root 4096 2013-02-26 06:25 naresh2/
-rw-r--r-- 1 root root 172 2013-03-02 07:45 NewExportData.txt
drwxr-xr-x 4 root root 4096 2013-02-23 07:00 Online/
drwxr-xr-x 3 root root 4096 2013-02-23 03:40 Personal/
drwxrwxrwx 2 root root 4096 2013-02-21 09:25 PersonalPrac/
-rw-r--r-- 1 root root 2807 2013-02-19 08:16 pig_1361290465615.log
drwxr-xr-x 33 root root 4096 2013-04-27 20:50 PIG-Latin/
-rw-r--r-- 1 root root 7572 2013-03-02 06:55 QueryResult.java
drwxr-xr-x 2 root root 4096 2012-12-14 06:39 RK/
drwxr-xr-x 2 root root 4096 2013-03-02 19:23 samia/
-rw-r--r-- 1 root root 0 2013-02-27 05:23 sample
-rw-r--r-- 1 root root 113 2012-12-30 06:36 sample.pig
lrwxrwxrwx 1 root root 14 2013-05-15 09:39 Sqoop -> Sqoop-Related//
drwxr-xr-x 3 root root 4096 2013-04-27 22:05 Sqoop-Related/
-rw-r--r-- 1 root root 55 2013-05-03 09:20 tcsmanagedInput.txt
-rw-r--r-- 1 root root 29 2012-12-14 05:24 third.txt
-rw-r--r-- 1 root root 125 2012-12-08 06:45 today.txt
root@ubuntu:/home/Gopal# mkdir Cisco
root@ubuntu:/home/Gopal# cd Cisco/
root@ubuntu:/home/Gopal/Cisco# nano RouterCount.txt
root@ubuntu:/home/Gopal/Cisco# nano SwithcesCount.log
root@ubuntu:/home/Gopal/Cisco# nano HubCenter.log
root@ubuntu:/home/Gopal/Cisco# hadoop fs -mkdir /DeviceCountHDFS
root@ubuntu:/home/Gopal/Cisco# hadoop fs -put RouterCount.txt /DeviceCountHDFS
root@ubuntu:/home/Gopal/Cisco# hadoop fs -put SwithcesCount.log /DeviceCountHDFS
root@ubuntu:/home/Gopal/Cisco# hadoop fs -put HubCenter.log
/DeviceCountHDFSroot@ubuntu:/home/Gopal/Cisco# hadoop fs -ls /DeviceCountHDFS
Found 3 items
-rw-r--r-- 1 root supergroup 211 2013-05-24 07:40
/DeviceCountHDFS/HubCenter.log
/DeviceCountHDFS/RouterCount.txt
/DeviceCountHDFS/SwithcesCount.log
root@ubuntu:/home/Gopal/Cisco# cp /home/mrinmoy/Downloads/MapReduce/DeviceCount.jar
.
root@ubuntu:/home/Gopal/Cisco# ll
total 28
-rwxr--r-- 1 root root 4163 2013-05-24 07:41 DeviceCount.jar*
-rw-r--r-- 1 root root 211 2013-05-24 07:39 HubCenter.log
-rw-r--r-- 1 root root 60 2013-05-24 07:36 RouterCount.txt
-rw-r--r-- 1 root root 75 2013-05-24 07:37 SwithcesCount.log
root@ubuntu:/home/Gopal/Cisco# hadoop fs jar DeviceCount.jar
com/gopal/mapred/DynamicFileReadMR /DeviceCountHDFS/* /DecountMROutput
ar: Unknown command
Usage: java FsShell
[-ls <path>]
[-lsr <path>]
[-df [<path>]]
[-du <path>]
[-dus <path>]
[-count[-q] <path>]
[-mv <src> <dst>]
[-cp <src> <dst>]
[-rm [-skipTrash] <path>]
[-rmr [-skipTrash] <path>]
[-expunge]
[-put <localsrc> ... <dst>]
[-copyFromLocal <localsrc> ... <dst>]
[-moveFromLocal <localsrc> ... <dst>]
[-get [-ignoreCrc] [-crc] <src> <localdst>]
[-getmerge <src> <localdst> [addnl]]
[-cat <src>]
[-text <src>]
[-copyToLocal [-ignoreCrc] [-crc] <src> <localdst>]
[-moveToLocal [-crc] <src> <localdst>]
[-mkdir <path>]
[-setrep [-R] [-w] <rep> <path/file>]
[-touchz <path>]
[-test -[ezd] <path>]
[-stat [format] <path>]
[-tail [-f] <file>]
[-chmod [-R] <MODE[,MODE]... | OCTALMODE> PATH...]
[-chown [-R] [OWNER][:[GROUP]] PATH...]
[-chgrp [-R] GROUP PATH...]
[-help [cmd]]
Generic options supported are

-conf <configuration file> specify an application configuration file
-D <property=value> use value for given property
-fs <local|namenode:port> specify a namenode
-jt <local|jobtracker:port> specify a job tracker
-files <comma separated list of files> specify comma separated files to be
copied to the map reduce cluster
-libjars <comma separated list of jars> specify comma separated jar files to
include in the classpath.
-archives <comma separated list of archives> specify comma separated archives to
be unarchived on the compute machines.
The general command line syntax is

bin/hadoop command [genericOptions] [commandOptions]
root@ubuntu:/home/Gopal/Cisco# hadoop jar DeviceCount.jar

com/gopal/mapred/DynamicFileReadMR /DeviceCountHDFS/* /DecountMROutput
13/05/24 07:44:26 WARN mapred.JobClient: Use GenericOptionsParser for parsing the
arguments. Applications should implement Tool for the same.
13/05/24 07:44:26 WARN snappy.LoadSnappy: Snappy native library is available
13/05/24 07:44:26 INFO util.NativeCodeLoader: Loaded the native-hadoop library
13/05/24 07:44:26 INFO snappy.LoadSnappy: Snappy native library loaded
13/05/24 07:44:27 INFO mapred.FileInputFormat: Total input paths to process : 3
13/05/24 07:44:28 INFO mapred.JobClient: Running job: job_201305240728_0001
13/05/24 07:44:29 INFO mapred.JobClient: map 0% reduce 0%
13/05/24 07:45:14 INFO mapred.JobClient: Task Id :
attempt_201305240728_0001_m_000003_0, Status : FAILED
java.lang.NumberFormatException: For input string: ""
at
java.lang.NumberFormatException.forInputString(NumberFormatException.java:48)
at java.lang.Integer.parseInt(Integer.java:470)
at
com.gopal.mapred.DynamicFileReadMR$MapperClass.map(DynamicFileReadMR.java:34)
at
at org.apache.hadoop.mapred.MapRunner.run(MapRunner.java:50)
at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:391)
at org.apache.hadoop.mapred.MapTask.run(MapTask.java:325)
at org.apache.hadoop.mapred.Child$4.run(Child.java:266)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:396)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1278
)
at org.apache.hadoop.mapred.Child.main(Child.java:260)

at
at
at
at
)

at
at
at
at
)
13/05/24 07:45:38 INFO mapred.JobClient: Job complete: job_201305240728_0001

13/05/24 07:45:38 INFO mapred.JobClient: Counters: 22
13/05/24 07:45:38 INFO mapred.JobClient: Job Counters
13/05/24 07:45:38 INFO mapred.JobClient: Launched reduce tasks=1
13/05/24 07:45:38 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=98718
13/05/24 07:45:38 INFO mapred.JobClient: Total time spent by all reduces
waiting after reserving slots (ms)=0
13/05/24 07:45:38 INFO mapred.JobClient: Total time spent by all maps waiting
after reserving slots (ms)=0
13/05/24 07:45:38 INFO mapred.JobClient: Launched map tasks=7
13/05/24 07:45:38 INFO mapred.JobClient: Data-local map tasks=7
13/05/24 07:45:38 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=36221
13/05/24 07:45:38 INFO mapred.JobClient: Failed map tasks=1
13/05/24 07:45:38 INFO mapred.JobClient: FileSystemCounters
13/05/24 07:45:38 INFO mapred.JobClient: HDFS_BYTES_READ=664
13/05/24 07:45:38 INFO mapred.JobClient: FILE_BYTES_WRITTEN=178573
13/05/24 07:45:38 INFO mapred.JobClient: Map-Reduce Framework
13/05/24 07:45:38 INFO mapred.JobClient: Combine output records=0
13/05/24 07:45:38 INFO mapred.JobClient: Map input records=15
13/05/24 07:45:38 INFO mapred.JobClient: Physical memory (bytes)
snapshot=682459136
13/05/24 07:45:38 INFO mapred.JobClient: Spilled Records=15
13/05/24 07:45:38 INFO mapred.JobClient: Map output bytes=298
13/05/24 07:45:38 INFO mapred.JobClient: CPU time spent (ms)=31280
13/05/24 07:45:38 INFO mapred.JobClient: Total committed heap usage
(bytes)=458489856
13/05/24 07:45:38 INFO mapred.JobClient: Virtual memory (bytes)
snapshot=1371619328
13/05/24 07:45:38 INFO mapred.JobClient: Map input bytes=315
13/05/24 07:45:38 INFO mapred.JobClient: Combine input records=0
13/05/24 07:45:38 INFO mapred.JobClient: Map output records=15
13/05/24 07:45:38 INFO mapred.JobClient: SPLIT_RAW_BYTES=318
13/05/24 07:45:38 INFO mapred.JobClient: Job Failed: NA
Exception in thread "main" java.io.IOException: Job failed!
at org.apache.hadoop.mapred.JobClient.runJob(JobClient.java:1301)
at com.gopal.mapred.DynamicFileReadMR.main(DynamicFileReadMR.java:80)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:2
5)
at java.lang.reflect.Method.invoke(Method.java:597)
at org.apache.hadoop.util.RunJar.main(RunJar.java:197)
root@ubuntu:/home/Gopal/Cisco# hadoop fs -cat /GopalNew/file1.txt
cat: File does not exist: /GopalNew/file1.txt
root@ubuntu:/home/Gopal/Cisco# hadoop fs -cat /GopalNewOne/file1.txt
10 12 13 14 1 5
root@ubuntu:/home/Gopal/Cisco# ll
total 28
-rwxr--r-- 1 root root 4163 2013-05-24 07:41 DeviceCount.jar*
-rw-r--r-- 1 root root 211 2013-05-24 07:39 HubCenter.log
-rw-r--r-- 1 root root 60 2013-05-24 07:36 RouterCount.txt
-rw-r--r-- 1 root root 75 2013-05-24 07:37 SwithcesCount.log
root@ubuntu:/home/Gopal/Cisco# cat HubCenter.log
20 20 20 20 20 20 20 20 20 20
20 20 20 20 20 20 20 20 20 20
20 20 20 20 20 20 20 20 20 20
20 20 20 20 20 20 20 20 20 20
20 20 20 20 20 20 20 20 20 20
20 20 20 20 20 20 20 20 20 20
20 20 20 20 20 20 20 20 20 20
root@ubuntu:/home/Gopal/Cisco# cat RouterCount.txt

10 10 10 10 10
20 20 20 20 20
30 30 30 30 30
40 40 40 40 40
root@ubuntu:/home/Gopal/Cisco# cat SwithcesCount.log
30 30 30 30 30
30 30 30 30 30
30 30 30 30 30
30 30 30 30 30
30 30 30 30 30
root@ubuntu:/home/Gopal/Cisco# hadoop fs -rm /DeviceCountHDFS/HubCenter.log
Deleted hdfs://localhost:8020/DeviceCountHDFS/HubCenter.log
root@ubuntu:/home/Gopal/Cisco# hadoop jar DeviceCount.jar
com/gopal/mapred/DynamicFileReadMR /DeviceCountHDFS/* /DeviceCountMROutput
13/05/24 07:51:01 WARN mapred.JobClient: Use GenericOptionsParser for parsing the
arguments. Applications should implement Tool for the same.
13/05/24 07:51:01 WARN snappy.LoadSnappy: Snappy native library is available
13/05/24 07:51:01 INFO util.NativeCodeLoader: Loaded the native-hadoop library
13/05/24 07:51:01 INFO snappy.LoadSnappy: Snappy native library loaded
13/05/24 07:51:01 INFO mapred.FileInputFormat: Total input paths to process : 2
13/05/24 07:51:02 INFO mapred.JobClient: Running job: job_201305240728_0002
13/05/24 07:51:31 INFO mapred.JobClient: Job complete: job_201305240728_0002
13/05/24 07:51:31 INFO mapred.JobClient: Counters: 27
13/05/24 07:51:31 INFO mapred.JobClient: Job Counters
13/05/24 07:51:31 INFO mapred.JobClient: Launched reduce tasks=1
13/05/24 07:51:31 INFO mapred.JobClient: SLOTS_MILLIS_MAPS=29615
13/05/24 07:51:31 INFO mapred.JobClient: Total time spent by all reduces
waiting after reserving slots (ms)=0
13/05/24 07:51:31 INFO mapred.JobClient: Total time spent by all maps waiting
after reserving slots (ms)=0
13/05/24 07:51:31 INFO mapred.JobClient: Launched map tasks=3
13/05/24 07:51:31 INFO mapred.JobClient: Data-local map tasks=3
13/05/24 07:51:31 INFO mapred.JobClient: SLOTS_MILLIS_REDUCES=12549
13/05/24 07:51:31 INFO mapred.JobClient: FileSystemCounters
13/05/24 07:51:31 INFO mapred.JobClient: FILE_BYTES_READ=214
13/05/24 07:51:31 INFO mapred.JobClient: HDFS_BYTES_READ=466
13/05/24 07:51:31 INFO mapred.JobClient: FILE_BYTES_WRITTEN=237962
13/05/24 07:51:31 INFO mapred.JobClient: HDFS_BYTES_WRITTEN=42
13/05/24 07:51:31 INFO mapred.JobClient: Map-Reduce Framework
13/05/24 07:51:31 INFO mapred.JobClient: Map input records=9
13/05/24 07:51:31 INFO mapred.JobClient: Reduce shuffle bytes=226
13/05/24 07:51:31 INFO mapred.JobClient: Spilled Records=18
13/05/24 07:51:31 INFO mapred.JobClient: Map output bytes=190
13/05/24 07:51:31 INFO mapred.JobClient: CPU time spent (ms)=8940
13/05/24 07:51:31 INFO mapred.JobClient: Total committed heap usage
(bytes)=502988800
13/05/24 07:51:31 INFO mapred.JobClient: Map input bytes=135
13/05/24 07:51:31 INFO mapred.JobClient: Combine input records=0
13/05/24 07:51:31 INFO mapred.JobClient: SPLIT_RAW_BYTES=322
13/05/24 07:51:31 INFO mapred.JobClient: Reduce input records=9
13/05/24 07:51:31 INFO mapred.JobClient: Reduce input groups=2
13/05/24 07:51:31 INFO mapred.JobClient: Combine output records=0
13/05/24 07:51:31 INFO mapred.JobClient: Physical memory (bytes)
snapshot=816041984
13/05/24 07:51:31 INFO mapred.JobClient: Reduce output records=2
13/05/24 07:51:31 INFO mapred.JobClient: Virtual memory (bytes)
snapshot=1821929472
13/05/24 07:51:31 INFO mapred.JobClient: Map output records=9
root@ubuntu:/home/Gopal/Cisco#
root@ubuntu:/home/Gopal/Cisco# hadoop fs -ls /DeviceCountMROutput
Found 3 items
/DeviceCountMROutput/_SUCCESS
drwxr-xr-x - root supergroup 0 2013-05-24 07:51
/DeviceCountMROutput/_logs
/DeviceCountMROutput/part-00000
root@ubuntu:/home/Gopal/Cisco# hadoop fs -cat /DeviceCountMROutput/part-00000
RouterCount.txt 500
SwithcesCount.log 750
root@ubuntu:/home/Gopal/Cisco#

DeviceCount MR Trace

Transféré par

Informations du document

Copyright

Formats disponibles

Partager ce document

Partager ou intégrer le document

Options de partage

Avez-vous trouvé ce document utile ?

Ce contenu est-il inapproprié ?

Droits d'auteur :

Formats disponibles

DeviceCount MR Trace

Transféré par

Droits d'auteur :

Formats disponibles

Device Count MR Program Example:

lo Link encap:Local Loopback

Generic options supported are

The general command line syntax is

root@ubuntu:/home/Gopal/Cisco# hadoop jar DeviceCount.jar

13/05/24 07:45:20 INFO mapred.JobClient: map 75% reduce 25%

13/05/24 07:45:28 INFO mapred.JobClient: Task Id :

13/05/24 07:45:38 INFO mapred.JobClient: Job complete: job_201305240728_0001

root@ubuntu:/home/Gopal/Cisco# cat RouterCount.txt

Vous aimerez peut-être aussi