thanoojgithub · March 21, 2022 05:14
diff --git a/Hadoop3.2.2_Start-up_commands.txt b/Hadoop3.2.2_Start-up_commands.txt
 Hadoop 3.2.2
 Start-up commands:
 --------------------------------
 1. Stop the dfs and yarn first.
 2. Remove the datanode and namenode directories as specified in the core-site.xml file.
 3. Re-create the directories.
 4. hdfs namenode -format
 5. Then re-start the dfs and the yarn as follows.
 start-dfs.sh
 start-yarn.sh
 6. mapred --daemon start historyserver




 - Check SSH servicee is up and running :
 huser@thanoojWin10Home:~/tmpdata$ sudo service ssh status
 [sudo] password for huser:
 * sshd is running
 huser@thanoojWin10Home:~/tmpdata$

 - If not restart
 thanooj@thanoojWin10Home:~$ sudo service ssh restart
 [sudo] password for thanooj:
 * Restarting OpenBSD Secure Shell server sshd                                                                                               [ OK ]
 thanooj@thanoojWin10Home:~$ su - huser
 Password:
 huser@thanoojWin10Home:~$


 huser@thanoojWin10Home:~$ ls -ltr
 -rw-rw-r--  1 huser huser 395448622 Jan 14  2021 hadoop-3.2.2.tar.gz
 drwxr-xr-x 10 huser huser      4096 Mar 20 15:34 hadoop-3.2.2
 drwxrwxr-x  4 huser huser      4096 Mar 21 09:47 dfsdata
 drwxrwxr-x  4 huser huser      4096 Mar 21 09:48 tmpdata
 huser@thanoojWin10Home:~/dfsdata$ ls -ltr
 drwxrwxr-x 2 huser huser 4096 Mar 21 09:47 namenode
 drwx------ 3 huser huser 4096 Mar 21 09:56 datanode
 huser@thanoojWin10Home:~/tmpdata$ ls -ltr
 drwxrwxr-x 4 huser huser 4096 Mar 21 09:55 dfs
 drwxr-xr-x 5 huser huser 4096 Mar 21 09:56 nm-local-dir
 huser@thanoojWin10Home:~/tmpdata$

 ref:
 https://hadoop.apache.org/docs/r3.1.1/hadoop-project-dist/hadoop-common/ClusterSetup.html











 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce$ mkdir wordcountSample
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce$ cd wordcountSample/
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi WordCount.java
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat WordCount.java
 import java.io.IOException;
 import java.util.StringTokenizer;

 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.hadoop.mapreduce.Reducer;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

 public class WordCount {

  public static class TokenizerMapper
       extends Mapper<Object, Text, Text, IntWritable>{

    private final static IntWritable one = new IntWritable(1);
    private Text word = new Text();

    public void map(Object key, Text value, Context context
                    ) throws IOException, InterruptedException {
      StringTokenizer itr = new StringTokenizer(value.toString());
      while (itr.hasMoreTokens()) {
        word.set(itr.nextToken());
        context.write(word, one);
      }
    }
  }

  public static class IntSumReducer
       extends Reducer<Text,IntWritable,Text,IntWritable> {
    private IntWritable result = new IntWritable();

    public void reduce(Text key, Iterable<IntWritable> values,
                       Context context
                       ) throws IOException, InterruptedException {
      int sum = 0;
      for (IntWritable val : values) {
        sum += val.get();
      }
      result.set(sum);
      context.write(key, result);
    }
  }

  public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "word count");
    job.setJarByClass(WordCount.class);
    job.setMapperClass(TokenizerMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

 Hello World Bye World
    FileInputFormat.addInputPath(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    System.exit(job.waitForCompletion(true) ? 0 : 1);

 Hello Hadoop Goodbye Hadoop
  }
 }
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
 total 4
 -rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ export HADOOP_CLASSPATH=${JAVA_HOME}/lib/tools.jar
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hadoop com.sun.tools.javac.Main WordCount.java
 /home/huser/hadoop-3.2.2/libexec/hadoop-functions.sh: line 2366: HADOOP_COM.SUN.TOOLS.JAVAC.MAIN_USER: invalid variable name
 /home/huser/hadoop-3.2.2/libexec/hadoop-functions.sh: line 2461: HADOOP_COM.SUN.TOOLS.JAVAC.MAIN_OPTS: invalid variable name
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ jar cf wc.jar WordCount*.class
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
 total 20
 -rw-rw-r-- 1 huser huser 2089 Mar 21 10:21  WordCount.java
 -rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
 -rw-rw-r-- 1 huser huser 1511 Mar 21 10:22  WordCount.class
 -rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
 -rw-rw-r-- 1 huser huser 3092 Mar 21 10:22  wc.jar
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /
 Found 1 items
 drwxrwx---   - huser supergroup          0 2022-03-21 10:07 /tmp
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one/input
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one/output
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/
 Found 2 items
 drwxr-xr-x   - huser supergroup          0 2022-03-21 10:24 /huser/data/wc/one/input
 drwxr-xr-x   - huser supergroup          0 2022-03-21 10:24 /huser/data/wc/one/output
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi file01.txt
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat file01.txt
 Hello World Bye World
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi file02.txt
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat file02.txt
 Hello Hadoop Goodbye Hadoop
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
 total 28
 -rw-rw-r-- 1 huser huser 2089 Mar 21 10:21  WordCount.java
 -rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
 -rw-rw-r-- 1 huser huser 1511 Mar 21 10:22  WordCount.class
 -rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
 -rw-rw-r-- 1 huser huser 3092 Mar 21 10:22  wc.jar
 -rw-rw-r-- 1 huser huser   22 Mar 21 10:26  file01.txt
 -rw-rw-r-- 1 huser huser   28 Mar 21 10:26  file02.txt

 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -put /home/huser/hadoop-3.2.2/share/hadoop/mapreduce/wordcoun
 tSample/file01.txt /huser/data/wc/one/input/
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -put /home/huser/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample/file02.txt /huser/data/wc/one/input/
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/input
 Found 2 items
 -rw-r--r--   1 huser supergroup         22 2022-03-21 10:29 /huser/data/wc/one/input/file01.txt
 -rw-r--r--   1 huser supergroup         28 2022-03-21 10:29 /huser/data/wc/one/input/file02.txt
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$




 /home/huser/hadoop-3.2.2

 huser@thanoojWin10Home:~/hadoop-3.2.2/etc/hadoop$ cat mapred-site.xml
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
 <!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
 -->

 <!-- Put site-specific property overrides in this file. -->

 <configuration>
 <property>
  <name>mapreduce.framework.name</name>
  <value>yarn</value>
 </property>
 <property>
  <name>yarn.app.mapreduce.am.env</name>
  <value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
 </property>
 <property>
  <name>mapreduce.map.env</name>
  <value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
 </property>
 <property>
  <name>mapreduce.reduce.env</name>
  <value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
 </property>
 </configuration>


 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -rm -r -f /huser/data/wc/one/output
 Deleted /huser/data/wc/one/output
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/
 Found 1 items
 drwxr-xr-x   - huser supergroup          0 2022-03-21 10:29 /huser/data/wc/one/input
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$


 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
 total 28
 -rw-rw-r-- 1 huser huser 2089 Mar 21 10:21  WordCount.java
 -rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
 -rw-rw-r-- 1 huser huser 1511 Mar 21 10:22  WordCount.class
 -rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
 -rw-rw-r-- 1 huser huser 3092 Mar 21 10:22  wc.jar
 -rw-rw-r-- 1 huser huser   22 Mar 21 10:26  file01.txt
 -rw-rw-r-- 1 huser huser   28 Mar 21 10:26  file02.txt
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hadoop jar wc.jar WordCount /huser/data/wc/one/input /huser/data/wc/one/output
 2022-03-21 10:41:32,060 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
 2022-03-21 10:41:32,655 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
 2022-03-21 10:41:32,692 INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for path: /tmp/hadoop-yarn/staging/huser/.staging/job_1647836809874_0002
 2022-03-21 10:41:33,064 INFO input.FileInputFormat: Total input files to process : 2
 2022-03-21 10:41:33,190 INFO mapreduce.JobSubmitter: number of splits:2
 2022-03-21 10:41:33,634 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1647836809874_0002
 2022-03-21 10:41:33,637 INFO mapreduce.JobSubmitter: Executing with tokens: []
 2022-03-21 10:41:33,933 INFO conf.Configuration: resource-types.xml not found
 2022-03-21 10:41:33,934 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'.
 2022-03-21 10:41:34,052 INFO impl.YarnClientImpl: Submitted application application_1647836809874_0002
 2022-03-21 10:41:34,116 INFO mapreduce.Job: The url to track the job: http://thanoojWin10Home.localdomain:8088/proxy/application_1647836809874_0002/
 2022-03-21 10:41:34,118 INFO mapreduce.Job: Running job: job_1647836809874_0002
 2022-03-21 10:41:44,438 INFO mapreduce.Job: Job job_1647836809874_0002 running in uber mode : false
 2022-03-21 10:41:44,444 INFO mapreduce.Job:  map 0% reduce 0%
 2022-03-21 10:41:53,673 INFO mapreduce.Job:  map 100% reduce 0%
 2022-03-21 10:42:01,783 INFO mapreduce.Job:  map 100% reduce 100%
 2022-03-21 10:42:01,805 INFO mapreduce.Job: Job job_1647836809874_0002 completed successfully
 2022-03-21 10:42:01,997 INFO mapreduce.Job: Counters: 54
        File System Counters
                FILE: Number of bytes read=79
                FILE: Number of bytes written=704452
                FILE: Number of read operations=0
                FILE: Number of large read operations=0
                FILE: Number of write operations=0
                HDFS: Number of bytes read=292
                HDFS: Number of bytes written=41
                HDFS: Number of read operations=11
                HDFS: Number of large read operations=0
                HDFS: Number of write operations=2
                HDFS: Number of bytes read erasure-coded=0
        Job Counters
                Launched map tasks=2
                Launched reduce tasks=1
                Data-local map tasks=2
                Total time spent by all maps in occupied slots (ms)=13631
                Total time spent by all reduces in occupied slots (ms)=4545
                Total time spent by all map tasks (ms)=13631
                Total time spent by all reduce tasks (ms)=4545
                Total vcore-milliseconds taken by all map tasks=13631
                Total vcore-milliseconds taken by all reduce tasks=4545
                Total megabyte-milliseconds taken by all map tasks=13958144
                Total megabyte-milliseconds taken by all reduce tasks=4654080
        Map-Reduce Framework
                Map input records=2
                Map output records=8
                Map output bytes=82
                Map output materialized bytes=85
                Input split bytes=242
                Combine input records=8
                Combine output records=6
                Reduce input groups=5
                Reduce shuffle bytes=85
                Reduce input records=6
                Reduce output records=5
                Spilled Records=12
                Shuffled Maps =2
                Failed Shuffles=0
                Merged Map outputs=2
                GC time elapsed (ms)=171
                CPU time spent (ms)=2510
                Physical memory (bytes) snapshot=785104896
                Virtual memory (bytes) snapshot=8188284928
                Total committed heap usage (bytes)=625999872
                Peak Map Physical memory (bytes)=291561472
                Peak Map Virtual memory (bytes)=2730262528
                Peak Reduce Physical memory (bytes)=213037056
                Peak Reduce Virtual memory (bytes)=2727878656
        Shuffle Errors
                BAD_ID=0
                CONNECTION=0
                IO_ERROR=0
                WRONG_LENGTH=0
                WRONG_MAP=0
                WRONG_REDUCE=0
        File Input Format Counters
                Bytes Read=50
        File Output Format Counters
                Bytes Written=41
 huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$
	Hadoop 3.2.2
	Start-up commands:
	--------------------------------
	1. Stop the dfs and yarn first.
	2. Remove the datanode and namenode directories as specified in the core-site.xml file.
	3. Re-create the directories.
	4. hdfs namenode -format
	5. Then re-start the dfs and the yarn as follows.
	start-dfs.sh
	start-yarn.sh
	6. mapred --daemon start historyserver




	- Check SSH servicee is up and running :
	huser@thanoojWin10Home:~/tmpdata$ sudo service ssh status
	[sudo] password for huser:
	* sshd is running
	huser@thanoojWin10Home:~/tmpdata$

	- If not restart
	thanooj@thanoojWin10Home:~$ sudo service ssh restart
	[sudo] password for thanooj:
	* Restarting OpenBSD Secure Shell server sshd [ OK ]
	thanooj@thanoojWin10Home:~$ su - huser
	Password:
	huser@thanoojWin10Home:~$


	huser@thanoojWin10Home:~$ ls -ltr
	-rw-rw-r-- 1 huser huser 395448622 Jan 14 2021 hadoop-3.2.2.tar.gz
	drwxr-xr-x 10 huser huser 4096 Mar 20 15:34 hadoop-3.2.2
	drwxrwxr-x 4 huser huser 4096 Mar 21 09:47 dfsdata
	drwxrwxr-x 4 huser huser 4096 Mar 21 09:48 tmpdata
	huser@thanoojWin10Home:~/dfsdata$ ls -ltr
	drwxrwxr-x 2 huser huser 4096 Mar 21 09:47 namenode
	drwx------ 3 huser huser 4096 Mar 21 09:56 datanode
	huser@thanoojWin10Home:~/tmpdata$ ls -ltr
	drwxrwxr-x 4 huser huser 4096 Mar 21 09:55 dfs
	drwxr-xr-x 5 huser huser 4096 Mar 21 09:56 nm-local-dir
	huser@thanoojWin10Home:~/tmpdata$

	ref:
	https://hadoop.apache.org/docs/r3.1.1/hadoop-project-dist/hadoop-common/ClusterSetup.html











	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce$ mkdir wordcountSample
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce$ cd wordcountSample/
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi WordCount.java
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat WordCount.java
	import java.io.IOException;
	import java.util.StringTokenizer;

	import org.apache.hadoop.conf.Configuration;
	import org.apache.hadoop.fs.Path;
	import org.apache.hadoop.io.IntWritable;
	import org.apache.hadoop.io.Text;
	import org.apache.hadoop.mapreduce.Job;
	import org.apache.hadoop.mapreduce.Mapper;
	import org.apache.hadoop.mapreduce.Reducer;
	import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
	import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

	public class WordCount {

	public static class TokenizerMapper
	extends Mapper<Object, Text, Text, IntWritable>{

	private final static IntWritable one = new IntWritable(1);
	private Text word = new Text();

	public void map(Object key, Text value, Context context
	) throws IOException, InterruptedException {
	StringTokenizer itr = new StringTokenizer(value.toString());
	while (itr.hasMoreTokens()) {
	word.set(itr.nextToken());
	context.write(word, one);
	}
	}
	}

	public static class IntSumReducer
	extends Reducer<Text,IntWritable,Text,IntWritable> {
	private IntWritable result = new IntWritable();

	public void reduce(Text key, Iterable<IntWritable> values,
	Context context
	) throws IOException, InterruptedException {
	int sum = 0;
	for (IntWritable val : values) {
	sum += val.get();
	}
	result.set(sum);
	context.write(key, result);
	}
	}

	public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();
	Job job = Job.getInstance(conf, "word count");
	job.setJarByClass(WordCount.class);
	job.setMapperClass(TokenizerMapper.class);
	job.setCombinerClass(IntSumReducer.class);
	job.setReducerClass(IntSumReducer.class);
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(IntWritable.class);

	Hello World Bye World
	FileInputFormat.addInputPath(job, new Path(args[0]));
	FileOutputFormat.setOutputPath(job, new Path(args[1]));
	System.exit(job.waitForCompletion(true) ? 0 : 1);

	Hello Hadoop Goodbye Hadoop
	}
	}
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
	total 4
	-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ export HADOOP_CLASSPATH=${JAVA_HOME}/lib/tools.jar
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hadoop com.sun.tools.javac.Main WordCount.java
	/home/huser/hadoop-3.2.2/libexec/hadoop-functions.sh: line 2366: HADOOP_COM.SUN.TOOLS.JAVAC.MAIN_USER: invalid variable name
	/home/huser/hadoop-3.2.2/libexec/hadoop-functions.sh: line 2461: HADOOP_COM.SUN.TOOLS.JAVAC.MAIN_OPTS: invalid variable name
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ jar cf wc.jar WordCount*.class
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
	total 20
	-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
	-rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
	-rw-rw-r-- 1 huser huser 1511 Mar 21 10:22 WordCount.class
	-rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
	-rw-rw-r-- 1 huser huser 3092 Mar 21 10:22 wc.jar
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /
	Found 1 items
	drwxrwx--- - huser supergroup 0 2022-03-21 10:07 /tmp
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one/input
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -mkdir /huser/data/wc/one/output
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/
	Found 2 items
	drwxr-xr-x - huser supergroup 0 2022-03-21 10:24 /huser/data/wc/one/input
	drwxr-xr-x - huser supergroup 0 2022-03-21 10:24 /huser/data/wc/one/output
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi file01.txt
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat file01.txt
	Hello World Bye World
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ vi file02.txt
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ cat file02.txt
	Hello Hadoop Goodbye Hadoop
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
	total 28
	-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
	-rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
	-rw-rw-r-- 1 huser huser 1511 Mar 21 10:22 WordCount.class
	-rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
	-rw-rw-r-- 1 huser huser 3092 Mar 21 10:22 wc.jar
	-rw-rw-r-- 1 huser huser 22 Mar 21 10:26 file01.txt
	-rw-rw-r-- 1 huser huser 28 Mar 21 10:26 file02.txt

	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -put /home/huser/hadoop-3.2.2/share/hadoop/mapreduce/wordcoun
	tSample/file01.txt /huser/data/wc/one/input/
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -put /home/huser/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample/file02.txt /huser/data/wc/one/input/
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/input
	Found 2 items
	-rw-r--r-- 1 huser supergroup 22 2022-03-21 10:29 /huser/data/wc/one/input/file01.txt
	-rw-r--r-- 1 huser supergroup 28 2022-03-21 10:29 /huser/data/wc/one/input/file02.txt
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$




	/home/huser/hadoop-3.2.2

	huser@thanoojWin10Home:~/hadoop-3.2.2/etc/hadoop$ cat mapred-site.xml
	<?xml version="1.0"?>
	<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
	<!--
	Licensed under the Apache License, Version 2.0 (the "License");
	you may not use this file except in compliance with the License.
	You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License. See accompanying LICENSE file.
	-->

	<!-- Put site-specific property overrides in this file. -->

	<configuration>
	<property>
	<name>mapreduce.framework.name</name>
	<value>yarn</value>
	</property>
	<property>
	<name>yarn.app.mapreduce.am.env</name>
	<value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
	</property>
	<property>
	<name>mapreduce.map.env</name>
	<value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
	</property>
	<property>
	<name>mapreduce.reduce.env</name>
	<value>HADOOP_MAPRED_HOME=/home/huser/hadoop-3.2.2</value>
	</property>
	</configuration>


	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -rm -r -f /huser/data/wc/one/output
	Deleted /huser/data/wc/one/output
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hdfs dfs -ls /huser/data/wc/one/
	Found 1 items
	drwxr-xr-x - huser supergroup 0 2022-03-21 10:29 /huser/data/wc/one/input
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$


	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ ls -ltr
	total 28
	-rw-rw-r-- 1 huser huser 2089 Mar 21 10:21 WordCount.java
	-rw-rw-r-- 1 huser huser 1752 Mar 21 10:22 'WordCount$TokenizerMapper.class'
	-rw-rw-r-- 1 huser huser 1511 Mar 21 10:22 WordCount.class
	-rw-rw-r-- 1 huser huser 1755 Mar 21 10:22 'WordCount$IntSumReducer.class'
	-rw-rw-r-- 1 huser huser 3092 Mar 21 10:22 wc.jar
	-rw-rw-r-- 1 huser huser 22 Mar 21 10:26 file01.txt
	-rw-rw-r-- 1 huser huser 28 Mar 21 10:26 file02.txt
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$ hadoop jar wc.jar WordCount /huser/data/wc/one/input /huser/data/wc/one/output
	2022-03-21 10:41:32,060 INFO client.RMProxy: Connecting to ResourceManager at /0.0.0.0:8032
	2022-03-21 10:41:32,655 WARN mapreduce.JobResourceUploader: Hadoop command-line option parsing not performed. Implement the Tool interface and execute your application with ToolRunner to remedy this.
	2022-03-21 10:41:32,692 INFO mapreduce.JobResourceUploader: Disabling Erasure Coding for path: /tmp/hadoop-yarn/staging/huser/.staging/job_1647836809874_0002
	2022-03-21 10:41:33,064 INFO input.FileInputFormat: Total input files to process : 2
	2022-03-21 10:41:33,190 INFO mapreduce.JobSubmitter: number of splits:2
	2022-03-21 10:41:33,634 INFO mapreduce.JobSubmitter: Submitting tokens for job: job_1647836809874_0002
	2022-03-21 10:41:33,637 INFO mapreduce.JobSubmitter: Executing with tokens: []
	2022-03-21 10:41:33,933 INFO conf.Configuration: resource-types.xml not found
	2022-03-21 10:41:33,934 INFO resource.ResourceUtils: Unable to find 'resource-types.xml'.
	2022-03-21 10:41:34,052 INFO impl.YarnClientImpl: Submitted application application_1647836809874_0002
	2022-03-21 10:41:34,116 INFO mapreduce.Job: The url to track the job: http://thanoojWin10Home.localdomain:8088/proxy/application_1647836809874_0002/
	2022-03-21 10:41:34,118 INFO mapreduce.Job: Running job: job_1647836809874_0002
	2022-03-21 10:41:44,438 INFO mapreduce.Job: Job job_1647836809874_0002 running in uber mode : false
	2022-03-21 10:41:44,444 INFO mapreduce.Job: map 0% reduce 0%
	2022-03-21 10:41:53,673 INFO mapreduce.Job: map 100% reduce 0%
	2022-03-21 10:42:01,783 INFO mapreduce.Job: map 100% reduce 100%
	2022-03-21 10:42:01,805 INFO mapreduce.Job: Job job_1647836809874_0002 completed successfully
	2022-03-21 10:42:01,997 INFO mapreduce.Job: Counters: 54
	File System Counters
	FILE: Number of bytes read=79
	FILE: Number of bytes written=704452
	FILE: Number of read operations=0
	FILE: Number of large read operations=0
	FILE: Number of write operations=0
	HDFS: Number of bytes read=292
	HDFS: Number of bytes written=41
	HDFS: Number of read operations=11
	HDFS: Number of large read operations=0
	HDFS: Number of write operations=2
	HDFS: Number of bytes read erasure-coded=0
	Job Counters
	Launched map tasks=2
	Launched reduce tasks=1
	Data-local map tasks=2
	Total time spent by all maps in occupied slots (ms)=13631
	Total time spent by all reduces in occupied slots (ms)=4545
	Total time spent by all map tasks (ms)=13631
	Total time spent by all reduce tasks (ms)=4545
	Total vcore-milliseconds taken by all map tasks=13631
	Total vcore-milliseconds taken by all reduce tasks=4545
	Total megabyte-milliseconds taken by all map tasks=13958144
	Total megabyte-milliseconds taken by all reduce tasks=4654080
	Map-Reduce Framework
	Map input records=2
	Map output records=8
	Map output bytes=82
	Map output materialized bytes=85
	Input split bytes=242
	Combine input records=8
	Combine output records=6
	Reduce input groups=5
	Reduce shuffle bytes=85
	Reduce input records=6
	Reduce output records=5
	Spilled Records=12
	Shuffled Maps =2
	Failed Shuffles=0
	Merged Map outputs=2
	GC time elapsed (ms)=171
	CPU time spent (ms)=2510
	Physical memory (bytes) snapshot=785104896
	Virtual memory (bytes) snapshot=8188284928
	Total committed heap usage (bytes)=625999872
	Peak Map Physical memory (bytes)=291561472
	Peak Map Virtual memory (bytes)=2730262528
	Peak Reduce Physical memory (bytes)=213037056
	Peak Reduce Virtual memory (bytes)=2727878656
	Shuffle Errors
	BAD_ID=0
	CONNECTION=0
	IO_ERROR=0
	WRONG_LENGTH=0
	WRONG_MAP=0
	WRONG_REDUCE=0
	File Input Format Counters
	Bytes Read=50
	File Output Format Counters
	Bytes Written=41
	huser@thanoojWin10Home:~/hadoop-3.2.2/share/hadoop/mapreduce/wordcountSample$