joy/Hadoop
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

joy-patch-1 (#1)

Browse Source 
Reviewed-on: #1
This commit is contained in:
joy 2026-04-14 18:13:02 +09:00
parent ae24818f5a
commit 2d642e982c
1 changed files with 123 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

123
Hadoop_Install_Guide.md Normal file
View File

@ -0,0 +1,123 @@
# Hadoop Install Guide
## Prerequirement
OS: Linux
Memory: 4G
## package requirement
Hadoop require `openssh-server`, `ssh`, `pdsh`
```sh
$apt update
$apt install openssh-server ssh pdsh -y
```
## Java prepare
Download `java` from [Orcale](https://www.oracle.com/tw/java/technologies/downloads/#java21) [direct link](https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.deb)
```sh
wget https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.deb
sudo dpkg -i jdk-21_linux-x64_bin.deb
```
## Hadoop prepare
Download `Hadoop` [here](https://hadoop.apache.org/releases.html) [direct link](https://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-3.5.0/hadoop-3.5.0.tar.gz)
```sh
wget https://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-3.5.0/hadoop-3.5.0.tar.gz
tar -zxvf hadoop-3.5.0.tar.gz
```
## Configure Hadoop
```sh
cd hadoop-3.5.0
export JAVA_HOME=/usre/java/jdk-21-orcale-x64
```
## Configure remote ssh access
Connect to server first
```sh
ssh localhost
```
Generate ssh key
```sh
$ ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
$ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
$ chmod 0600 ~/.ssh/authorized_keys
```
Configure Hadoop environment
```sh
# hadoop-3.5.0/etc/hadoop/hadoop-env.sh
JAVA_HOME=/usre/java/jdk-21-orcale-x64
```
```xml
# hadoop-3.5.0/etc/hadoop/core-site.xml
<configuration>
<property>
<name>fs.defaultFS</name>
<value>hdfs://localhost:9000</value>
</property>
</configuration>
```
```xml
# hadoop-3.5.0/etc/hadoop/hdfs-site.xml
<configuration>
<property>
<name>dfs.replication</name>
<value>1</value>
</property>
</configuration>
```
```xml
# hadoop-3.5.0/etc/hadoop/mapred-site.xml
<configuration>
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<property>
<name>mapreduce.application.classpath</name>
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
</property>
</configuration>
```
```xml
# hadoop-3.5.0/etc/hadoop/yarn-site.xml
<configuration>
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<property>
<name>yarn.nodemanager.env-whitelist</name>
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
</property>
</configuration>
```
## Format hadoop
```sh
bin/hdfs namenode -format
```
## Startup all hadoop services.
```sh
sbin/start-all.sh
```