feat: Add automatic installation script and update README

2026-04-22 18:52:08 +08:00 · 2026-04-22 18:52:08 +08:00 · 02c68a07aa
commit 02c68a07aa
parent f79ba6c198
3 changed files with 288 additions and 2 deletions
--- a/README.md
+++ b/README.md
@ -15,6 +15,18 @@ docker exec -it hadoop bash
 ### Manual installation
 Please refer to [Hadoop Installation Guide](Hadoop_Install_Guide.md) for manual installation of Hadoop.

+### Automatic installation
+```bash
+# clone the repository
+git clone https://gitea.revservices.tw/joy/Hadoop.git
+
+# access the Hadoop directory
+cd Hadoop
+
+# Run the installation script
+chmod +x install_hadoop.sh
+./install_hadoop.sh
+```

 ### Run Wordcount example 
 ```bash
@ -22,7 +34,8 @@ Please refer to [Hadoop Installation Guide](Hadoop_Install_Guide.md) for manual
 hdfs dfs -mkdir -p /user/root/input

 # push test text to test.txt
-echo "I am a student. You are also student." | hdfs dfs -put - /user/root/input/test.txt
+echo "I am a student. You are also student." > test.txt
+hdfs dfs -put test.txt /user/root/input/

 # run workcount 
 hadoop jar /opt/hadoop-3.5.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.5.0.jar wordcount /user/root/input /user/root/output
--- a/entrypoint.sh
+++ b/entrypoint.sh
@ -9,4 +9,7 @@ fi

 ${HADOOP_HOME}/sbin/start-all.sh

-tail -f ${HADOOP_HOME}/logs/*.log
+trap "${HADOOP_HOME}/sbin/stop-all.sh; exit 0" SIGTERM SIGINT
+
+tail -f ${HADOOP_HOME}/logs/*.log &
+wait $!
--- a/install_hadoop.sh
+++ b/install_hadoop.sh
@ -0,0 +1,270 @@
+#!/bin/bash
+set -e  # Exit on any error
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+LIGHT_CYAN='\033[1;36m'
+NC='\033[0m' # No Color
+
+# Logging functions
+log() {
+    echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}"
+}
+
+error() {
+    echo -e "${RED}[ERROR] $1${NC}" >&2
+}
+
+warning() {
+    echo -e "${YELLOW}[WARNING] $1${NC}"
+}
+
+info() {
+    echo -e "${BLUE}[INFO] $1${NC}"
+}
+
+# ─────────────────────────────────────────────
+# Config
+# ─────────────────────────────────────────────
+HADOOP_VERSION="3.5.0"
+HADOOP_DIR="hadoop-${HADOOP_VERSION}"
+HADOOP_TAR="${HADOOP_DIR}.tar.gz"
+HADOOP_URL="https://dlcdn.apache.org/hadoop/common/${HADOOP_DIR}/${HADOOP_TAR}"
+
+JDK_DEB="jdk-21_linux-x64_bin.deb"
+JDK_URL="https://download.oracle.com/java/21/latest/${JDK_DEB}"
+JAVA_HOME_PATH="/usr/java/jdk-21-oracle-x64"
+
+# ─────────────────────────────────────────────
+# Pre-flight checks
+# ─────────────────────────────────────────────
+check_root() {
+    if [[ $EUID -ne 0 ]]; then
+        error "This script must be run as root (use sudo)."
+        exit 1
+    fi
+}
+
+check_memory() {
+    local mem_kb
+    mem_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}')
+    local mem_gb=$(( mem_kb / 1024 / 1024 ))
+    if [[ $mem_gb -lt 4 ]]; then
+        warning "System has ${mem_gb}GB RAM. Hadoop recommends at least 4GB."
+    else
+        info "Memory check passed: ${mem_gb}GB available."
+    fi
+}
+
+# ─────────────────────────────────────────────
+# Step 1 — Install OS packages
+# ─────────────────────────────────────────────
+install_packages() {
+    log "Installing required packages: openssh-server, ssh, pdsh ..."
+    apt update -qq
+    apt install -y openssh-server ssh pdsh
+    log "Packages installed."
+}
+
+# ─────────────────────────────────────────────
+# Step 2 — Install Java 21 (Oracle JDK)
+# ─────────────────────────────────────────────
+install_java() {
+    if java -version 2>&1 | grep -q "21"; then
+        info "Java 21 is already installed. Skipping."
+        return
+    fi
+
+    log "Downloading Oracle JDK 21 ..."
+    wget -q --show-progress "${JDK_URL}" -O "/tmp/${JDK_DEB}"
+    log "Installing JDK ..."
+    dpkg -i "/tmp/${JDK_DEB}"
+    rm -f "/tmp/${JDK_DEB}"
+    log "Java installed at ${JAVA_HOME_PATH}."
+}
+
+# ─────────────────────────────────────────────
+# Step 3 — Download & extract Hadoop
+# ─────────────────────────────────────────────
+install_hadoop() {
+    if [[ -d "/opt/${HADOOP_DIR}" ]]; then
+        info "Hadoop ${HADOOP_VERSION} already extracted at /opt/${HADOOP_DIR}. Skipping download."
+        return
+    fi
+
+    log "Downloading Hadoop ${HADOOP_VERSION} ..."
+    wget -q --show-progress "${HADOOP_URL}" -O "/tmp/${HADOOP_TAR}"
+    log "Extracting Hadoop to /opt/ ..."
+    tar -zxf "/tmp/${HADOOP_TAR}" -C /opt/
+    rm -f "/tmp/${HADOOP_TAR}"
+    log "Hadoop extracted to /opt/${HADOOP_DIR}."
+}
+
+# ─────────────────────────────────────────────
+# Step 4 — Configure SSH (passwordless localhost)
+# ─────────────────────────────────────────────
+configure_ssh() {
+    log "Configuring passwordless SSH for localhost ..."
+
+    # Ensure sshd is running
+    service ssh start || true
+
+    local SSH_DIR="${HOME}/.ssh"
+    mkdir -p "${SSH_DIR}"
+    chmod 700 "${SSH_DIR}"
+
+    if [[ ! -f "${SSH_DIR}/id_rsa" ]]; then
+        ssh-keygen -t rsa -P '' -f "${SSH_DIR}/id_rsa"
+    else
+        info "SSH key already exists. Skipping keygen."
+    fi
+
+    cat "${SSH_DIR}/id_rsa.pub" >> "${SSH_DIR}/authorized_keys"
+    chmod 0600 "${SSH_DIR}/authorized_keys"
+
+    # Accept localhost host key automatically
+    ssh-keyscan -H localhost >> "${SSH_DIR}/known_hosts" 2>/dev/null
+    log "SSH configured."
+}
+
+# ─────────────────────────────────────────────
+# Step 5 — Write Hadoop config files
+# ─────────────────────────────────────────────
+configure_hadoop() {
+    local ETC="/opt/${HADOOP_DIR}/etc/hadoop"
+    log "Writing Hadoop configuration files to ${ETC} ..."
+
+    # hadoop-env.sh — set JAVA_HOME
+    sed -i "s|# export JAVA_HOME=.*|export JAVA_HOME=${JAVA_HOME_PATH}|" "${ETC}/hadoop-env.sh"
+    # Append if the line wasn't there
+    grep -q "^export JAVA_HOME=${JAVA_HOME_PATH}" "${ETC}/hadoop-env.sh" \
+        || echo "export JAVA_HOME=${JAVA_HOME_PATH}" >> "${ETC}/hadoop-env.sh"
+
+    # core-site.xml
+    cat > "${ETC}/core-site.xml" <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+    </property>
+</configuration>
+EOF
+
+    # hdfs-site.xml
+    cat > "${ETC}/hdfs-site.xml" <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+    <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+    </property>
+</configuration>
+EOF
+
+    # mapred-site.xml
+    cat > "${ETC}/mapred-site.xml" <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+    <property>
+        <name>mapreduce.framework.name</name>
+        <value>yarn</value>
+    </property>
+    <property>
+        <name>mapreduce.application.classpath</name>
+        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
+    </property>
+</configuration>
+EOF
+
+    # yarn-site.xml
+    cat > "${ETC}/yarn-site.xml" <<'EOF'
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle</value>
+    </property>
+    <property>
+        <name>yarn.nodemanager.env-whitelist</name>
+        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
+    </property>
+</configuration>
+EOF
+
+    log "Configuration files written."
+}
+
+# ─────────────────────────────────────────────
+# Step 6 — Set environment variables system-wide
+# ─────────────────────────────────────────────
+set_env() {
+    local PROFILE="/etc/profile.d/hadoop.sh"
+    log "Writing environment variables to ${PROFILE} ..."
+    cat > "${PROFILE}" <<EOF
+export JAVA_HOME=${JAVA_HOME_PATH}
+export HADOOP_HOME=/opt/${HADOOP_DIR}
+export HADOOP_MAPRED_HOME=\$HADOOP_HOME
+export HADOOP_CONF_DIR=\$HADOOP_HOME/etc/hadoop
+export PATH=\$PATH:\$HADOOP_HOME/bin:\$HADOOP_HOME/sbin:\$JAVA_HOME/bin
+EOF
+    # Also export for current session
+    # shellcheck source=/dev/null
+    source "${PROFILE}"
+    log "Environment variables set."
+}
+
+# ─────────────────────────────────────────────
+# Step 7 — Format HDFS NameNode
+# ─────────────────────────────────────────────
+format_hdfs() {
+    log "Formatting HDFS NameNode ..."
+    export JAVA_HOME=${JAVA_HOME_PATH}
+    /opt/${HADOOP_DIR}/bin/hdfs namenode -format -force
+    log "HDFS formatted."
+}
+
+# ─────────────────────────────────────────────
+# Step 8 — Start all Hadoop services
+# ─────────────────────────────────────────────
+start_hadoop() {
+    log "Starting all Hadoop services ..."
+    export JAVA_HOME=${JAVA_HOME_PATH}
+    /opt/${HADOOP_DIR}/sbin/start-all.sh
+    log "Hadoop services started."
+}
+
+# ─────────────────────────────────────────────
+# Main
+# ─────────────────────────────────────────────
+main() {
+    echo -e "${LIGHT_CYAN}  Hadoop ${HADOOP_VERSION} Auto-Installer${NC}"
+    echo ""
+
+    check_root
+    check_memory
+
+    install_packages
+    install_java
+    install_hadoop
+    configure_ssh
+    configure_hadoop
+    set_env
+    format_hdfs
+    start_hadoop
+
+    echo -e "${LIGHT_CYAN}  Installation Complete!${NC}"
+    echo ""
+    info "HDFS Web UI → http://localhost:9870"
+    info "YARN Web UI → http://localhost:8088"
+    info "Run 'source /etc/profile.d/hadoop.sh' or re-login to reload PATH."
+}
+
+main "$@"