From 02c68a07aa237499ccc953e334d032220f19d019 Mon Sep 17 00:00:00 2001 From: ParrotXray Date: Wed, 22 Apr 2026 18:52:08 +0800 Subject: [PATCH] feat: Add automatic installation script and update README --- README.md | 15 ++- entrypoint.sh | 5 +- install_hadoop.sh | 270 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 install_hadoop.sh diff --git a/README.md b/README.md index 9b4a4d5..9fafc95 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,18 @@ docker exec -it hadoop bash ### Manual installation Please refer to [Hadoop Installation Guide](Hadoop_Install_Guide.md) for manual installation of Hadoop. +### Automatic installation +```bash +# clone the repository +git clone https://gitea.revservices.tw/joy/Hadoop.git + +# access the Hadoop directory +cd Hadoop + +# Run the installation script +chmod +x install_hadoop.sh +./install_hadoop.sh +``` ### Run Wordcount example ```bash @@ -22,7 +34,8 @@ Please refer to [Hadoop Installation Guide](Hadoop_Install_Guide.md) for manual hdfs dfs -mkdir -p /user/root/input # push test text to test.txt -echo "I am a student. You are also student." | hdfs dfs -put - /user/root/input/test.txt +echo "I am a student. You are also student." > test.txt +hdfs dfs -put test.txt /user/root/input/ # run workcount hadoop jar /opt/hadoop-3.5.0/share/hadoop/mapreduce/hadoop-mapreduce-examples-3.5.0.jar wordcount /user/root/input /user/root/output diff --git a/entrypoint.sh b/entrypoint.sh index f091a70..7cce292 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -9,4 +9,7 @@ fi ${HADOOP_HOME}/sbin/start-all.sh -tail -f ${HADOOP_HOME}/logs/*.log \ No newline at end of file +trap "${HADOOP_HOME}/sbin/stop-all.sh; exit 0" SIGTERM SIGINT + +tail -f ${HADOOP_HOME}/logs/*.log & +wait $! \ No newline at end of file diff --git a/install_hadoop.sh b/install_hadoop.sh new file mode 100644 index 0000000..0948736 --- /dev/null +++ b/install_hadoop.sh @@ -0,0 +1,270 @@ +#!/bin/bash +set -e # Exit on any error + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +LIGHT_CYAN='\033[1;36m' +NC='\033[0m' # No Color + +# Logging functions +log() { + echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}" +} + +error() { + echo -e "${RED}[ERROR] $1${NC}" >&2 +} + +warning() { + echo -e "${YELLOW}[WARNING] $1${NC}" +} + +info() { + echo -e "${BLUE}[INFO] $1${NC}" +} + +# ───────────────────────────────────────────── +# Config +# ───────────────────────────────────────────── +HADOOP_VERSION="3.5.0" +HADOOP_DIR="hadoop-${HADOOP_VERSION}" +HADOOP_TAR="${HADOOP_DIR}.tar.gz" +HADOOP_URL="https://dlcdn.apache.org/hadoop/common/${HADOOP_DIR}/${HADOOP_TAR}" + +JDK_DEB="jdk-21_linux-x64_bin.deb" +JDK_URL="https://download.oracle.com/java/21/latest/${JDK_DEB}" +JAVA_HOME_PATH="/usr/java/jdk-21-oracle-x64" + +# ───────────────────────────────────────────── +# Pre-flight checks +# ───────────────────────────────────────────── +check_root() { + if [[ $EUID -ne 0 ]]; then + error "This script must be run as root (use sudo)." + exit 1 + fi +} + +check_memory() { + local mem_kb + mem_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}') + local mem_gb=$(( mem_kb / 1024 / 1024 )) + if [[ $mem_gb -lt 4 ]]; then + warning "System has ${mem_gb}GB RAM. Hadoop recommends at least 4GB." + else + info "Memory check passed: ${mem_gb}GB available." + fi +} + +# ───────────────────────────────────────────── +# Step 1 — Install OS packages +# ───────────────────────────────────────────── +install_packages() { + log "Installing required packages: openssh-server, ssh, pdsh ..." + apt update -qq + apt install -y openssh-server ssh pdsh + log "Packages installed." +} + +# ───────────────────────────────────────────── +# Step 2 — Install Java 21 (Oracle JDK) +# ───────────────────────────────────────────── +install_java() { + if java -version 2>&1 | grep -q "21"; then + info "Java 21 is already installed. Skipping." + return + fi + + log "Downloading Oracle JDK 21 ..." + wget -q --show-progress "${JDK_URL}" -O "/tmp/${JDK_DEB}" + log "Installing JDK ..." + dpkg -i "/tmp/${JDK_DEB}" + rm -f "/tmp/${JDK_DEB}" + log "Java installed at ${JAVA_HOME_PATH}." +} + +# ───────────────────────────────────────────── +# Step 3 — Download & extract Hadoop +# ───────────────────────────────────────────── +install_hadoop() { + if [[ -d "/opt/${HADOOP_DIR}" ]]; then + info "Hadoop ${HADOOP_VERSION} already extracted at /opt/${HADOOP_DIR}. Skipping download." + return + fi + + log "Downloading Hadoop ${HADOOP_VERSION} ..." + wget -q --show-progress "${HADOOP_URL}" -O "/tmp/${HADOOP_TAR}" + log "Extracting Hadoop to /opt/ ..." + tar -zxf "/tmp/${HADOOP_TAR}" -C /opt/ + rm -f "/tmp/${HADOOP_TAR}" + log "Hadoop extracted to /opt/${HADOOP_DIR}." +} + +# ───────────────────────────────────────────── +# Step 4 — Configure SSH (passwordless localhost) +# ───────────────────────────────────────────── +configure_ssh() { + log "Configuring passwordless SSH for localhost ..." + + # Ensure sshd is running + service ssh start || true + + local SSH_DIR="${HOME}/.ssh" + mkdir -p "${SSH_DIR}" + chmod 700 "${SSH_DIR}" + + if [[ ! -f "${SSH_DIR}/id_rsa" ]]; then + ssh-keygen -t rsa -P '' -f "${SSH_DIR}/id_rsa" + else + info "SSH key already exists. Skipping keygen." + fi + + cat "${SSH_DIR}/id_rsa.pub" >> "${SSH_DIR}/authorized_keys" + chmod 0600 "${SSH_DIR}/authorized_keys" + + # Accept localhost host key automatically + ssh-keyscan -H localhost >> "${SSH_DIR}/known_hosts" 2>/dev/null + log "SSH configured." +} + +# ───────────────────────────────────────────── +# Step 5 — Write Hadoop config files +# ───────────────────────────────────────────── +configure_hadoop() { + local ETC="/opt/${HADOOP_DIR}/etc/hadoop" + log "Writing Hadoop configuration files to ${ETC} ..." + + # hadoop-env.sh — set JAVA_HOME + sed -i "s|# export JAVA_HOME=.*|export JAVA_HOME=${JAVA_HOME_PATH}|" "${ETC}/hadoop-env.sh" + # Append if the line wasn't there + grep -q "^export JAVA_HOME=${JAVA_HOME_PATH}" "${ETC}/hadoop-env.sh" \ + || echo "export JAVA_HOME=${JAVA_HOME_PATH}" >> "${ETC}/hadoop-env.sh" + + # core-site.xml + cat > "${ETC}/core-site.xml" <<'EOF' + + + + + fs.defaultFS + hdfs://localhost:9000 + + +EOF + + # hdfs-site.xml + cat > "${ETC}/hdfs-site.xml" <<'EOF' + + + + + dfs.replication + 1 + + +EOF + + # mapred-site.xml + cat > "${ETC}/mapred-site.xml" <<'EOF' + + + + + mapreduce.framework.name + yarn + + + mapreduce.application.classpath + $HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/* + + +EOF + + # yarn-site.xml + cat > "${ETC}/yarn-site.xml" <<'EOF' + + + + + yarn.nodemanager.aux-services + mapreduce_shuffle + + + yarn.nodemanager.env-whitelist + JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME + + +EOF + + log "Configuration files written." +} + +# ───────────────────────────────────────────── +# Step 6 — Set environment variables system-wide +# ───────────────────────────────────────────── +set_env() { + local PROFILE="/etc/profile.d/hadoop.sh" + log "Writing environment variables to ${PROFILE} ..." + cat > "${PROFILE}" <