271 lines
9.7 KiB
Bash
271 lines
9.7 KiB
Bash
#!/bin/bash
|
|
set -e # Exit on any error
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
LIGHT_CYAN='\033[1;36m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Logging functions
|
|
log() {
|
|
echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] $1${NC}"
|
|
}
|
|
|
|
error() {
|
|
echo -e "${RED}[ERROR] $1${NC}" >&2
|
|
}
|
|
|
|
warning() {
|
|
echo -e "${YELLOW}[WARNING] $1${NC}"
|
|
}
|
|
|
|
info() {
|
|
echo -e "${BLUE}[INFO] $1${NC}"
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Config
|
|
# ─────────────────────────────────────────────
|
|
HADOOP_VERSION="3.5.0"
|
|
HADOOP_DIR="hadoop-${HADOOP_VERSION}"
|
|
HADOOP_TAR="${HADOOP_DIR}.tar.gz"
|
|
HADOOP_URL="https://dlcdn.apache.org/hadoop/common/${HADOOP_DIR}/${HADOOP_TAR}"
|
|
|
|
JDK_DEB="jdk-21_linux-x64_bin.deb"
|
|
JDK_URL="https://download.oracle.com/java/21/latest/${JDK_DEB}"
|
|
JAVA_HOME_PATH="/usr/java/jdk-21-oracle-x64"
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Pre-flight checks
|
|
# ─────────────────────────────────────────────
|
|
check_root() {
|
|
if [[ $EUID -ne 0 ]]; then
|
|
error "This script must be run as root (use sudo)."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
check_memory() {
|
|
local mem_kb
|
|
mem_kb=$(grep MemTotal /proc/meminfo | awk '{print $2}')
|
|
local mem_gb=$(( mem_kb / 1024 / 1024 ))
|
|
if [[ $mem_gb -lt 4 ]]; then
|
|
warning "System has ${mem_gb}GB RAM. Hadoop recommends at least 4GB."
|
|
else
|
|
info "Memory check passed: ${mem_gb}GB available."
|
|
fi
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Step 1 — Install OS packages
|
|
# ─────────────────────────────────────────────
|
|
install_packages() {
|
|
log "Installing required packages: openssh-server, ssh, pdsh ..."
|
|
apt update -qq
|
|
apt install -y openssh-server ssh pdsh
|
|
log "Packages installed."
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Step 2 — Install Java 21 (Oracle JDK)
|
|
# ─────────────────────────────────────────────
|
|
install_java() {
|
|
if java -version 2>&1 | grep -q "21"; then
|
|
info "Java 21 is already installed. Skipping."
|
|
return
|
|
fi
|
|
|
|
log "Downloading Oracle JDK 21 ..."
|
|
wget -q --show-progress "${JDK_URL}" -O "/tmp/${JDK_DEB}"
|
|
log "Installing JDK ..."
|
|
dpkg -i "/tmp/${JDK_DEB}"
|
|
rm -f "/tmp/${JDK_DEB}"
|
|
log "Java installed at ${JAVA_HOME_PATH}."
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Step 3 — Download & extract Hadoop
|
|
# ─────────────────────────────────────────────
|
|
install_hadoop() {
|
|
if [[ -d "/opt/${HADOOP_DIR}" ]]; then
|
|
info "Hadoop ${HADOOP_VERSION} already extracted at /opt/${HADOOP_DIR}. Skipping download."
|
|
return
|
|
fi
|
|
|
|
log "Downloading Hadoop ${HADOOP_VERSION} ..."
|
|
wget -q --show-progress "${HADOOP_URL}" -O "/tmp/${HADOOP_TAR}"
|
|
log "Extracting Hadoop to /opt/ ..."
|
|
tar -zxf "/tmp/${HADOOP_TAR}" -C /opt/
|
|
rm -f "/tmp/${HADOOP_TAR}"
|
|
log "Hadoop extracted to /opt/${HADOOP_DIR}."
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Step 4 — Configure SSH (passwordless localhost)
|
|
# ─────────────────────────────────────────────
|
|
configure_ssh() {
|
|
log "Configuring passwordless SSH for localhost ..."
|
|
|
|
# Ensure sshd is running
|
|
service ssh start || true
|
|
|
|
local SSH_DIR="${HOME}/.ssh"
|
|
mkdir -p "${SSH_DIR}"
|
|
chmod 700 "${SSH_DIR}"
|
|
|
|
if [[ ! -f "${SSH_DIR}/id_rsa" ]]; then
|
|
ssh-keygen -t rsa -P '' -f "${SSH_DIR}/id_rsa"
|
|
else
|
|
info "SSH key already exists. Skipping keygen."
|
|
fi
|
|
|
|
cat "${SSH_DIR}/id_rsa.pub" >> "${SSH_DIR}/authorized_keys"
|
|
chmod 0600 "${SSH_DIR}/authorized_keys"
|
|
|
|
# Accept localhost host key automatically
|
|
ssh-keyscan -H localhost >> "${SSH_DIR}/known_hosts" 2>/dev/null
|
|
log "SSH configured."
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Step 5 — Write Hadoop config files
|
|
# ─────────────────────────────────────────────
|
|
configure_hadoop() {
|
|
local ETC="/opt/${HADOOP_DIR}/etc/hadoop"
|
|
log "Writing Hadoop configuration files to ${ETC} ..."
|
|
|
|
# hadoop-env.sh — set JAVA_HOME
|
|
sed -i "s|# export JAVA_HOME=.*|export JAVA_HOME=${JAVA_HOME_PATH}|" "${ETC}/hadoop-env.sh"
|
|
# Append if the line wasn't there
|
|
grep -q "^export JAVA_HOME=${JAVA_HOME_PATH}" "${ETC}/hadoop-env.sh" \
|
|
|| echo "export JAVA_HOME=${JAVA_HOME_PATH}" >> "${ETC}/hadoop-env.sh"
|
|
|
|
# core-site.xml
|
|
cat > "${ETC}/core-site.xml" <<'EOF'
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
|
<configuration>
|
|
<property>
|
|
<name>fs.defaultFS</name>
|
|
<value>hdfs://localhost:9000</value>
|
|
</property>
|
|
</configuration>
|
|
EOF
|
|
|
|
# hdfs-site.xml
|
|
cat > "${ETC}/hdfs-site.xml" <<'EOF'
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
|
<configuration>
|
|
<property>
|
|
<name>dfs.replication</name>
|
|
<value>1</value>
|
|
</property>
|
|
</configuration>
|
|
EOF
|
|
|
|
# mapred-site.xml
|
|
cat > "${ETC}/mapred-site.xml" <<'EOF'
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
|
<configuration>
|
|
<property>
|
|
<name>mapreduce.framework.name</name>
|
|
<value>yarn</value>
|
|
</property>
|
|
<property>
|
|
<name>mapreduce.application.classpath</name>
|
|
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
|
|
</property>
|
|
</configuration>
|
|
EOF
|
|
|
|
# yarn-site.xml
|
|
cat > "${ETC}/yarn-site.xml" <<'EOF'
|
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
|
|
<configuration>
|
|
<property>
|
|
<name>yarn.nodemanager.aux-services</name>
|
|
<value>mapreduce_shuffle</value>
|
|
</property>
|
|
<property>
|
|
<name>yarn.nodemanager.env-whitelist</name>
|
|
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
|
|
</property>
|
|
</configuration>
|
|
EOF
|
|
|
|
log "Configuration files written."
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Step 6 — Set environment variables system-wide
|
|
# ─────────────────────────────────────────────
|
|
set_env() {
|
|
local PROFILE="/etc/profile.d/hadoop.sh"
|
|
log "Writing environment variables to ${PROFILE} ..."
|
|
cat > "${PROFILE}" <<EOF
|
|
export JAVA_HOME=${JAVA_HOME_PATH}
|
|
export HADOOP_HOME=/opt/${HADOOP_DIR}
|
|
export HADOOP_MAPRED_HOME=\$HADOOP_HOME
|
|
export HADOOP_CONF_DIR=\$HADOOP_HOME/etc/hadoop
|
|
export PATH=\$PATH:\$HADOOP_HOME/bin:\$HADOOP_HOME/sbin:\$JAVA_HOME/bin
|
|
EOF
|
|
# Also export for current session
|
|
# shellcheck source=/dev/null
|
|
source "${PROFILE}"
|
|
log "Environment variables set."
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Step 7 — Format HDFS NameNode
|
|
# ─────────────────────────────────────────────
|
|
format_hdfs() {
|
|
log "Formatting HDFS NameNode ..."
|
|
export JAVA_HOME=${JAVA_HOME_PATH}
|
|
/opt/${HADOOP_DIR}/bin/hdfs namenode -format -force
|
|
log "HDFS formatted."
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Step 8 — Start all Hadoop services
|
|
# ─────────────────────────────────────────────
|
|
start_hadoop() {
|
|
log "Starting all Hadoop services ..."
|
|
export JAVA_HOME=${JAVA_HOME_PATH}
|
|
/opt/${HADOOP_DIR}/sbin/start-all.sh
|
|
log "Hadoop services started."
|
|
}
|
|
|
|
# ─────────────────────────────────────────────
|
|
# Main
|
|
# ─────────────────────────────────────────────
|
|
main() {
|
|
echo -e "${LIGHT_CYAN} Hadoop ${HADOOP_VERSION} Auto-Installer${NC}"
|
|
echo ""
|
|
|
|
check_root
|
|
check_memory
|
|
|
|
install_packages
|
|
install_java
|
|
install_hadoop
|
|
configure_ssh
|
|
configure_hadoop
|
|
set_env
|
|
format_hdfs
|
|
start_hadoop
|
|
|
|
echo -e "${LIGHT_CYAN} Installation Complete!${NC}"
|
|
echo ""
|
|
info "HDFS Web UI → http://localhost:9870"
|
|
info "YARN Web UI → http://localhost:8088"
|
|
info "Run 'source /etc/profile.d/hadoop.sh' or re-login to reload PATH."
|
|
}
|
|
|
|
main "$@"
|