feat/add-dockerfile (#3)

Reviewed-on: #3
Co-authored-by: ParrotXray <b05817691@gmail.com>
Co-committed-by: ParrotXray <b05817691@gmail.com>

Dockerfile

@@ -0,0 +1,116 @@
+FROM ubuntu:24.04 AS builder
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV HADOOP_VERSION=3.5.0
+ENV HADOOP_HOME=/opt/hadoop-${HADOOP_VERSION}
+
+RUN cat > /etc/apt/sources.list.d/ubuntu.sources <<'EOF'
+Types: deb
+URIs: http://tw.archive.ubuntu.com/ubuntu/
+Suites: noble noble-updates noble-backports noble-security
+Components: main restricted universe multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+EOF
+
+RUN apt-get update && \
+    apt-get install -y aria2 pigz && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN aria2c -x 16 -s 16 -q \
+        https://downloads.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz \
+        -o /tmp/hadoop.tar.gz && \
+    tar -I pigz -xf /tmp/hadoop.tar.gz -C /opt && \
+    rm /tmp/hadoop.tar.gz
+
+RUN echo "export JAVA_HOME=/usr/lib/jvm/zulu21-ca-amd64" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
+    echo "export HDFS_NAMENODE_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
+    echo "export HDFS_DATANODE_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
+    echo "export HDFS_SECONDARYNAMENODE_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
+    echo "export YARN_RESOURCEMANAGER_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
+    echo "export YARN_NODEMANAGER_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
+
+RUN cat > ${HADOOP_HOME}/etc/hadoop/core-site.xml <<'EOF'
+<configuration>
+    <property>
+        <name>fs.defaultFS</name>
+        <value>hdfs://localhost:9000</value>
+    </property>
+</configuration>
+EOF
+
+RUN cat > ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml <<'EOF'
+<configuration>
+    <property>
+        <name>dfs.replication</name>
+        <value>1</value>
+    </property>
+</configuration>
+EOF
+
+RUN cat > ${HADOOP_HOME}/etc/hadoop/mapred-site.xml <<'EOF'
+<configuration>
+    <property>
+        <name>mapreduce.framework.name</name>
+        <value>yarn</value>
+    </property>
+    <property>
+        <name>mapreduce.application.classpath</name>
+        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
+    </property>
+</configuration>
+EOF
+
+RUN cat > ${HADOOP_HOME}/etc/hadoop/yarn-site.xml <<'EOF'
+<configuration>
+    <property>
+        <name>yarn.nodemanager.aux-services</name>
+        <value>mapreduce_shuffle</value>
+    </property>
+    <property>
+        <name>yarn.nodemanager.env-whitelist</name>
+        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
+    </property>
+</configuration>
+EOF
+
+FROM ubuntu:24.04
+
+LABEL authors="ParrotXray"
+LABEL description="Hadoop with Cloud Computing Course"
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV HADOOP_VERSION=3.5.0
+ENV JAVA_HOME=/usr/lib/jvm/zulu21-ca-amd64
+ENV HADOOP_HOME=/opt/hadoop-${HADOOP_VERSION}
+ENV PATH=${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:${JAVA_HOME}/bin:${PATH}
+
+RUN cat > /etc/apt/sources.list.d/ubuntu.sources <<'EOF'
+Types: deb
+URIs: http://tw.archive.ubuntu.com/ubuntu/
+Suites: noble noble-updates noble-backports noble-security
+Components: main restricted universe multiverse
+Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+EOF
+
+RUN apt-get update && \
+    apt-get install -y curl gnupg openssh-server ssh pdsh sudo dumb-init && \
+    curl -fsSL https://repos.azul.com/azul-repo.key | gpg --dearmor -o /usr/share/keyrings/azul.gpg && \
+    echo "deb [signed-by=/usr/share/keyrings/azul.gpg] https://repos.azul.com/zulu/deb stable main" \
+        > /etc/apt/sources.list.d/zulu.list && \
+    apt-get update && \
+    apt-get install -y zulu21-ca-jdk && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /opt/hadoop-${HADOOP_VERSION} /opt/hadoop-${HADOOP_VERSION}
+
+RUN ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa && \
+    cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && \
+    chmod 0600 ~/.ssh/authorized_keys && \
+    echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
+
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 9000 9870 8088 8042 19888
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint.sh"]

README.md

@@ -1,2 +1,16 @@
 # Hadoop
 
+### Using docker 
+```bash
+# Pull the Hadoop image from the registry
+docker pull gitea.revservices.tw/parrotxray/hadoop-3.5.0:master
+
+# Run the Hadoop container
+docker run -d --name hadoop -p 9000:9000 -p 9870:9870 -p 8088:8088 -p 8042:8042 -p 19888:19888 gitea.revservices.tw/parrotxray/hadoop-3.5.0:master
+
+# Access the Hadoop container
+docker exec -it hadoop bash
+```
+
+### Manual installation
+Please refer to [Hadoop Installation Guide](Hadoop_Install_Guide.md) for manual installation of Hadoop.

entrypoint.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+
+service ssh start
+
+if [ ! -d "${HADOOP_HOME}/data/namenode/current" ]; then
+    ${HADOOP_HOME}/bin/hdfs namenode -format -force
+fi
+
+${HADOOP_HOME}/sbin/start-all.sh
+
+tail -f ${HADOOP_HOME}/logs/*.log