ParrotXray
0d49371692
Reviewed-on: #3 Co-authored-by: ParrotXray <b05817691@gmail.com> Co-committed-by: ParrotXray <b05817691@gmail.com>
116 lines
4.1 KiB
Docker
116 lines
4.1 KiB
Docker
FROM ubuntu:24.04 AS builder
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
ENV HADOOP_VERSION=3.5.0
|
|
ENV HADOOP_HOME=/opt/hadoop-${HADOOP_VERSION}
|
|
|
|
RUN cat > /etc/apt/sources.list.d/ubuntu.sources <<'EOF'
|
|
Types: deb
|
|
URIs: http://tw.archive.ubuntu.com/ubuntu/
|
|
Suites: noble noble-updates noble-backports noble-security
|
|
Components: main restricted universe multiverse
|
|
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
|
|
EOF
|
|
|
|
RUN apt-get update && \
|
|
apt-get install -y aria2 pigz && \
|
|
apt-get clean && rm -rf /var/lib/apt/lists/*
|
|
|
|
RUN aria2c -x 16 -s 16 -q \
|
|
https://downloads.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz \
|
|
-o /tmp/hadoop.tar.gz && \
|
|
tar -I pigz -xf /tmp/hadoop.tar.gz -C /opt && \
|
|
rm /tmp/hadoop.tar.gz
|
|
|
|
RUN echo "export JAVA_HOME=/usr/lib/jvm/zulu21-ca-amd64" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
|
|
echo "export HDFS_NAMENODE_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
|
|
echo "export HDFS_DATANODE_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
|
|
echo "export HDFS_SECONDARYNAMENODE_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
|
|
echo "export YARN_RESOURCEMANAGER_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh && \
|
|
echo "export YARN_NODEMANAGER_USER=root" >> ${HADOOP_HOME}/etc/hadoop/hadoop-env.sh
|
|
|
|
RUN cat > ${HADOOP_HOME}/etc/hadoop/core-site.xml <<'EOF'
|
|
<configuration>
|
|
<property>
|
|
<name>fs.defaultFS</name>
|
|
<value>hdfs://localhost:9000</value>
|
|
</property>
|
|
</configuration>
|
|
EOF
|
|
|
|
RUN cat > ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml <<'EOF'
|
|
<configuration>
|
|
<property>
|
|
<name>dfs.replication</name>
|
|
<value>1</value>
|
|
</property>
|
|
</configuration>
|
|
EOF
|
|
|
|
RUN cat > ${HADOOP_HOME}/etc/hadoop/mapred-site.xml <<'EOF'
|
|
<configuration>
|
|
<property>
|
|
<name>mapreduce.framework.name</name>
|
|
<value>yarn</value>
|
|
</property>
|
|
<property>
|
|
<name>mapreduce.application.classpath</name>
|
|
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
|
|
</property>
|
|
</configuration>
|
|
EOF
|
|
|
|
RUN cat > ${HADOOP_HOME}/etc/hadoop/yarn-site.xml <<'EOF'
|
|
<configuration>
|
|
<property>
|
|
<name>yarn.nodemanager.aux-services</name>
|
|
<value>mapreduce_shuffle</value>
|
|
</property>
|
|
<property>
|
|
<name>yarn.nodemanager.env-whitelist</name>
|
|
<value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
|
|
</property>
|
|
</configuration>
|
|
EOF
|
|
|
|
FROM ubuntu:24.04
|
|
|
|
LABEL authors="ParrotXray"
|
|
LABEL description="Hadoop with Cloud Computing Course"
|
|
|
|
ENV DEBIAN_FRONTEND=noninteractive
|
|
ENV HADOOP_VERSION=3.5.0
|
|
ENV JAVA_HOME=/usr/lib/jvm/zulu21-ca-amd64
|
|
ENV HADOOP_HOME=/opt/hadoop-${HADOOP_VERSION}
|
|
ENV PATH=${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin:${JAVA_HOME}/bin:${PATH}
|
|
|
|
RUN cat > /etc/apt/sources.list.d/ubuntu.sources <<'EOF'
|
|
Types: deb
|
|
URIs: http://tw.archive.ubuntu.com/ubuntu/
|
|
Suites: noble noble-updates noble-backports noble-security
|
|
Components: main restricted universe multiverse
|
|
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
|
|
EOF
|
|
|
|
RUN apt-get update && \
|
|
apt-get install -y curl gnupg openssh-server ssh pdsh sudo dumb-init && \
|
|
curl -fsSL https://repos.azul.com/azul-repo.key | gpg --dearmor -o /usr/share/keyrings/azul.gpg && \
|
|
echo "deb [signed-by=/usr/share/keyrings/azul.gpg] https://repos.azul.com/zulu/deb stable main" \
|
|
> /etc/apt/sources.list.d/zulu.list && \
|
|
apt-get update && \
|
|
apt-get install -y zulu21-ca-jdk && \
|
|
apt-get clean && rm -rf /var/lib/apt/lists/*
|
|
|
|
COPY --from=builder /opt/hadoop-${HADOOP_VERSION} /opt/hadoop-${HADOOP_VERSION}
|
|
|
|
RUN ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa && \
|
|
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys && \
|
|
chmod 0600 ~/.ssh/authorized_keys && \
|
|
echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
|
|
|
|
COPY entrypoint.sh /entrypoint.sh
|
|
RUN chmod +x /entrypoint.sh
|
|
|
|
EXPOSE 9000 9870 8088 8042 19888
|
|
|
|
ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint.sh"] |