Hadoop 3.4.1 单机版
#
root操作
#
export HADOOP_VERSION=3.4.1
wget https://dlcdn.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
sed -i "s|bookworm-updates|bookworm-updates bullseye|g" /etc/apt/sources.list.d/debian.sources
apt update && apt install openjdk-11-jdk-headless -y
sed -i '$aexport HADOOP_VERSION=\$HADOOP_VERSION' /etc/profile
sed -i '$aexport JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64' /etc/profile
sed -i '$aexport HADOOP_HOME=/opt/hadoop' /etc/profile
sed -i '$aPATH=\$PATH:\$JAVA_HOME/bin/:\$HADOOP_HOME/bin:\$HADOOP_HOME/sbin' /etc/profile
sudo useradd -ms/bin/bash -k/etc/skel/ hadoop
sudo usermod -aG sudo hadoop
echo hadoop:. | chpasswd
tar -zxvf hadoop-${HADOOP_VERSION}.tar.gz -C /opt/ --transform="s|hadoop-${HADOOP_VERSION}|hadoop|g"
rm hadoop-${HADOOP_VERSION}.tar.gz
sed -i "1 i JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64" /opt/hadoop/etc/hadoop/hadoop-env.sh
find /opt/hadoop/ -name *cmd -delete
mkdir /opt/hadoop/data
chown hadoop /opt -R
文档(可以不做)
apt install nginx -y
cp /opt/hadoop/share/doc/hadoop/* /var/www/html -r
nginx
hadoop免登录hadoop
#
mkdir ~/.ssh
echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDR5HpVwM8VvMcfS+gJBkW/IS6n7Pgcd1Rkq6AbxP7aShCFJewuPhQM6FslaARkWoAgH4wC3qtMxLkZ5fNgAPFewYEYuJXARGOIuBoAtwJYc7KYoDeYwkhFcnsWWAWezYOtqG3IapRfIgOcx3+j7HditZAn5nCsZit4WZAD+VdW3w== " >> ~/.ssh/authorized_keys
chmod 700 ~/.ssh
chmod 644 ~/.ssh/authorized_keys
cat > ~/.ssh/id_rsa <<EOF
-----BEGIN OPENSSH PRIVATE KEY-----
b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAlwAAAAdzc2gtcn
NhAAAAAwEAAQAAAIEA0eR6VcDPFbzHH0voCQZFvyEup+z4HHdUZKugG8T+2koQhSXsLj4U
DOhbJWgEZFqAIB+MAt6rTMS5GeXzYADxXsGBGLiVwERjiLgaALcCWHOymKA3mMJIRXJ7Fl
gFns2DrahtyGqUXyIDnMd/o+x3YrWQJ+ZwrGYreFmQA/lXVt8AAAIIeMsZxnjLGcYAAAAH
c3NoLXJzYQAAAIEA0eR6VcDPFbzHH0voCQZFvyEup+z4HHdUZKugG8T+2koQhSXsLj4UDO
hbJWgEZFqAIB+MAt6rTMS5GeXzYADxXsGBGLiVwERjiLgaALcCWHOymKA3mMJIRXJ7FlgF
ns2DrahtyGqUXyIDnMd/o+x3YrWQJ+ZwrGYreFmQA/lXVt8AAAADAQABAAAAgEsyyIGgqp
DnKZAU6fURwjtP8B9JzzqjpiDLPHm2gnhCwNhzVcLbiuN0+/LS6X/qzKy1oLwGeujuMkMN
F7z1mYPqqcZmpVj97BF7ITQuzVJdEs6ujva/arp3V/FQYECLGHJ+pz+IBkJhVaXpKeLJOD
aeESEOy3jCp66fH6mFiFvBAAAAQGNHUW02lSyOGdoIPvxVk+boTXuZNYENCKOtBHTxX1Fd
vQBFMuWWvk0dEHZ1sSbGds+CHrrMGEJN6aIxKXSuKdoAAABBAPyYTB4PV6WpKiJPAy/F6m
MViOnfjfiX9/KqFYENd5EElm8FjjJ0lmC6YwJsSOPaB18BSmi2LpG9+QB7ujMHcgUAAABB
ANS40Jx9hA9HJ3gFQ06GS5Xr3sPjoOqkmCPFC+nhNBvdOxUwxfVsKly3qI5jjs9zZ3sben
5ge2hwzEq0xZMtxpMAAAAMcm9vdEB0ZXN0Lm1tAQIDBAUGBw==
-----END OPENSSH PRIVATE KEY-----
EOF
chmod 600 ~/.ssh/id_rsa
hadoop操作
#
- /opt/hadoop/etc/core-site.xml:①使用单机网络模式 ②网页默认用户为hadoop
- /opt/hadoop/etc/hdfs-site.xml:①只使用一个副本
- /opt/hadoop/etc/hadoop/yarn-site.xml
- /opt/hadoop/etc/hadoop/mapred-site.xml
- /opt/hadoop/etc/hadoop/workers
sed -i "20 i <property><name>fs.defaultFS</name><value>hdfs://localhost:9000</value></property>" ${HADOOP_HOME}/etc/hadoop/core-site.xml
sed -i "20 i <property><name>hadoop.http.staticuser.user</name><value>hadoop</value></property>" ${HADOOP_HOME}/etc/hadoop/core-site.xml
sed -i "20 i <property><name>hadoop.tmp.dir</name><value>${HADOOP_HOME}/data/</value></property>" ${HADOOP_HOME}/etc/hadoop/core-site.xml
sed -i "20 i <property><name>dfs.replication</name><value>1</value></property>" ${HADOOP_HOME}/etc/hadoop/hdfs-site.xml
sed -i "16 i <property><name>yarn.resourcemanager.hostname</name><value>0.0.0.0</value></property>" ${HADOOP_HOME}/etc/hadoop/yarn-site.xml
sed -i "16 i <property><name>yarn.nodemanager.aux-services</name><value>mapreduce_shuffle</value></property>" ${HADOOP_HOME}/etc/hadoop/yarn-site.xml
sed -i "16 i <property><name>yarn.application.classpath</name><value>`hadoop classpath`</value></property>" ${HADOOP_HOME}/etc/hadoop/yarn-site.xml
sed -i "20 i <property><name>mapreduce.framework.name</name><value>yarn</value></property>" ${HADOOP_HOME}/etc/hadoop/mapred-site.xml
启动命令
#
- 必须先格式化节点 hdfs namenode -format && start-dfs.sh
echo "formathadoop=hdfs namenode -format" >> ~/.bashrc
echo "alias starthadoop=\${HADOOP_HOME}/sbin/start-all.sh" >> ~/.bashrc
echo "alias stophadoop=\${HADOOP_HOME}/sbin/stop-all.sh" >> ~/.bashrc