hadoop 3.4.1 运维篇
#
块损坏与丢失
#
hdfs dfsadmin -safemode get #查看安全模式状态
hdfs dfsadmin -safemode enter #进入安全模式状态
hdfs dfsadmin -safemode leave #离开安全模式状态
hdfs dfsadmin -safemode wait #等待安全模式状态
hadoop fsck #检查时候存在丢失或者损坏
hdfs fsck / -delete #删除问题文件,在获得问题的解决办法前不建议使用
测试数据
#
for x in {1..100};do dd if=/dev/random of=$RANDOM bs=1M count=128;done
hadoop fs -put * /4m
查找文件
#
find /opt/ -mmin -30
find /opt/ -exec touch -ht 197001010000.00 {} \;
rm /opt/data/* /opt/hadoop-3.4.1/logs/* -rf
find /opt/ -name *meta
空间管理
#
- 只分配1G能启动,但存储不了文件
- The volume with the most available space (=950 190 080 B) is less than the block size (=134 217 728 B)
sudo mkdir /home/hadoop/space/ -p
sudo dd if=/dev/zero of=/home/hadoop/space/hadoop0 bs=1G count=2
sudo dd if=/dev/zero of=/home/hadoop/space/hadoop1 bs=1G count=2
sudo dd if=/dev/zero of=/home/hadoop/space/hadoop2 bs=1G count=2
sudo dd if=/dev/zero of=/home/hadoop/space/hadoop3 bs=1G count=2
sudo losetup /dev/loop0 /home/hadoop/space/hadoop0
sudo losetup /dev/loop1 /home/hadoop/space/hadoop1
sudo losetup /dev/loop2 /home/hadoop/space/hadoop2
sudo losetup /dev/loop3 /home/hadoop/space/hadoop3
sudo mkfs.ext4 /dev/loop0
sudo mkfs.ext4 /dev/loop1
sudo mkfs.ext4 /dev/loop2
sudo mkfs.ext4 /dev/loop3
sudo mount -t ext4 /dev/loop0 /home/hadoop/hadoop0
sudo mount -t ext4 /dev/loop1 /home/hadoop/hadoop1
sudo mount -t ext4 /dev/loop2 /home/hadoop/hadoop2
sudo mount -t ext4 /dev/loop3 /home/hadoop/hadoop3
sudo chmod 777 /home/hadoop/hadoop* -R
sudo losetup -d /dev/loop0 /dev/loop1 /dev/loop2 /dev/loop3