3. hadoop完全分布式搭建¶

1. 部署方案¶

# 1 master: 192.168.178.101
# 3 slave:  192.168.178.102  192.168.178.103  192.168.178.104

# 建议：8G内存： 1G*4   16G内存：2G*4

2. 克隆虚拟机¶

# 使用刚才搭建伪分布式的虚拟机，链接克隆4个新的虚拟机出来

1. 首先关闭hadoop进程¶

stop-all.sh

2. 关闭虚拟机¶

shutdown -h 0

3. 链接克隆虚拟机¶

# 名称是：hadoop-s101 hadoop-s102 hadoop-s103 hadoop-s104

4. 启动4台虚拟机¶

5. 分别修改静态IP¶

# 192.168.178.101 192.168.178.102  192.168.178.103  192.168.178.104
vim /etc/sysconfig/network-scripts/ifcfg-ens33
# 只修改1处将IPADDR修改成自己的IP！

6. 分别重启虚拟机¶

reboot -h 0

7. 远程工具连接¶

8. 修改主机名¶

hostnamectl set-hostname s101
hostnamectl set-hostname s102
hostnamectl set-hostname s103
hostnamectl set-hostname s104

3. 添加主机映射¶

# 所有节点
# root权限
vim /etc/hosts
192.168.178.101 s101
192.168.178.102 s102
192.168.178.103 s103
192.168.178.104 s104

4. 配置免密登录¶

# s101
# 需要设置 s101>s102 s103 s104

#0. 切换到centos用户下！
su - centos

#1. 在s101生成公私密钥对
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa

#2. 发送密钥，配置免密登录
ssh-copy-id centos@s101
ssh-copy-id centos@s102
ssh-copy-id centos@s103
ssh-copy-id centos@s104

#3. 验证
ssh s102
exit

5. 修改配置文件¶

1. 修改符号链接¶

# s101
cd /soft/hadoop/etc
ln -sfT full hadoop

2. core-site.xml¶

cd hadoop
vim core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://s101:8020</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/centos/hadoop</value>
    </property>
</configuration>

3. hdfs-site.xml¶

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
</configuration>

4. mapred-site.xml¶

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
</configuration>

5. yarn-site.xml¶

<?xml version="1.0"?>
<configuration>
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>s101</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
</configuration>

6. hadoop-env.sh¶

vim hadoop-env.sh
# 25行如下就行了
export JAVA_HOME=/soft/jdk

7. slaves¶

vim slaves
s102
s103
s104

6. 分发配置文件¶

scp -r /soft/hadoop/etc/full centos@s102:/soft/hadoop/etc/
scp -r /soft/hadoop/etc/full centos@s103:/soft/hadoop/etc/
scp -r /soft/hadoop/etc/full centos@s104:/soft/hadoop/etc/

7. 修改s102-s104的符号连接¶

ssh s102 "ln -sfT /soft/hadoop/etc/full /soft/hadoop/etc/hadoop"
ssh s103 "ln -sfT /soft/hadoop/etc/full /soft/hadoop/etc/hadoop"
ssh s104 "ln -sfT /soft/hadoop/etc/full /soft/hadoop/etc/hadoop"

# 如果之前有hadoop文件夹，则先删除！
ssh s102 "rm -rf /soft/hadoop/etc/hadoop"
ssh s103 "rm -rf /soft/hadoop/etc/hadoop"
ssh s104 "rm -rf /soft/hadoop/etc/hadoop"

8. 同步hosts文件¶

# 因为hosts文件需要root权限，所以使用root，然后输入密码
scp /etc/hosts root@s102:/etc
scp /etc/hosts root@s103:/etc
scp /etc/hosts root@s104:/etc

9. 格式化文件系统¶

hdfs namenode -format

10. 启动hadoop¶

start-all.sh

11. 验证安装¶

# s101
jps
# s102-s104
jps

#主节点上有3个进程
[centos@s101 hadoop]$ jps
18963 NameNode
19732 Jps
19241 SecondaryNameNode
19418 ResourceManager

#slave节点上有2个进程
[root@s102 ~]# jps
8964 NodeManager
8869 DataNode
9084 Jps

12. 打开web界面查看¶

# 192.168.178.101:50070
# 如果打不开网页，则关闭防火墙
sudo iptables -F

2. 打开slave节点的web界面¶

# 192.168.178.102:50075

至此，hadoop完全分布式搭建完成！¶

最后更新: 2022-02-20 08:44:07