梦想还是要有的，万一忘了咋办？

0%

搭建Hadoop环境

发表于 分类于
本文字数： 12k 阅读时长 ≈ 11 分钟

目录

  • 物料清单
  • 安装虚拟机
  • 安装系统
  • 初始化系统

物料清单

软件 选择 备注
虚拟机 vmware https://www.vmware.com/
操作系统 Centos7.x https://mirrors.aliyun.com/centos/7/isos/x86_64/
Hadoop hadoop3.4.0 https://dlcdn.apache.org/hadoop/common/
Jdk 1.8.0-u41 https://download.oracle.com/otn/java/jdk/8u411b09/43d62d619bee416215729597d70b8ac/jdk-8u411-linux-x64.tar.gz
ZooKeeper 3.8.4 https://zookeeper.apache.org/
Oracle 账号 908344069@qq.com/Java_2024

安装虚拟机

安装过程 略过

虚拟网络配置

在虚拟机/编辑/虚拟网络编辑器
可以修改你喜欢的网段，我选择的是192.168.100.0
虚拟网络编辑器

安装系统

新建虚拟机

选择系统镜像

虚拟机名称

设置存储空间

安装系统

语言

选择磁盘

设置root

选择Root密码设置

准备系统节点

系统初始化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
# ssh 允许证书登陆
sed -i '/^#PubkeyAuthentication/s/^#//g' /etc/ssh/sshd_config
systemctl restart sshd


# 关闭selinux
cat > /etc/selinux/config <<\EOF
SELINUX=disabled
SELINUXTYPE=targeted
EOF

# 关闭防火墙
systemctl stop firewalld
systemctl disable firewalld 



# 修改为静态IP
cat > /etc/sysconfig/network-scripts/ifcfg-ens33 << \EOF
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
UUID=b2f7a405-628b-48a1-9c4a-1436fca70006
DEVICE=ens33
ONBOOT=yes

IPADDR=192.168.100.120
NETMASK=255.255.255.0
GATEWAY=192.168.100.2
DNS1=114.114.114.114
EOF

systemctl restart network.service


# 修改 hostname
cat >> /etc/hostname << \EOF
192.168.100.131 node1
192.168.100.132 node2
192.168.100.133 node3
192.168.100.134 node4
EOF

# 安装pdsh
yum install -y epel-release
yum install -y pdsh
echo 'export PDSH_RCMD_TYPE=ssh' >> /etc/profile
source /etc/profile


克隆节点

克隆4个节点

节点 IP 备注
node1 192.168.100.131
node2 192.168.100.132
node3 192.168.100.133
node4 192.168.100.134

节点初始化

每台机器执行一次：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
#修改变量后执行
#name="node1"
#ip="192.168.100.131"

#修改hostname
hostnamectl set-hostname $name

#修改静态ip
sudo sed -i "s/192.168.100.130/$ip/g"  /etc/sysconfig/network-scripts/ifcfg-ens33

# 配置hosts
cat >> /etc/hosts << \EOF
192.168.100.131 node1
192.168.100.132 node2
192.168.100.133 node3
192.168.100.134 node4
EOF

ssh互信

每台机器执行一次：

1
2
3
4
5
6
7
#ssh互通
ssh-keygen -q -t rsa -N "" -f ～/.ssh/id_rsa

ssh-copy-id node1;
ssh-copy-id node2;
ssh-copy-id node3;
ssh-copy-id node4;

安装软件

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
cd /opt/
tar -zxvf jdk-8u411-linux-x64.tar.gz
tar -zxvf  hadoop-3.4.0.tar.gz

cat >> /etc/profile <<\EOF
export JAVA_HOME=/opt/jdk1.8.0_411
export PATH=$JAVA_HOME/bin:$PATH
export HADOOP_HOME=/opt/hadoop-3.4.0
export PATH=$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$PATH
EOF

# 分发软件
pdcp -r -w node2,node3,node4 /opt/* /opt/
pdcp -w node2,node3,node4 /etc/profile /etc/

#检查结果
pdsh -w node2,node3,node4 '. /etc/profile ; java -version'

配置hadoop

机器 NN RM NM DN ZK JournalNode
node1 O O – –
node2 O O O O O –
node3 – O O O –
node4 – O O O –

手动HA

启用NameNode、DataNode、JournalNode,RM,NM 不启用zk

以下操作在node1 的 /opt/hadoop-3.4.0 目录下进行

**

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229

# etc/hadoop/hadoop-env.sh
# 配置启动账号 root

cat >> etc/hadoop/hadoop-env.sh <<\EOF
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_JOURNALNODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
EOF


# etc/hadoop/core-site.xml
cat > etc/hadoop/core-site.xml <<\EOF
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://mycluster</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/data/cluster</value>
    </property>
    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>

</configuration>

EOF

# etc/hadoop/hdfs-site.xml
cat > etc/hadoop/hdfs-site.xml <<\EOF

<configuration>
	<property>
        	<name>dfs.nameservices</name>
        	<value>mycluster</value>
	</property>
	<property>
        	<name>dfs.ha.namenodes.mycluster</name>
        	<value>nn1,nn2,nn3</value>
	</property>
	<property>
        	<name>dfs.ha.namenode.id.nn1</name>
        	<value>nn1</value>
	</property>
	<property>
        	<name>dfs.ha.namenode.id.nn2</name>
        	<value>nn2</value>
	</property>
	<property>
        	<name>dfs.ha.namenode.id.nn3</name>
        	<value>nn3</value>
	</property>
	<property>
        	<name>dfs.namenode.rpc-address.mycluster.nn1</name>
        	<value>node1:8020</value>
	</property>
	<property>
        	<name>dfs.namenode.rpc-address.mycluster.nn2</name>
        	<value>node2:8020</value>
	</property>
	<property>
        	<name>dfs.namenode.rpc-address.mycluster.nn3</name>
        	<value>node3:8020</value>
	</property>
	<property>
        	<name>dfs.namenode.http-address.mycluster.nn1</name>
        	<value>node1:9870</value>
	</property>
	<property>
        	<name>dfs.namenode.http-address.mycluster.nn2</name>
        	<value>node2:9870</value>
	</property>
	<property>
        	<name>dfs.namenode.http-address.mycluster.nn3</name>
        	<value>node3:9870</value>
	</property>
	<property>
        	<name>dfs.namenode.shared.edits.dir</name>
        	<value>qjournal://node1:8485;node2:8485;node3:8485/mycluster</value>
	</property>
	<property>
        	<name>dfs.namenode.edits.qjournals.resolution-enabled</name>
        	<value>false</value>
	</property>

	<property>
        	<name>dfs.client.failover.proxy.provider.mycluster</name>
        	<value>org.apache.hadoop.hdfs.server.namenode.ha.RequestHedgingProxyProvider</value>
	</property>
<!-- shell(/bin/true)-- 非常重要、会导致无法切换...>
	<property>
        	<name>dfs.ha.fencing.methods</name>
        	<value>
               sshfence
               shell(/bin/true)
            </value>
	</property>
	<property>
        	<name>dfs.ha.fencing.ssh.private-key-files</name>
        	<value>/home/root/.ssh/id_rsa</value>
	</property>
	<property>
        	<name>dfs.journalnode.edits.dir</name>
        	<value>/data/cluster/dfs/journal</value>
	</property>
	<property>
        	<name>dfs.ha.nn.not-become-active-in-safemode</name>
        	<value>true</value>
	</property>

	<property>
        	<name>dfs.replication</name>
        	<value>3</value>
	</property>
	<property>
        	<name>dfs.namenode.name.dir</name>
        	<value>/data/cluster/dfs/name</value>
	</property>
	<property>
        	<name>dfs.datenode.data.dir</name>
        	<value>/data/cluster/dfs/data</value>
	</property>
	<property>
        	<name>dfs.blocksize</name>
        	<value>4096</value>
	</property>
	<property>
        	<name>dfs.namenode.handler.count</name>
        	<value>50</value>
	</property>
</configuration>
EOF

# etc/hadoop/mapred-site.xml
cat > etc/hadoop/mapred-site.xml <\EOF
<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.application.classpath</name
<value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
    </property>

</configuration>

EOF

# 配置yarn、未启动zk时 yarn的rm无法高可用（手动的也不行）
# etc/hadoop/yarn-site.xml

cat > etc/hadoop/yarn-site.xml <<\EOF
    <!-- 非HA的配置-- >
    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>node1</value>
    </property>
    <!-- 非HA的配置-- >

    <property>
        <name>yarn.acl.enable</name>
        <value>false</value>
    </property>
    <property>
        <name>yarn.admin.acl</name>
        <value>*</value>
    </property>
    <property>
        <name>yarn.log-aggregation-enable</name>
        <value>true</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
    </property>
</configuration>
EOF

#分发配置文件
pdcp -w node2,node3,node4 etc/hadoop/* /opt/hadoop-3.4.0/etc/hadoop/

#初始化hdfs
#启动journalNode
pdsh -w node1,node2,node3 '. /etc/profile ; hdfs --daemon start journalnode'

#格式化namenode
hdfs namenode -format

#同步namenode元数据
hdfs --daemon start namenode
pdsh -w node2,node3 '. /etc/profie; hdfs namenode -bootstrapStandby'

#关闭所有进程
hdfs --daemon stop namenode
pdsh -w node1,node2,node3 '. /etc/profile ; hdfs --daemon stop journalnode'

#正式启动
#启动dfs系统
start-dfs.sh
#启动yarn服务
start-yarn.sh

#手动制定者namenode
hdfs haadmin -transitionToActive nn1

# 验证系统
## 验证 namenode 又一个时active状态
hdfs haadmin -getAllServiceState

## 验证dfs系统
hdfs dfs -ls /
hdfs dfs -mkdir -p /user/root
hdfs dfs -mkdir input
hdfs dfs -put etc/hadoop/*.xml input
hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-3.4.0.jar  grep input output 'dfs[a-z.]+'

hdfs dfs -cat output/*

安装zk

在node1 下操作

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

cd /opt
tar -zxvf apache-zookeeper-3.8.4-bin.tar.gz
mv apache-zookeeper-3.8.4-bin zookeeper-3.8.4

cd zookeeper-3.8.4/conf
cat > zoo.cfg <<\EOF
tickTime=2000
dataDir=/data/zookeeper/
clientPort=2181
initLimit=5
syncLimit=2
server.1=zk1:2888:3888
server.2=zk2:2888:3888
server.3=zk3:2888:3888
EOF

#添加hosts
cat >> /etc/hosts <<\EOF
192.168.100.131 zk1
192.168.100.132 zk2
192.168.100.133 zk3

#添加环境变量
cat >> /etc/profile <<\EOF
export ZK_HOME=/opt/zookeeper-3.8.4
export PATH=$ZK_HOME/bin:$PATH
EOF

# 分发程序
pdcp  -w node2,node3,node4 /etc/hosts /etc/hosts
pdcp  -w node2,node3,node4 /etc/profile /etc/profile
pdcp  -r -w node2,node3,node4 /opt/zoo* /opt/
pdsh  -w node1,node2,node3,node4 'mkdir -p /data/zookeeper;touch /data/zookeeper/myid'
pdsh  -w node1 'echo 1 >> /data/zookeeper/myid'
pdsh  -w node2 'echo 1 >> /data/zookeeper/myid'
pdsh  -w node3 'echo 3 >> /data/zookeeper/myid'
pdsh  -w node4 'echo 4 >> /data/zookeeper/myid'

#启动zk
pdsh  -w node1,node2,node3 '. /etc/profile;zkServer.sh start'

配置自动高可用

涉及角色：NameNode、ResourceManager
以下操作在 node1 、/opt/hadoop-3.4.0 下进行：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61

#关闭集群
stop-dfds.sh
stop-yarn.sh

# etc/hadoop/hadoop-env.sh
echo 'export HDFS_ZKFC_USER=root'>> etc/hadoop/hadoop-env.sh

# etc/hadoop/core-site.xml
cat > etc/hadoop/core-site.xml <<\EOF
<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://mycluster</value>
    </property>
    <property>
        <name>ha.zookeeper.quorum</name>
        <value>zk1:2181,zk2:2181,zk3:2181</value>
    </property> 
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/data/cluster</value>
    </property>
    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>
</configuration>

EOF
#etc/hadoop/hdfs-site.xml
#添加以下配置
	<property>
        	<name>dfs.ha.automatic-failover.enabled</name>
        	<value>true</value>
	</property>

#etc/hadoop/yarn-site.xml
#添加以下配置
<property>
  <name>hadoop.zk.address</name>
  <value>zk1:2181,zk2:2181,zk3:2181</value>
</property>

#分发程序
pdcp -r -w node2,node3,node4 etc/hadoop/* /opt/hadoop-3.4.0/etc/hadoop/


#初始化zk信息
hdfs zkfc -formatZK

#启动集群
start-dfs.sh
start-yarn.sh

#验证 namenode 高可用
hdfs haadmin -getAllServiceState
#kill -9 当前active的节点上的进程
#查看是否有active的节点
hdfs haadmin -getAllServiceState