1.解壓Hadoop，zookeeper，配置環(huán)境變量
#JAVA_HOME
JAVA_HOME=/home/admin/modules/jdk1.8.0_144
export CLASSPATH=.:$JAVA_HOME/jre/lib/rt.jar:$JAVA_HOME/lib/dt.jar:JAVA_HOME/lib/tools.jar
export PATH=$PATH:$JAVA_HOME/bin
#HADOOP_HOME
export HADOOP_HOME=/home/admin/modules/hadoop-3.1.3
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
##ZOOKEEPER_HOME
export ZOOKEEPER_HOME=/home/admin/modules/zookeeper-3.4.10
export PATH=$PATH:$ZOOKEEPER_HOME/bin
2.修改配置文件
zoo.cfg
[root@node3 conf]# cat zoo.cfg
# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=10
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=5
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/home/admin/modules/zookeeper-3.4.10/zkData
# the port at which the clients will connect
clientPort=2181
# the maximum number of client connections.
# increase this if you need to handle more clients
#maxClientCnxns=60
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to "0" to disable auto purge feature
#autopurge.purgeInterval=1
server.1=node1:2888:3888
server.2=node2:2888:3888
server.3=node3:2888:3888
#server.4=node4:2888:3888
3.在zookeeper解壓文件夾創(chuàng)建zkData
touch myid
echo 1 > myid
4.配置Hadoop
修改.env文件
在hadoop-env.sh, mapred-env.sh,yarn-env.sh? 添加JAVA-HOME
export JAVA_HOME=/home/admin/modules/jdk1.8.0_144
5.添加節(jié)點(diǎn)
修改workers，添加三個(gè)節(jié)點(diǎn)
node1
node2
node3
6.修改? core-site.xml
<property>
? <name>fs.defaultFS</name>
? <value>hdfs://mycluster</value>
</property>
<!-- 指定hadoop運(yùn)行時(shí)產(chǎn)生文件的存儲(chǔ)目錄 -->
<property>
<name>hadoop.tmp.dir</name>
<value>/home/admin/modules/hadoop-3.1.3/hadoop-data</value>
</property>
<!--? 監(jiān)測(cè)集群-->
? <property>
? <name>ha.zookeeper.quorum</name>
? <value>node1:2181,node2:2181,node3:2181</value>
</property>
修改hdfs-site.xml
<!-- 副本數(shù)量-->
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
? <name>dfs.nameservices</name>
? <value>mycluster</value>
</property>
<property>
? <name>dfs.ha.namenodes.mycluster</name>
? <value>nn1,nn2</value>
</property>
<property>
? <name>dfs.namenode.rpc-address.mycluster.nn1</name>
? <value>node1:8020</value>
</property>
<property>
? <name>dfs.namenode.rpc-address.mycluster.nn2</name>
? <value>node2:8020</value>
</property>
<property>
? <name>dfs.namenode.http-address.mycluster.nn1</name>
? <value>node1:9870</value>
</property>
<property>
? <name>dfs.namenode.http-address.mycluster.nn2</name>
? <value>node2:9870</value>
</property>
<!--? 部署journalnode集群的目錄 -->
<property>
? <name>dfs.namenode.shared.edits.dir</name>
? <value>qjournal://node1:8485;node2:8485;node3:8485/mycluster</value>
</property>
<property>
? <name>dfs.client.failover.proxy.provider.mycluster</name>
? <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<!--免密鑰，和隔離 -->
? ? <property>
? ? ? <name>dfs.ha.fencing.methods</name>
? ? ? <value>sshfence</value>
? ? </property>
? ? <property>
? ? ? <name>dfs.ha.fencing.ssh.private-key-files</name>
? ? ? <value>/root/.ssh/id_rsa</value>
? ? </property>
<!--指定journalnode的數(shù)據(jù)路徑 -->
<property>
? <name>dfs.journalnode.edits.dir</name>
? <value>/home/admin/modules/hadoop-3.1.3/journalnode</value>
</property>
<!-- 自動(dòng)故障轉(zhuǎn)移-->
<property>
? <name>dfs.ha.automatic-failover.enabled</name>
? <value>true</value>
</property>
<!--關(guān)閉權(quán)限檢查-->
<property>
<name>dfs.permissions.enable</name>
<value>false</value>
</property>
<!-- 開啟web方式-->
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
</property>
修改mapred-site.xml
<!-- 指定mr運(yùn)行在yarn上 -->
<property>
<name>mapreduce.framework.name</name>
<value>yarn</value>
</property>
<!--配置map reduce的job history server地址仿粹，默認(rèn)10020-->
<property>
<name>mapreduce.jobhistory.address</name>
<value>node1:10020</value>
</property>
<!--配置map reduce job history server web ui地址，默認(rèn)19888-->
<property>
<name>mapreduce.jobhistory.webapp.address</name>
<value>node1:19888</value>
</property>
修改yarn-site.xml
<!-- reducer獲取數(shù)據(jù)的方式 -->
<property>
<name>yarn.nodemanager.aux-services</name>
<value>mapreduce_shuffle</value>
</property>
<!-- resourcemanager高可用-->
<property>
? <name>yarn.resourcemanager.ha.enabled</name>
? <value>true</value>
</property>
<property>
? <name>yarn.resourcemanager.cluster-id</name>
? <value>cluster1</value>
</property>
<property>
? <name>yarn.resourcemanager.ha.rm-ids</name>
? <value>rm1,rm2</value>
</property>
<!-- 指定YARN的ResourceManager的地址 -->
<property>
? <name>yarn.resourcemanager.hostname.rm1</name>
? <value>node2</value>
</property>
<property>
? <name>yarn.resourcemanager.hostname.rm2</name>
? <value>node3</value>
? </property>
? <property>
? <name>yarn.resourcemanager.webapp.address.rm1</name>
? <value>node2:8088</value>
</property>
<property>
? <name>yarn.resourcemanager.webapp.address.rm2</name>
? <value>node3:8088</value>
</property>
? <!-- zookeeper 地址 -->
? <property>
? <name>yarn.resourcemanager.zk-address</name>
? <value>node1:2181,node2:2181,node3:2181</value>
</property>
<!--日志 -->
<property>
<name>yarn.log-aggregation-enable</name>
<value>true</value>
? </property>
? <property>
<name>yarn.log-aggregation.retain-seconds</name>
<value>86400</value>
? </property>
? <property>
<name>yarn.log.server.url</name>
<value>http://node1:19888/jobhistory/logs/</value>
? </property>
7.修改sbin下面的
start（stop）-dfs.sh,在最開頭添加如下
HDFS_DATANODE_USER=root
HDFS_DATANODE_SECURE_USER=hdfs
HDFS_NAMENODE_USER=root
HDFS_JOURNALNODE_USER=root
HDFS_ZKFC_USER=root
start(stop)-yarn.sh,在最開頭添加如下
YARN_RESOURCEMANAGER_USER=root
HADOOP_SECURE_DN_USER=yarn
YARN_NODEMANAGER_USER=root
8.分發(fā)Hadoop原茅，zookeeper到各個(gè)節(jié)點(diǎn)
9.每個(gè)節(jié)點(diǎn)啟動(dòng)zookeeper
zkServer.sh start
10.每個(gè)節(jié)點(diǎn)啟動(dòng)journalnode
hdfs --daemon start journalnode
11. 格式化主namenode
hdfs namenode -format
12.啟動(dòng)主namenode
啟動(dòng)主namenoe hdfs --daemon start namenode
13.從namenode同步主namenode
hdfs namenode -bootstrapStandby 同步主節(jié)點(diǎn)
14.啟動(dòng)從namenode
namenoe hdfs --daemon start namenode
15.初始化zookeeper
在主節(jié)點(diǎn) hdfs zkfc -formatZK
16.主節(jié)點(diǎn)啟動(dòng)集群
sbin/start-dfs.sh
17.查看進(jìn)程
[root@node1 hadoop-3.1.3]# jps
3824 DataNode
1413 QuorumPeerMain
4422 NodeManager
4729 Jps
3692 NameNode
4108 JournalNode
4287 DFSZKFailoverController
18.啟動(dòng)resource manager 高可用
sbin/start-yarn.sh????
查看進(jìn)程
[root@node2 hadoop-3.1.3]# jps
1713 NameNode
3971 Jps
1571 JournalNode
3350 NodeManager
1254 QuorumPeerMain
2438 DataNode
2606 DFSZKFailoverController
3231 ResourceManager
至此集群搭建完吭历。
訪問(wèn)9870：
完美搭建！
謝謝大家~~~~~
