大数据平台实时数仓从0到1搭建之 - 08 集群脚本

发布于2021-10-05 12:57 阅读(442) 评论(0) 点赞(12) 收藏(2)

大数据平台实时数仓从0到1搭建之 - 08 集群脚本

概述
环境变量
jps & jpsall.sh
zkServer.sh & zk.sh
kafka-server-start & kafka.sh
测试
完成

概述

目前涉及到的zookeeper和kafka，都是需要在每台机器上做启动停止操作，
本文使用shell脚本，处理集群的操作
重点语法：ssh host 命令
在对应的host节点上执行某个命令

环境变量

需要修改对应的用户的环境变量，
使用root用户执行脚本，则需要在/root/.bashrc中添加JAVA_HOME和PATH
每个节点都需要配置

eg.server110

[root@server110 ~]# vim /root/.bashrc
JAVA_HOME=/opt/modules/jdk1.8.0_181
PATH=$PATH:$JAVA_HOME/bin
export JAVA_HOME PATH

jps & jpsall.sh

jps命令是java里用来查看java进程的工具，但是只能查看当前节点的进程情况
思路就是循环所有节点，将jps命令发送到对应的节点上执行，脚本如下

[root@server110 opt]# vim jpsall.sh
#!/bin/bash
for i in server110 server111 server112
do
        echo ----------------jps $i --------------------
        ssh $i "jps"
done

zkServer.sh & zk.sh

思路：
1、循环所有节点
2、根据传入的参数，执行对应的命令，start | stop | status

[root@server110 opt]# vim zk.sh 
#!/bin/bash
case $1 in
"start")
{
        for s in server110 server111 server112
        do 
                ssh $s "/opt/modules/zookeeper-3.5.8/bin/zkServer.sh start"
                echo ----------$s zookeeper start-----------------             
        done
}
;;
"stop")
{
        for s in server110 server111 server112
        do
                ssh $s  "/opt/modules/zookeeper-3.5.8/bin/zkServer.sh stop"
                echo ----------$s zookeeper stop-----------------
        done
}
;;
"status")
{
        for s in server120 server121 server122
        do
                echo ----------$s zookeeper status----------------
                ssh $s  "/opt/modules/zookeeper-3.5.8/bin/zkServer.sh status"
        done
}
;;
*)
{
        echo "------./zk.sh start|stop|status-----"
}
;;
esac

kafka-server-start & kafka.sh

思路：
1、循环所有节点
2、根据传入的参数，执行对应的命令，start | stop

[root@server110 opt]# vim kafka.sh
#!/bin/bash
case $1 in
"start")
{
        for i in server110 server111 server112
        do
                ssh $i "/opt/modules/kafka_2.11-2.4.1/bin/kafka-server-start.sh -daemon /opt/modules/kafka_2.11-2.4.1/config/server.properties"
                echo --------kafka $i start------------------
        done
}
;;
"stop")
{
        for i in server110 server111 server112
        do
                ssh $i "/opt/modules/kafka_2.11-2.4.1/bin/kafka-server-stop.sh  /opt/modules/kafka_2.11-2.4.1/config/server.properties"
                echo --------kafka $i stop------------------
        done
}
;;
*)
{
        echo "------./kafka.sh start|stop-----"
}
;;
esac

测试

jpsall.sh

[root@server110 opt]# ./jpsall.sh 
----------------jps server110 --------------------
30802 Jps
26823 QuorumPeerMain
17387 NameNode
17916 NodeManager
29500 Kafka
17549 DataNode
----------------jps server111 --------------------
17265 DataNode
17540 NodeManager
128820 QuorumPeerMain
130404 Jps
129287 Kafka
17383 ResourceManager
129638 ConsoleProducer
18029 JobHistoryServer
----------------jps server112 --------------------
24642 ConsoleConsumer
17207 DataNode
24264 Kafka
17289 SecondaryNameNode
25436 Jps
23773 QuorumPeerMain
17407 NodeManager

kafka.sh

stop

[root@server110 opt]# ./kafka.sh stop
--------kafka server110 stop------------------
--------kafka server111 stop------------------
--------kafka server112 stop------------------
[root@server110 opt]# ./jpsall.sh 
----------------jps server110 --------------------
26823 QuorumPeerMain
17387 NameNode
31067 Jps
17916 NodeManager
17549 DataNode
----------------jps server111 --------------------
17265 DataNode
17540 NodeManager
128820 QuorumPeerMain
130644 Jps
17383 ResourceManager
18029 JobHistoryServer
----------------jps server112 --------------------
17207 DataNode
17289 SecondaryNameNode
23773 QuorumPeerMain
25662 Jps
17407 NodeManager

start

[root@server110 opt]# ./kafka.sh start
--------kafka server110 start------------------
--------kafka server111 start------------------
--------kafka server112 start------------------
[root@server110 opt]# ./jpsall.sh 
----------------jps server110 --------------------
26823 QuorumPeerMain
31450 Kafka
17387 NameNode
17916 NodeManager
17549 DataNode
31534 Jps
----------------jps server111 --------------------
17265 DataNode
17540 NodeManager
128820 QuorumPeerMain
131028 Kafka
17383 ResourceManager
18029 JobHistoryServer
334 Jps
----------------jps server112 --------------------
17207 DataNode
17289 SecondaryNameNode
23773 QuorumPeerMain
26046 Kafka
26110 Jps
17407 NodeManager

zk.sh

stop

[root@server110 opt]# ./zk.sh stop
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.5.8/bin/../conf/zoo.cfg
Stopping zookeeper ... STOPPED
----------server110 zookeeper stop-----------------
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.5.8/bin/../conf/zoo.cfg
Stopping zookeeper ... STOPPED
----------server111 zookeeper stop-----------------
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.5.8/bin/../conf/zoo.cfg
Stopping zookeeper ... STOPPED
----------server112 zookeeper stop-----------------
[root@server110 opt]# ./jpsall.sh 
----------------jps server110 --------------------
17387 NameNode
17916 NodeManager
17549 DataNode
31679 Jps
----------------jps server111 --------------------
17265 DataNode
17540 NodeManager
470 Jps
17383 ResourceManager
18029 JobHistoryServer
----------------jps server112 --------------------
26276 Jps
17207 DataNode
17289 SecondaryNameNode
17407 NodeManager

start

[root@server110 opt]# ./zk.sh start
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.5.8/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
----------server110 zookeeper start-----------------
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.5.8/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
----------server111 zookeeper start-----------------
ZooKeeper JMX enabled by default
Using config: /opt/modules/zookeeper-3.5.8/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
----------server112 zookeeper start-----------------
[root@server110 opt]# ./jpsall.sh 
----------------jps server110 --------------------
31744 QuorumPeerMain
17387 NameNode
17916 NodeManager
17549 DataNode
31805 Jps
----------------jps server111 --------------------
17265 DataNode
595 Jps
17540 NodeManager
532 QuorumPeerMain
17383 ResourceManager
18029 JobHistoryServer
----------------jps server112 --------------------
26338 QuorumPeerMain
17207 DataNode
17289 SecondaryNameNode
26394 Jps
17407 NodeManager