Redis哨兵集群创建脚本–v2

1. 基础环境

         操作系统版本  CentOS Linux release 7.6.1810 (Core)

         Docker 版本  19.03.11, build 42e35e61f3

         Redis  版本  3.2.3-alpine

2. 效果展示

  

 3. 脚本参数解释

cmd                     定义redis容器启动命令
sentinel_cmd            定义sentinel容器启动命令
image                  容器启动镜像
redis_conf             redis容器配置文件路径
sentinel_conf           sentinel容器配置文件路径
redis_quota             redis容器cpu、内存配额
sentinel_quota        sentinel容器cpu、内存配额
base_dir                  脚本运行的家目录
base_conf_dir         redis配置文件目录
record_dir               集群配置记录目录
log_dir                    脚本日志目录
bak_dir                   备份目录
ipmark                     可用IP记录文件
ipmark_assigned     已用IP记录文件
op_cmd                   脚本命令参数
envv                        脚本环境参数
cluster_name          脚本集群名参数
cluster_port            脚本集群端口参数

 4. 脚本运行说明

  a. 脚本中,包含了ceph rdb 的创建、删除功能,如不需要,可以注释

   b. docker网络,使用macvlan,在执行脚本的初始化时,需要针对本地的实际情况,进行修改

  c. redis 配置文件,可以在 init_cluster 中修改

   d. 创建集群前,先执行初始化命令

   e. 目录结构如下图:

      

 

 

5. 脚本内容 (内容冗余,未做优化)

 

#!/bin/bash
#Author:Jixson
#Date: 2020-07-07
cmd="/usr/local/bin/redis-server /usr/local/etc/redis/redis.conf"
sentinel_cmd="/usr/local/bin/redis-sentinel /usr/local/etc/redis/sentinel.conf"
image="redis:latest"
redis_conf="/usr/local/etc/redis/redis.conf"
sentinel_conf="/usr/local/etc/redis/sentinel.conf"
redis_quota="-c2 -m 4g"
sentinel_quota="-c2 -m 1g"
base_dir="/opt/redis"
base_conf_dir="${base_dir}/conf"
record_dir="${base_dir}/record"
log_dir="${base_dir}/logs"
bak_dir="${base_dir}/backup"
ipmark="${base_dir}/ipmark"
ipmark_assigned="${base_dir}/ipmark_assigned"
op_cmd=$1
envv=$2
cluster_name=$3
cluster_port=$4

Color_Text()
{
  echo -e " \e[0;$2m$1\e[0m"
}
Echo_Yellow()
{
  echo "$(Color_Text "$1" "33")"
}
Echo_Red()
{
  echo "$(Color_Text "$1" "31")"
}
Echo_Green()
{
  echo "$(Color_Text "$1" "32")"
}
write_log()
{
	mkdir -p ${log_dir} 
	local log_type=$1
	local log_msg=$2
	local log_name="${log_dir}/redis-shell.log"
	echo "`date +"%Y-%m-%d %H:%M:%S"` $log_type $log_msg " >> $log_name
}
init_cluster()
{
	if [ -d ${base_dir} ];then
		mv ${base_dir} ${base_dir}.bak
		write_log info "已备份 原${base_dir} 为 ${base_dir}.bak "
		Echo_Yellow " 已备份 原${base_dir} 为 ${base_dir}.bak ."
	fi
	mkdir -p ${base_conf_dir}
	write_log info "${base_conf_dir} 已创建 ."
	Echo_Yellow " ${base_conf_dir} 已创建 ."
	mkdir -p ${record_dir}
	write_log info "${record_dir} 已创建 ."
	Echo_Yellow " ${record_dir} 已创建 ."
	mkdir -p ${log_dir}
	write_log info "${log_dir} 已创建 ."
	Echo_Yellow " ${log_dir} 已创建 ."
	mkdir -p ${bak_dir}
	write_log info "${bak_dir} 已创建 ."
	Echo_Yellow " ${bak_dir} 已创建 ."
	cat << EOF > ${base_conf_dir}/sentinel.conf
port \$cluster_port
daemonize no
logfile "/data/sentinel.log"
protected-mode no
sentinel monitor \$cluster_name redis-master.xxx.com 6379 2
sentinel down-after-milliseconds \$cluster_name 15000
EOF
	write_log info "Sentinel 配置文件已经初始化 ."
	Echo_Yellow " Sentinel 配置文件已经初始化 ."
	cat << EOF > ${base_conf_dir}/redis-master.conf
bind 0.0.0.0
port 6379
pidfile "/data/redis_6379.pid"
logfile "/data/redis_6379.log"
dir "/data"
dbfilename "redis_6379.rdb"
maxmemory 1gb
appendfilename "redis_6379.aof"
protected-mode yes
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize no
supervised no
loglevel notice
databases 16
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync yes
repl-diskless-sync-delay 5
repl-ping-slave-period 10
repl-timeout 60
repl-disable-tcp-nodelay no
repl-backlog-size 150mb
repl-backlog-ttl 3600
slave-priority 100
maxclients 10000
maxmemory-policy allkeys-lru
appendonly no
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
EOF
	write_log info "Redis-master 配置文件已经初始化 ."
	Echo_Yellow " Redis-master 配置文件已经初始化"
	\cp ${base_conf_dir}/redis-master.conf ${base_conf_dir}/redis-slave.conf
	echo "slaveof redis-master.xxx.com 6379" >> ${base_conf_dir}/redis-slave.conf
	write_log info "Redis-slave 配置文件已经初始化 ."
	Echo_Yellow " Redis-slave 配置文件已经初始化"
}
init_network()
{
	docker network create -d macvlan --subnet=10.6.88.0/24 --gateway=10.6.88.1 --ip-range=10.6.88.64/27 -o parent=eth0 sentinel-net
	for i in $(seq 64 95)
	do
		echo "10.6.88.$i" >> $ipmark
	done
}
envv_true_check()
{
	local envv_dir="${record_dir}/${envv}"
	if [ -d ${envv_dir} ];then
		Echo_Red "    哨兵集群 ${envv} 已存在 ."
		exit 1
	fi
}
envv_false_check()
{
	local envv_dir="${record_dir}/${envv}"
	if [ -d ${envv_dir} ];then
		echo '' > /dev/null
	else
		Echo_Red "    哨兵集群 ${envv} 不存在 ."
		exit 1
	fi
}
create_rbd()
{
	base_data_dir="${base_dir}/cluster-data/${envv}"
	mkdir -p ${base_data_dir}/data_sen01
	sen01=${base_data_dir}/data_sen01
	write_log info "ceph data directory ${sen01} 已创建 ."
	Echo_Yellow " ceph data directory ${sen01} 已创建 ."
	mkdir -p ${base_data_dir}/data_sen02
	sen02=${base_data_dir}/data_sen02
	write_log info "ceph data directory ${sen02} 已创建 ."
	Echo_Yellow " ceph data directory ${sen02} 已创建 ."
	mkdir -p ${base_data_dir}/data_sen03
	sen03=${base_data_dir}/data_sen03
	write_log info "ceph data directory ${sen03} 已创建 ."
	Echo_Yellow " ceph data directory ${sen03} 已创建 ."
	mkdir -p ${base_data_dir}/data_master
	master=${base_data_dir}/data_master
	write_log info "ceph data directory ${master} 已创建 ."
	Echo_Yellow " ceph data directory ${master} 已创建 ."
	mkdir -p ${base_data_dir}/data_slave
	slave=${base_data_dir}/data_slave
	write_log info "ceph data directory ${slave} 已创建 ."
	Echo_Yellow " ceph data directory ${slave} 已创建 ."

	rbd create ${envv}-sentinel01  --size 1G --image-feature layering -p rbd
	write_log info "ceph image ${envv}-sentinel01 已创建 ."
	Echo_Yellow " ceph image ${envv}-sentinel01 已创建 ."
	rbd create ${envv}-sentinel02  --size 1G --image-feature layering -p rbd
	write_log info "ceph image ${envv}-sentinel02 已创建 ."
	Echo_Yellow " ceph image ${envv}-sentinel02 已创建 ."
	rbd create ${envv}-sentinel03  --size 1G --image-feature layering -p rbd
	write_log info "ceph image ${envv}-sentinel03 已创建 ."
	Echo_Yellow " ceph image ${envv}-sentinel03 已创建 ."
	rbd create ${envv}-redismaster --size 50G --image-feature layering -p rbd
	write_log info "ceph image ${envv}-redismaster 已创建 ."
	Echo_Yellow " ceph image ${envv}-redismaster 已创建 ."
	rbd create ${envv}-redisslave  --size 50G --image-feature layering -p rbd
	write_log info "ceph image ${envv}-redisslave 已创建 ."
	Echo_Yellow " ceph image ${envv}-redisslave 已创建 ."

	data_sen01=$(rbd map ${envv}-sentinel01  --name client.admin -m 10.6.88.14,10.6.88.15,10.6.88.16)
	write_log info "ceph image map ${envv}-sentinel01 ."
	Echo_Yellow " ceph image map ${envv}-sentinel01 ."
	data_sen02=$(rbd map ${envv}-sentinel02  --name client.admin -m 10.6.88.14,10.6.88.15,10.6.88.16)
	write_log info "ceph image map ${envv}-sentinel02 ."
	Echo_Yellow " ceph image map ${envv}-sentinel02 ."
	data_sen03=$(rbd map ${envv}-sentinel03  --name client.admin -m 10.6.88.14,10.6.88.15,10.6.88.16)
	write_log info "ceph image map ${envv}-sentinel03 ."
	Echo_Yellow " ceph image map ${envv}-sentinel03 ."
	data_master=$(rbd map ${envv}-redismaster --name client.admin -m 10.6.88.14,10.6.88.15,10.6.88.16)
	write_log info "ceph image map ${envv}-redismaster ."
	Echo_Yellow " ceph image map ${envv}-redismaster ."
	data_slave=$(rbd map ${envv}-redisslave  --name client.admin -m 10.6.88.14,10.6.88.15,10.6.88.16)
	write_log info "ceph image map ${envv}-redisslave ."
	Echo_Yellow " ceph image map ${envv}-redisslave ."

	mkfs.xfs $data_sen01
	write_log info "ceph rbd ${data_sen01} 已格式化 xfs."
	Echo_Yellow " ceph rbd ${data_sen01} 已格式化 xfs."
	mkfs.xfs $data_sen02
	write_log info "ceph rbd ${data_sen02} 已格式化 xfs."
	Echo_Yellow " ceph rbd ${data_sen02} 已格式化 xfs."
	mkfs.xfs $data_sen03
	write_log info "ceph rbd ${data_sen03} 已格式化 xfs."
	Echo_Yellow " ceph rbd ${data_sen03} 已格式化 xfs."
	mkfs.xfs $data_master
	write_log info "ceph rbd ${data_master} 已格式化 xfs."
	Echo_Yellow " ceph rbd ${data_master} 已格式化 xfs."
	mkfs.xfs $data_slave
	write_log info "ceph rbd ${data_slave} 已格式化 xfs."
	Echo_Yellow " ceph rbd ${data_slave} 已格式化 xfs."

	mount -o discard $data_sen01  ${sen01}
	write_log info "ceph rbd ${data_sen01} 已挂载 ."
	Echo_Yellow " ceph rbd ${data_sen01} 已挂载 ."
	mount -o discard $data_sen02  ${sen02}
	write_log info "ceph rbd ${data_sen02} 已挂载 ."
	Echo_Yellow " ceph rbd ${data_sen02} 已挂载 ."
	mount -o discard $data_sen03  ${sen03}
	write_log info "ceph rbd ${data_sen03} 已挂载 ."
	Echo_Yellow " ceph rbd ${data_sen03} 已挂载 ."
	mount -o discard $data_master ${master}
	write_log info "ceph rbd ${data_master} 已挂载 ."
	Echo_Yellow " ceph rbd ${data_master} 已挂载 ."
	mount -o discard $data_slave  ${slave}
	write_log info "ceph rbd ${data_slave} 已挂载 ."
	Echo_Yellow " ceph rbd ${data_slave} 已挂载 ."
}
delete_rbd()
{
	base_data_dir="${base_dir}/cluster-data/${envv}"
	sen01="${base_data_dir}/data_sen01"
	sen02="${base_data_dir}/data_sen02"
	sen03="${base_data_dir}/data_sen03"
	master="${base_data_dir}/data_master"
	slave="${base_data_dir}/data_slave"

	rbd_id01=$(mount | grep "${sen01}" | awk '{print $1}')
	rbd_id02=$(mount | grep "${sen02}" | awk '{print $1}')
	rbd_id03=$(mount | grep "${sen03}" | awk '{print $1}')
	rbd_id04=$(mount | grep "${master}" | awk '{print $1}')
	rbd_id05=$(mount | grep "${slave}" | awk '{print $1}')

	umount $sen01 $sen02 $sen03 $master $slave
	write_log warn "ceph data directory $sen01 $sen02 $sen03 $master $slave 已卸载 ."
	Echo_Yellow " ceph data directory $sen01 $sen02 $sen03 $master $slave 已卸载 ."

	if [ -d ${base_data_dir} ];then
		rm -rf ${base_data_dir}
	fi
	write_log warn "ceph data directory  ${base_data_dir} 已删除 ."
	Echo_Yellow " ceph data directory ${base_data_dir} 已删除 ."

	rbd unmap $rbd_id01
	write_log warn "ceph rbd unmap ${rbd_id01} ."
	Echo_Yellow " ceph rbd unmap ${rbd_id01} ."
	rbd rm ${envv}-sentinel01
	write_log warn "ceph rbd image delete ${envv}-sentinel01 ."
	Echo_Yellow " ceph rbd image delete ${envv}-sentinel01 ."

	rbd unmap $rbd_id02
	write_log warn "ceph rbd unmap ${rbd_id02} ."
	Echo_Yellow " ceph rbd unmap ${rbd_id02} ."
	rbd rm ${envv}-sentinel02
	write_log warn "ceph rbd image delete ${envv}-sentinel02 ."
	Echo_Yellow " ceph rbd image delete ${envv}-sentinel02 ."

	rbd unmap $rbd_id03
	write_log warn "ceph rbd unmap ${rbd_id03} ."
	Echo_Yellow " ceph rbd unmap ${rbd_id03} ."
	rbd rm ${envv}-sentinel03
	write_log warn "ceph rbd image delete ${envv}-sentinel03 ."
	Echo_Yellow " ceph rbd image delete ${envv}-sentinel03 ."

	rbd unmap $rbd_id04
	write_log warn "ceph rbd unmap ${rbd_id04} ."
	Echo_Yellow " ceph rbd unmap ${rbd_id04} ."
	rbd rm ${envv}-redismaster
	write_log warn "ceph rbd image delete ${envv}-redismaster ."
	Echo_Yellow " ceph rbd image delete ${envv}-redismaster ."

	rbd unmap $rbd_id05
	write_log warn "ceph rbd unmap ${rbd_id05} ."
	Echo_Yellow " ceph rbd unmap ${rbd_id05} ."
	rbd rm ${envv}-redisslave
	write_log warn "ceph rbd image delete ${envv}-redisslave ."
	Echo_Yellow " ceph rbd image delete ${envv}-redisslave ."

}
create_usage()
{
	Echo_Yellow " Usage : "
	Echo_Red    "      集群创建"
	Echo_Yellow "      $0 create [environment] [cluster_name] [cluster_port]"
	Echo_Yellow "      [environment]  集群所属环境标识"
	Echo_Yellow "      [cluster_name] 哨兵集群 monitor name"
	Echo_Yellow "      [cluster_port] 哨兵集群对外端口"
	Echo_Yellow "      E.g.: $0 create leo bytest 27001"
}
delete_usage()
{
	Echo_Yellow " Usage : "
	Echo_Red    "      集群删除"
	Echo_Yellow "      $0 delete [environment]"
	Echo_Yellow "      E.g.: $0 delete leo"
}
stop_usage()
{
	Echo_Yellow " Usage : "
	Echo_Red    "      停止集群"
	Echo_Yellow "      $0 stop [environment]"
	Echo_Yellow "      E.g.: $0 stop leo"
}
start_usage()
{
	Echo_Yellow " Usage : "
	Echo_Red    "      启动集群"
	Echo_Yellow "      $0 start [environment]"
	Echo_Yellow "      E.g.: $0 start leo"
}
restart_usage()
{
	Echo_Yellow " Usage : "
	Echo_Red    "      重启集群"
	Echo_Yellow "      $0 restart [environment]"
	Echo_Yellow "      E.g.: $0 restart leo"
}
backup_usage()
{
	Echo_Yellow " Usage : "
	Echo_Red    "      备份集群"
	Echo_Yellow "      $0 backup [environment]"
	Echo_Yellow "      E.g.: $0 backup leo"
}
show_usage()
{
	Echo_Yellow " Usage : "
	Echo_Red    "      显示集群运行信息"
	Echo_Yellow "      $0 show [environment]"
	Echo_Yellow "      E.g.: $0 show leo"
}
ls_usage()
{
	Echo_Yellow " Usage : "
	Echo_Red    "      显示已创建集群"
	Echo_Yellow "      $0 ls"
	Echo_Yellow "      E.g.: $0 ls"
}
install_usage()
{
	Echo_Yellow " Usage : "
	Echo_Red    "      脚本初始化"
	Echo_Yellow "      $0 \"install --yes --yes\""
	Echo_Yellow "      E.g.: $0 \"install --yes --yes\""
}
create_cluster()
{
	envv_true_check
	mkdir -p $record_dir/${envv}
	#Get IP address
	master_ip=$(cat $ipmark | head -1)
	sed -i "/^${master_ip}/d" $ipmark
	echo "${master_ip} assigned $envv redis-master" >> $ipmark_assigned
	write_log info "已分配 哨兵集群 ${envv} Redis-master IP: ${master_ip}"
	Echo_Yellow " 已分配 哨兵集群 ${envv} Redis-master IP: ${master_ip}"

	slave_ip=$(cat $ipmark | head -1)
	sed -i "/^${slave_ip}/d" $ipmark
	echo "${slave_ip} assigned $envv redis-slave" >> $ipmark_assigned
	write_log info "已分配 哨兵集群 ${envv} redis-slave IP: ${slave_ip}"
	Echo_Yellow " 已分配 哨兵集群 ${envv} redis-slave IP: ${slave_ip}"

	sentinel_ip01=$(cat $ipmark | head -1)
	sed -i "/^${sentinel_ip01}/d" $ipmark
	echo "${sentinel_ip01} assigned $envv sentinel01" >> $ipmark_assigned
	write_log info "已分配 哨兵集群 ${envv} Sentinel01 IP: ${sentinel_ip01}"
	Echo_Yellow " 已分配 哨兵集群 ${envv} Sentinel01 IP: ${sentinel_ip01}"

	sentinel_ip02=$(cat $ipmark | head -1)
	sed -i "/^${sentinel_ip02}/d" $ipmark
	echo "${sentinel_ip02} assigned $envv sentinel02" >> $ipmark_assigned
	write_log info "已分配 哨兵集群 ${envv} Sentinel02 IP: ${sentinel_ip02}"
	Echo_Yellow " 已分配 哨兵集群 ${envv} Sentinel02 IP: ${sentinel_ip02}"

	sentinel_ip03=$(cat $ipmark | head -1)
	sed -i "/^${sentinel_ip03}/d" $ipmark
	echo "${sentinel_ip03} assigned $envv sentinel03" >> $ipmark_assigned
	write_log info "已分配 哨兵集群 ${envv} Sentinel03 IP: ${sentinel_ip03}"
	Echo_Yellow " 已分配 哨兵集群 ${envv} Sentinel03 IP: ${sentinel_ip03}"
	#ceph create rbd 
	create_rbd
	mountsen01="-v ${sen01}:/data"
	mountsen02="-v ${sen02}:/data"
	mountsen03="-v ${sen03}:/data"
	mountmaster="-v ${master}:/data"
	mountslave="-v ${slave}:/data"
	#声明主机
	hn1="--add-host=redis-master.xxx.com:${master_ip}"
	hn2="--add-host=redis-slave.xxx.com:${slave_ip}"
	hn3="--add-host=sentinel01.xxx.com:${sentinel_ip01}"
	hn4="--add-host=sentinel02.xxx.com:${sentinel_ip02}"
	hn5="--add-host=sentinel03.xxx.com:${sentinel_ip03}"
	hosts="$hn1 $hn2 $hn3 $hn4 $hn5"

	#master conf
	\cp -f "${base_conf_dir}/redis-master.conf" "${record_dir}/${envv}/redis-master.conf"
	master_conf="${record_dir}/${envv}/redis-master.conf"
	write_log info "已创建 哨兵集群 ${envv} Redis-master 配置文件 ${master_conf}"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Redis-master 配置文件 ${master_conf}"
	masterconf="-v ${master_conf}:${redis_conf}"

	#slave conf
	\cp -f "${base_conf_dir}/redis-slave.conf" "${record_dir}/${envv}/redis-slave.conf"
	slave_conf="${record_dir}/${envv}/redis-slave.conf"
	write_log info "已创建 哨兵集群 ${envv} Redis-slave 配置文件 ${slave_conf}"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Redis-slave 配置文件 ${slave_conf}"
	slaveconf="-v ${slave_conf}:${redis_conf}"

	#sentinel conf
	\cp -f "${base_conf_dir}/sentinel.conf" "${record_dir}/${envv}/sentinel01.conf"
	sentinel01_conf="${record_dir}/${envv}/sentinel01.conf"
	sed -i 's/\$cluster_name/'${cluster_name}'/g' ${sentinel01_conf}
	sed -i 's/\$cluster_port/'${cluster_port}'/g' ${sentinel01_conf}
	write_log info "已创建 哨兵集群 ${envv} Sentinel01 配置文件 ${sentinel01_conf}"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Sentinel01 配置文件 ${sentinel01_conf}"
	sentinel01conf="-v ${sentinel01_conf}:${sentinel_conf}"

	\cp -f "${base_conf_dir}/sentinel.conf" "${record_dir}/${envv}/sentinel02.conf"
	sentinel02_conf="${record_dir}/${envv}/sentinel02.conf"
	sed -i 's/\$cluster_name/'${cluster_name}'/g' ${sentinel02_conf}
	sed -i 's/\$cluster_port/'${cluster_port}'/g' ${sentinel02_conf}
	write_log info "已创建 哨兵集群 ${envv} Sentinel02 配置文件 ${sentinel02_conf}"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Sentinel02 配置文件 ${sentinel02_conf}"
	sentinel02conf="-v ${sentinel02_conf}:${sentinel_conf}"

	\cp -f "${base_conf_dir}/sentinel.conf" "${record_dir}/${envv}/sentinel03.conf"
	sentinel03_conf="${record_dir}/${envv}/sentinel03.conf"
	sed -i 's/\$cluster_name/'${cluster_name}'/g' ${sentinel03_conf}
	sed -i 's/\$cluster_port/'${cluster_port}'/g' ${sentinel03_conf}
	write_log info "已创建 哨兵集群 ${envv} Sentinel03 配置文件 ${sentinel03_conf}"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Sentinel03 配置文件 ${sentinel03_conf}"
	sentinel03conf="-v ${sentinel03_conf}:${sentinel_conf}"

	#run redis
	docker run -d --network sentinel-net --ip=$master_ip $redis_quota $hosts $mountmaster --name ${envv}-redis-master ${masterconf} ${image} ${cmd}
	write_log info "已创建 哨兵集群 ${envv} Redis-master 节点"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Redis-master 节点"
	write_log info "创建命令: docker run -d --network sentinel-net --ip=$master_ip $redis_quota $hosts $mountmaster --name ${envv}-redis-master ${masterconf} ${image} ${cmd}"
	
	docker run -d --network sentinel-net --ip=$slave_ip  $redis_quota $hosts $mountslave --name ${envv}-redis-slave  ${slaveconf}  ${image} ${cmd}
	write_log info "已创建 哨兵集群 ${envv} Redis-slave 节点"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Redis-slave 节点"
	write_log info "创建命令: docker run -d --network sentinel-net --ip=$slave_ip  $redis_quota $hosts $mountslave --name ${envv}-redis-slave  ${slaveconf}  ${image} ${cmd}"
	#run sentinel
	docker run -d --network sentinel-net --ip=$sentinel_ip01 $sentinel_quota $hosts $mountsen01 --name ${envv}-sentinel01 ${sentinel01conf} ${image} ${sentinel_cmd}
	write_log info "已创建 哨兵集群 ${envv} Sentinel01 节点"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Sentinel01 节点"
	write_log info "创建命令: docker run -d --network sentinel-net --ip=$sentinel_ip01 $sentinel_quota $hosts $mountsen01 --name ${envv}-sentinel01 ${sentinel01conf} ${image} ${sentinel_cmd}"

	docker run -d --network sentinel-net --ip=$sentinel_ip02 $sentinel_quota $hosts $mountsen02 --name ${envv}-sentinel02 ${sentinel02conf} ${image} ${sentinel_cmd}
	write_log info "已创建 哨兵集群 ${envv} Sentinel02 节点"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Sentinel02 节点"
	write_log info "创建命令: docker run -d --network sentinel-net --ip=$sentinel_ip02 $sentinel_quota $hosts $mountsen02 --name ${envv}-sentinel02 ${sentinel02conf} ${image} ${sentinel_cmd}"

	docker run -d --network sentinel-net --ip=$sentinel_ip03 $sentinel_quota $hosts $mountsen03 --name ${envv}-sentinel03 ${sentinel03conf} ${image} ${sentinel_cmd}
	write_log info "已创建 哨兵集群 ${envv} Sentinel03 节点"
	Echo_Yellow " 已创建 哨兵集群 ${envv} Sentinel03 节点"
	write_log info "创建命令: docker run -d --network sentinel-net --ip=$sentinel_ip03 $sentinel_quota $hosts $mountsen03 --name ${envv}-sentinel03 ${sentinel03conf} ${image} ${sentinel_cmd}"

	write_log info "哨兵集群 ${envv} 已创建 ."
	Echo_Yellow " 哨兵集群 ${envv} 已创建 ."
}
stop_cluster()
{	
	envv_false_check
	docker stop ${envv}-sentinel01
	write_log warn "哨兵集群 ${envv} sentinel01 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel01 已停止 ."
	docker stop ${envv}-sentinel02
	write_log warn "哨兵集群 ${envv} sentinel02 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel02 已停止 ."
	docker stop ${envv}-sentinel03
	write_log warn "哨兵集群 ${envv} sentinel03 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel03 已停止 ."
	docker stop ${envv}-redis-master
	write_log warn "哨兵集群 ${envv} redis-master 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-master 已停止 ."
	docker stop ${envv}-redis-slave
	write_log warn "哨兵集群 ${envv} redis-slave 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-slave 已停止 ."
	write_log warn "哨兵集群 ${envv} 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} 已停止 ."
}
delete_ip()
{
	cat $ipmark_assigned | grep " ${envv} "  | awk '{print $1}' | while read line 
	do
		sed -i "/^${line}/d" $ipmark_assigned
		echo "${line}" >> $ipmark
	done
	write_log warn "哨兵集群 ${envv} IP 资源已回收 ."
	Echo_Yellow " 哨兵集群 ${envv} IP 资源已回收 ."
}
delete_cluster()
{
	envv_false_check
	delete_ip
	rm -rf ${record_dir}/${envv}/
	write_log warn "哨兵集群 ${envv} 配置文件已删除 ."
	Echo_Yellow " 哨兵集群 ${envv} 配置文件已删除 ."
	docker stop ${envv}-sentinel01
	write_log warn "哨兵集群 ${envv} sentinel01 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel01 已停止 ."
	docker stop ${envv}-sentinel02
	write_log warn "哨兵集群 ${envv} sentinel02 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel02 已停止 ."
	docker stop ${envv}-sentinel03
	write_log warn "哨兵集群 ${envv} sentinel03 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel03 已停止 ."
	docker stop ${envv}-redis-master
	write_log warn "哨兵集群 ${envv} redis-master 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-master 已停止 ."
	docker stop ${envv}-redis-slave
	write_log warn "哨兵集群 ${envv} redis-slave 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-slave 已停止 ."
	write_log warn "哨兵集群 ${envv} 已停止 ."
	Echo_Yellow " 哨兵集群 ${envv} 已停止 ."
	docker rm ${envv}-sentinel01
	write_log warn "哨兵集群 ${envv} sentinel01 已删除 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel01 已删除 ."
	docker rm ${envv}-sentinel02
	write_log warn "哨兵集群 ${envv} sentinel02 已删除 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel02 已删除 ."
	docker rm ${envv}-sentinel03
	write_log warn "哨兵集群 ${envv} sentinel03 已删除 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel03 已删除 ."
	docker rm ${envv}-redis-master
	write_log warn "哨兵集群 ${envv} redis-master 已删除 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-master 已删除 ."
	docker rm ${envv}-redis-slave
	write_log warn "哨兵集群 ${envv} redis-slave 已删除 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-slave 已删除 ."
	#ceph rbd delete 
	delete_rbd
	write_log warn "哨兵集群 ${envv} 已删除 ."
	Echo_Yellow " 哨兵集群 ${envv} 已删除 ."
}
restart_cluster()
{
	envv_false_check
	docker restart ${envv}-redis-master
	write_log warn "哨兵集群 ${envv} redis-master 已重启 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-master 已重启 ."
	docker restart ${envv}-redis-slave
	write_log warn "哨兵集群 ${envv} redis-slave 已重启 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-slave 已重启 ."
	docker restart ${envv}-sentinel01
	write_log warn "哨兵集群 ${envv} sentinel01 已重启 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel01 已重启 ."
	docker restart ${envv}-sentinel02
	write_log warn "哨兵集群 ${envv} sentinel02 已重启 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel02 已重启 ."
	docker restart ${envv}-sentinel03
	write_log warn "哨兵集群 ${envv} sentinel03 已重启 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel03 已重启 ."
	write_log warn "哨兵集群 ${envv} 已重启 ."
	Echo_Yellow " 哨兵集群 ${envv} 已重启 ."
}
start_cluster()
{
	envv_false_check
	docker start ${envv}-redis-master
	write_log info "哨兵集群 ${envv} redis-master 已启动 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-master 已启动 ."
	docker start ${envv}-redis-slave
	write_log info "哨兵集群 ${envv} redis-slave 已启动 ."
	Echo_Yellow " 哨兵集群 ${envv} redis-slave 已启动 ."
	docker start ${envv}-sentinel01
	write_log info "哨兵集群 ${envv} sentinel01 已启动 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel01 已启动 ."
	docker start ${envv}-sentinel02
	write_log info "哨兵集群 ${envv} sentinel02 已启动 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel02 已启动 ."
	docker start ${envv}-sentinel03
	write_log info "哨兵集群 ${envv} sentinel03 已启动 ."
	Echo_Yellow " 哨兵集群 ${envv} sentinel03 已启动 ."
	write_log info "哨兵集群 ${envv} 已启动 ."
	Echo_Yellow " 哨兵集群 ${envv} 已启动 ."
}
ls_cluster()
{
	local var=$(ls ${record_dir})
	if [[ -z ${var} ]];then
		Echo_Red " 目前还未创建任何哨兵集群 , 请先创建."
		create_usage
		exit 1
	else
		Echo_Yellow " 已创建的哨兵集群:"
		ls ${record_dir} | xargs -n1 | while read line
		do
		Echo_Green "      $line"
		done
		docker ps | grep "redis:latest" | awk '{print $NF}' | grep -v NAMES  | awk -F '-' '{print $1}' | uniq | while read line
		do
		        Echo_Yellow " 正在运行哨兵集群: $line"
		        docker ps  | awk '{print $NF}' | grep -v NAMES | grep "$line-" > /tmp/conlist
		        cat /tmp/conlist |xargs -n1 | while read line
		        do
		        	Echo_Green  "      $line"
		        done
		        rm -rf /tmp/conlist
		done
	fi
}
show_cluster()
{
	envv_false_check
	docker ps  | awk '{print $NF}' | grep -v NAMES | grep "${envv}-" > /tmp/conlist
	Echo_Yellow " 哨兵集群 ${envv} 节点如下:"
	cat /tmp/conlist |xargs -n1 | while read line
	do
		Echo_Green  "      $line"
	done
	rm -rf /tmp/conlist
	
}
bak_cluster()
{
	envv_false_check
	now_time=$(date +%Y%m%d%H%M)
	backupdir="${bak_dir}/${envv}.${now_time}"
	mkdir -p ${backupdir}
	write_log info "已创建 哨兵集群 ${envv} 备份目录: ${backupdir}"
	Echo_Yellow " 已创建 哨兵集群 ${envv} 备份目录: ${backupdir}"
	\cp -rf ${record_dir}/${envv}/*.conf ${backupdir}/
	write_log info "已备份 哨兵集群 ${envv} 配置文件 ."
	Echo_Yellow  " 已备份 哨兵集群 ${envv} 配置文件 ."
	docker cp ${envv}-sentinel01:/data ${backupdir}/sentinel01_data
	write_log info "已备份 哨兵集群 ${envv} sentinel01_data ."
	Echo_Yellow  " 已备份 哨兵集群 ${envv} sentinel01_data ."
	docker cp ${envv}-sentinel02:/data ${backupdir}/sentinel02_data
	write_log info "已备份 哨兵集群 ${envv} sentinel02_data ."
	Echo_Yellow  " 已备份 哨兵集群 ${envv} sentinel02_data ."
	docker cp ${envv}-sentinel03:/data ${backupdir}/sentinel03_data
	write_log info "已备份 哨兵集群 ${envv} sentinel03_data ."
	Echo_Yellow " 已备份 哨兵集群 ${envv} sentinel03_data ."
	docker cp ${envv}-redis-master:/data ${backupdir}/redis-master_data
	write_log info "已备份 哨兵集群 ${envv} master_data ."
	Echo_Yellow " 已备份 哨兵集群 ${envv} master_data ."
	docker cp ${envv}-redis-slave:/data ${backupdir}/redis-slave_data
	write_log info "已备份 哨兵集群 ${envv} slave_data ."
	Echo_Yellow " 已备份 哨兵集群 ${envv} slave_data ."
	Echo_Yellow " 哨兵集群 ${envv} 配置文件和数据目录已备份"
}


case $op_cmd in 
	create)
		if [[ -z ${envv} ]] || [[ -z ${cluster_port} ]] || [[ -z ${cluster_name} ]] ;then
			create_usage
		else
			create_cluster
		fi
	;;
	delete)
		if [ -z ${envv} ];then
			delete_usage
		else
			delete_cluster
		fi
	;;
	restart)
		if [ -z ${envv} ];then
			restart_usage
		else
			restart_cluster
		fi
	;;
	stop)
		if [ -z ${envv} ];then
			stop_usage
		else
			stop_cluster
		fi
	;;
	start)
		if [ -z ${envv} ];then
			start_usage
		else
			start_cluster
		fi
	;;
	ls)
		ls_cluster
	;;
	show)
		if [ -z ${envv} ];then
			show_usage
		else
			show_cluster
		fi
	;;
	"install --yes --yes")
		init_cluster
		init_network
	;;
	backup)
		if [ -z ${envv} ];then
			backup_usage
		else
			bak_cluster
		fi
	;;
	--help)
		create_usage
		delete_usage
		restart_usage
		stop_usage
		start_usage
		backup_usage
		show_usage
		ls_usage
		install_usage
	;;
	*)
		$0 --help
	;;
esac