Python自動化部署
- 2020 年 1 月 7 日
- 筆記
# -*- coding: utf-8 -*- #!/bin/env python ''' #Auth: karl #Function: released version #Date:2017/6/27 #Version:V1.0 ''' import sys,re,os,time,datetime import paramiko import logging import socket import ConfigParser import traceback from progressbar import * import Auto_Mysql_release import platform import smtplib import email.mime.multipart import email.mime.text import json import os import struct import requests receivers = "[email protected]" receiver = "XXX@com" #reg 為0時正常執行命令,為1時開始檢查服務啟動是否正常,為3時不用再備份原文件 def ssh2(host, port, username, password, cmds,reg=0): # 鏈接遠程伺服器並執行命令p try: paramiko.util.log_to_file('./../log/exec_cmd_' + time_str + '.log') ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(host, int(port), username, password, timeout=5) # 執行命令文件中的命令 if reg == 3: stdin, stdout, stderr = ssh.exec_command('ls /tmp/backup/old/ADMIN/integration.properties') if stdout.readline() != '': pass else: for cmd in cmds: logging.info('running: ' + cmd) print 'running: ' + cmd time.sleep(2) stdin, stdout, stderr = ssh.exec_command(cmd) for out_msg in stdout.readlines(): print out_msg else: for cmd in cmds: print "=======>>",cmd logging.info('running: ' + cmd) print 'running: ' + cmd time.sleep(5) stdin, stdout, stderr = ssh.exec_command(cmd) if reg == 1: out_msg=str("".join(stdout.readlines())) print "000000------>>",out_msg if "[main] INFO org.eclipse.jetty.server.Server.doStart(Server.java:379)" in out_msg and "error" not in out_msg and "Failed startup" not in out_msg: print "THIS IS ok..." print "------->> %s"%(host) else: #回滾到當前版本 # repair_cmd_file = conf.get(section, 'repair_cmd_file') # exec_file_cmd(conf, section, repair_cmd_file) copy_cmd_file(conf, section, host,1) for ip_f in host_ip[1:]: num_f = host_ip.index(ip_f) num_f += 1 process_cmd_file(conf, section,ip_f, num_f) for ip_s in host_ip: num_s = host_ip.index(ip_s) num_s += 1 print "+++++++ %s" % (ip_s) start_server(conf, section, ip_s, num_s) exit(1) else: print "++++++++++++++",reg for out_msg in stdout.readlines(): print "----", out_msg for err_msg in stderr.readlines(): print err_msg exit(1) ssh.close() print 'command execute successful!' logging.info('command execute successful!') except Exception, e: print '%stErrorn' % (host) print traceback.format_exc() __err_exit_show_msg(str(e)) def upload(conf, section): ''' 上傳文件到遠程伺服器 ''' host = conf.get(section, 'host') port = conf.get(section, 'port') username = conf.get(section, 'username') password = conf.get(section, 'password') local_file = conf.get(section, 'local_file') remote_file = conf.get(section, 'remote_file') try: paramiko.util.log_to_file('../log/upload_' + time_str + '.log') logging.info('paramiko log created') t = paramiko.Transport((host, int(port))) t.connect(username=username, password=password) logging.info('connected host <' + host + '> successful') logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now()) sftp = paramiko.SFTPClient.from_transport(t) logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now()) print 'Beginning to upload file to %s %s ' % (host, datetime.datetime.now()) # 定義上傳進度條樣式 widgets = ['File: ', Percentage(), ' ', Bar(marker='#', left='[', right=']'), ' ', ETA(), ' ', FileTransferSpeed()] file_size = os.path.getsize(local_file) pbar = ProgressBar(widgets=widgets, maxval=file_size) # 開始進度條 pbar.start() # 使用匿名方法接收上傳返回值，並且顯示進度條 progress_bar = lambda transferred, toBeTransferred: pbar.update(transferred) sftp.put(local_file,remote_file, callback=progress_bar) pbar.finish() logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now()) print 'Upload file SUCCESSFUL %s ' % datetime.datetime.now() t.close() logging.info('sftp closed!') cmd="tar -xvf %s -C /home/appdeploy/version/ >/dev/null " % local_file if "Linux" == platform.system(): os.system(cmd) report_cmd_file = __checke_conf_key_value_empty(conf, section, 'test_report') except Exception, e: logging.error('host: <' + host + '> connect error!') print host, 'connect error!' print traceback.format_exc() __err_exit_show_msg(str(e)) def email_send(section,version): evn=section V=version msg = email.mime.multipart.MIMEMultipart() msg['Subject'] = '版本發布通知郵件' msg['From'] = '[email protected]' msg['To'] = ','.join(receivers) content = ''' 你好，各位同事： 本次%s版本(%s)發布: 發布成功,祝賀！！！ '''%(evn,V) txt = email.mime.text.MIMEText(content) msg.attach(txt) smtp = smtplib.SMTP() smtp.connect('smtp.163.com', '25') smtp.login('[email protected]', 'passwd') smtp.sendmail(msg['From'], receivers, msg.as_string()) smtp.quit() print('郵件發送成功email has send out !') def email_linux(receivers, subject=None, bodyhtml=None,attachments=None): ''' 對接統一通知平台，發郵件樣例 receivers 收件人郵箱 subject 主題 bodyhtml 郵件內容 attachment即附件路徑默認為空，如有附件傳入文件路徑''' file_name = attachments.split("/")[-1] lis = '' time_str =str(time.strftime('%Y-%m-%d',time.localtime(time.time()))) # 把附件內容轉換為字元列表 if attachments != None: file_name = os.path.basename(attachments) file = open(attachments, 'rb') _content = file.read() lis = struct.unpack('%db' % len(_content),_content) # 對應渠道模板中`message`中參數 templateJson = { 'version' : bodyhtml, 'time' :time_str, 'question' :"詳情請審閱附件，謝謝！" } data = { 'userId': receivers, #收件人郵箱,支援多收件人,分號隔離;如下面的抄送人格式 'ccId': receiver, #抄送人,如果沒有可以屏蔽該語句 'templateCode': 'version_release_code', #業務模板code 'templateParam': templateJson, #如果對應渠道模板中沒有類{{}}格式的參數，可以屏蔽該語句 'subject': subject, #郵件主題，默認是渠道模板名稱 'attachmentName' : file_name, #郵件附件名稱， 如果沒有附件可以屏蔽該語句 'attachmentArray': lis, #郵件附件內容,如果沒有附件可以屏蔽該語句 'msgType': 'txt', #消息類型，目前只支援txt 'accessId': '52HX1CYE', #通知平台接入Id 'accessToken': 'ebf1dd3140cf4f0abd79872d7d237c3d' #通知平台接入Token } json_str = json.dumps(data) url = "http://public-int-gw.int.sfdc.com.cn:1080/unp/notice/single" headers = {'content-type': 'application/json; charset=UTF-8'} try: response = requests.post(url, data=json_str, headers=headers) print response.text print "-------------" result_json = json.loads(response.text) print(result_json) #列印返回內容 except Exception as e: print('調用統一通知平台介面失敗:',str(e)) def copy_cmd_file(conf, section, ip,reg): if reg == 0: filep="/home/appdeploy/version/Version_3.0" else: filep="/tmp/backup/old" host=ip port = conf.get(section, 'port') username = conf.get(section, 'username') password = conf.get(section, 'password') remote_file = conf.get(section, 'remote_file') print "----------------------------copy files--- %s--------------------" % (host) cmd_c = [ 'cp -vr {files}/ADMIN/*.war {path}_ADMIN_01/deploy/webapps/'.format(files=filep, path=Filepath), 'cp -vr {files}/ADMIN/integration.properties {path}_ADMIN_01/deploy/resources/'.format(files=filep,path=Filepath), 'cp -vr {files}/TRAPP/*.war {path}_TRAPP_01/deploy/webapps/'.format(files=filep, path=Filepath), 'cp -vr {files}/TRAPP/integration.properties {path}_TRAPP_01/deploy/resources/'.format(files=filep,path=Filepath), 'cp -vr {files}/TRTS/*.war {path}_TRTS_01/deploy/webapps/'.format(files=filep, path=Filepath), 'cp -vr {files}/TRTS/integration.properties {path}_TRTS_01/deploy/resources/'.format(files=filep, path=Filepath), ] ssh2(host, port, username, password, cmd_c) #針對腳本進行參數化設置 def process_cmd_file(conf, section,ip_f, num_f): host = ip_f port = conf.get(section, 'port') username = conf.get(section, 'username') password = conf.get(section, 'password') remote_file = conf.get(section, 'remote_file') print "----------------------------copy remote files--- %s--------------------" % (host) print "-------->>>>>",host_ip[0],port,username,password,remote_file cmd=[ 'scp {path}_ADMIN_01/deploy/webapps/*.war {user}@{ip}:{path}_ADMIN_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f), # 'scp {path}_ADMIN_01/deploy/resources/integration.properties {user}@{ip}:{path}_ADMIN_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f), 'scp {path}_TRTS_01/deploy/webapps/*.war {user}@{ip}:{path}_TRTS_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f), # 'scp {path}_TRTS_01/deploy/resources/integration.properties {user}@{ip}:{path}_TRTS_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f), 'scp {path}_TRAPP_01/deploy/webapps/*.war {user}@{ip}:{path}_TRAPP_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f), # 'scp {path}_TRAPP_01/deploy/resources/integration.properties {user}@{ip}:{path}_TRAPP_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f) ] ssh2(host_ip[0], port, username, password, cmd) def exec_file_cmd(conf, section, cmd_file): ''' 執行文件中的命令 ''' host = conf.get(section, 'host') port = conf.get(section, 'port') username = conf.get(section, 'username') password = conf.get(section, 'password') cmds = __get_cmds(cmd_file) ssh2(host, port, username, password, cmds) def backup_ori(conf, section): ''' 備份遠程原文件 ''' host = conf.get(section, 'host') port = conf.get(section, 'port') username = conf.get(section, 'username') password = conf.get(section, 'password') remote_file = conf.get(section, 'remote_file') remote_ori_backup_dir = conf.get(section, 'remote_ori_backup_dir') # 獲得備份後綴 suffix_time = time.strftime('%Y-%m-%d',time.localtime(time.time())) backup_ori_cmd = [ 'mkdir -p {dir}/ADMIN {dir}/TRTS {dir}/TRAPP'.format(dir=remote_ori_backup_dir), 'cp -vr {path}_TRAPP_01/deploy/webapps/*.war {dir}/TRAPP'.format(path=Filepath,dir=remote_ori_backup_dir), 'cp -vr {path}_ADMIN_01/deploy/webapps/*.war {dir}/ADMIN'.format(path=Filepath,dir=remote_ori_backup_dir), 'cp -vr {path}_TRTS_01/deploy/webapps/*.war {dir}/TRTS'.format(path=Filepath,dir=remote_ori_backup_dir), 'cp -vr {path}_TRAPP_01/deploy/resources/integration.properties {dir}/TRAPP'.format(path=Filepath,dir=remote_ori_backup_dir), 'cp -vr {path}_ADMIN_01/deploy/resources/integration.properties {dir}/ADMIN'.format(path=Filepath,dir=remote_ori_backup_dir), 'cp -vr {path}_TRTS_01/deploy/resources/integration.properties {dir}/TRTS'.format(path=Filepath,dir=remote_ori_backup_dir) ] ssh2(host, port, username, password, backup_ori_cmd,3) def backup_new(conf, section): ''' 備份遠程新上傳的文件 ''' host = conf.get(section, 'host') port = conf.get(section, 'port') username = conf.get(section, 'username') password = conf.get(section, 'password') remote_file = conf.get(section, 'remote_file') remote_backup_dir = conf.get(section, 'remote_backup_dir') # 獲得備份後綴 suffix_time = time.strftime('%Y-%m-%d_%H-%M-%S',time.localtime(time.time())) backup_new_cmd = [ 'mkdir -p {dir}'.format(dir=remote_backup_dir), 'cp -vr {new_file} {dir}/{new_bak_file}_{time}'.format(new_file=remote_file, dir=remote_backup_dir, new_bak_file=os.path.basename(remote_file), time=str(suffix_time)) ] ssh2(host, port, username, password, backup_new_cmd) def select_section(conf_file_name): ''' 選擇指定讀取的配置文件項 例如：*.conf配置文件中有多個配置項 a 和 b: [a] xxxxx [b] yyyyy ''' # 檢測指定的配置文件是否存在 __check_file_exists(conf_file_name) # 讀取配置文件 conf = ConfigParser.ConfigParser() conf.read(conf_file_name) sections = conf.sections() # 選擇配置文件選項介面 print 'please choose confit item:' for index, value in enumerate(sections): print ' ', index, ':', value while True: sec_index = raw_input('please choose one item default [0]:') if not sec_index.isdigit() or int(sec_index) >= len(sections): print 'choose invalid!' continue return conf, sections[int(sec_index)] return conf, sections[0] def check_config(conf, section): ''' 檢測配置文件的正確性 ''' logging.info('check config starting...') print 'check config starting...' # 檢測配置文件中值是否都填寫 host = __checke_conf_key_value_empty(conf, section, 'host') # 檢測配置文件中主機名 port = __checke_conf_key_value_empty(conf, section, 'port') # 檢測配置文件中埠 username = __checke_conf_key_value_empty(conf, section, 'username') # 檢測配置文件用戶名 password = __checke_conf_key_value_empty(conf, section, 'password') # 檢測配置文件密碼 local_file = __checke_conf_key_value_empty(conf, section, 'local_file') # 檢測配置文件本地需要上傳文件 remote_file = __checke_conf_key_value_empty(conf, section, 'remote_file') # 檢測配置文件上傳到遠程的文件 remote_backup_dir = __checke_conf_key_value_empty(conf, section, 'remote_backup_dir') # 檢測配置文件遠程備份目錄 remote_ori_backup_dir = __checke_conf_key_value_empty(conf, section, 'remote_ori_backup_dir') # 檢測配置文件遠程臨時備份目錄 start_cmd_file = __checke_conf_key_value_empty(conf, section, 'start_cmd_file') # 檢測配置文件啟動服務文件 report_cmd_file = __checke_conf_key_value_empty(conf, section, 'test_report') # 檢測配置文件停止服務文件 # 檢測配置文件中的網路是否可用 __check_network_ping(host) # 檢測ssh鏈接是否成功 __check_ssh(host, int(port), username, password) # 檢測本地需要上傳的文件是否存在 __check_file_exists(local_file) # 檢測命令文件是否存在 __check_file_exists(start_cmd_file) print 'check config successful!!' logging.info('check config successful!!') def __valid_ip(address): ''' 檢測IP是否合法IP ''' try: socket.inet_aton(address) return True except: print traceback.format_exc() return False def __check_file_exists(conf_file_name): ''' 檢測指定的配置文件是否存在 ''' if not os.path.exists(conf_file_name): logging.error('can not find config file: ' + conf_file_name) __err_exit_show_msg('can not find config file: ' + conf_file_name) return conf_file_name def __checke_conf_key_value_empty(conf, section, key): ''' 檢測配置文件的key是否存在 ''' try: value = conf.get(section, key) # 檢測配置文件中的值是否為空 if value: return value else: msg = ''' ERROR The key:{key} value is empty in conf file '''.format(key=key) __err_exit_show_msg(msg) except ConfigParser.NoOptionError: print traceback.format_exc() msg = ''' ERROR cannot find key:{key} in conf file '''.format(key=key) __err_exit_show_msg(msg) def __check_network_ping(host): if not __valid_ip(host): __err_exit_show_msg('host: ' + host + ' invalid') if "Linux" == platform.system(): if 0 <> os.system('ping -c 3 ' + host): __err_exit_show_msg('host: ' + host + ' cannot ping...') else: if 0 <> os.system('ping -n 1 -w 5 ' + host): __err_exit_show_msg('host: ' + host + ' cannot ping...') def __check_ssh(host, port, username, password): try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(host, port, username, password, timeout=5) ssh.close() except Exception as e: print traceback.format_exc() msg = ''' SSH connect failure. please check your host/port/username/password host : {host} port : {port} username: {username} password: {password} '''.format(host=host, port=port, username=username, password=password) __err_exit_show_msg(msg) def __get_cmds(cmd_file): ''' 文件中獲取執行命令 ''' with open(cmd_file, 'r') as cmd_f: pattern = re.compile('(^s*#|^s*$)') func = lambda x: x if not re.match(pattern, x) else None cmds = [cmd for cmd in cmd_f] return filter(func, cmds) def check_server(conf, section,ip,num): host = ip port = conf.get(section, 'port') username = conf.get(section, 'username') password = conf.get(section, 'password') suffix_time = time.strftime('%Y%m%d', time.localtime(time.time())) print "----------------------------check result--- %s--------------------"%(ip) checkserver_cmd=[ 'cat {path}_ADMIN_0{num}/logs/novatar_{time}.0.log |grep "error|org.eclipse.jetty.server.Server.doStart(Server.java:379)|Failed startup"'.format(path=Filepath,time=suffix_time,num=num), 'cat {path}_TRAPP_0{num}/logs/novatar_{time}.0.log |grep "error|org.eclipse.jetty.server.Server.doStart(Server.java:379)|Failed startup"'.format(path=Filepath,time=suffix_time,num=num), 'cat {path}_TRTS_0{num}/logs/novatar_{time}.0.log |grep "error|org.eclipse.jetty.server.Server.doStart(Server.java:379)|Failed startup"'.format(path=Filepath,time=suffix_time,num=num) ] ssh2(host, port, username, password, checkserver_cmd,1) def start_server(conf, section, ip, num): host = ip port = conf.get(section, 'port') username = conf.get(section, 'username') password = conf.get(section, 'password') print "----------------------------restart server --- %s--------------------" % (ip) checkserver_cmd = [ 'sh {path}_ADMIN_0{num}_run.sh stop; sh {path}_ADMIN_0{num}_run.sh start'.format(path=Restartfile,num=num), 'sh {path}_TRAPP_0{num}_run.sh stop; sh {path}_TRAPP_0{num}_run.sh start'.format(path=Restartfile,num=num), 'sh {path}_TRTS_0{num}_run.sh stop; sh {path}_TRTS_0{num}_run.sh start'.format(path=Restartfile,num=num) ] ssh2(host, port, username, password, checkserver_cmd) def __err_exit_show_msg(msg): ''' 發生錯誤的時候顯示相關錯誤資訊並且退出程式 ''' print 'ERROR: ' + msg logging.error('ERROR: ' + msg) os.system('pause') sys.exit() if __name__ == '__main__': try: start = time.clock() # 設置日誌文件 time_str = time.strftime('%Y-%m-%d',time.localtime(time.time())) log_file = '../log/upload_distribution_' + str(time_str) + '.log' logging.basicConfig(level=logging.INFO, format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', filename=log_file, filemode='w', datefmt='%Y-%m-%d %X') # 定義配置文件路徑 conf_file_name = '../conf/release.conf' # 選擇配置文件section conf, section = select_section(conf_file_name) if section == "STG": Filepath="/app/jetty/server/SCS_ATP_CNSZ99_JETTY_APP" Restartfile="/app/jetty/logs/SCS_ATP_CNSZ99_JETTY_APP" host_ip=[] elif section == "DEVTEST": Filepath="/app/jetty/server/SCS_ATP_CORE_CNSZ22_JETTY_APP" Restartfile="/app/jetty/logs/SCS_ATP_CORE_CNSZ22_JETTY_APP" host_ip=[] else : Filepath="/app/jetty/server/SCS_ATP_CORE_CNSZ17_JETTY_APP" Restartfile="/app/jetty/logs/SCS_ATP_CNSZ17_JETTY_APP" host_ip=[] # 檢測配置文件正確性 check_config(conf, section) print(' 33[1;35m You can view the configuration file in the version number:"/home/appdeploy/version/Version_ 33[1;32m3.0 33[1;35m.tar" 33[0m!') VERSION=raw_input("Please Enter Version: eg 3.0 n") # 備份原文件 backup_ori(conf, section) # 上傳文件 upload(conf, section) # 備份新上傳的文件 backup_new(conf, section) # 解壓文件 start_cmd_file = conf.get(section, 'start_cmd_file') exec_file_cmd(conf, section, start_cmd_file) # 上傳新文件到應用目錄 copy_cmd_file(conf, section, host_ip[0],0) # 拷貝文件到遠程伺服器 if len(host_ip) > 1: for ip_f in host_ip[1:]: num_f = host_ip.index(ip_f) num_f += 1 process_cmd_file(conf, section,ip_f, num_f) # 執行拷貝excel 等文件,記錄到資料庫中 agre=["3306", "XXX", "root",host_ip[0],VERSION] print "agre :",agre stg=Auto_Mysql_release.Mysql_connect(*agre) stg.File_get() stg.File_deal() #啟動所有伺服器的服務 for ip_s in host_ip: num_s = host_ip.index(ip_s) num_s += 1 start_server(conf, section,ip_s,num_s) # 監聽服務是否啟動成功 time.sleep(90) for ip in host_ip: num=host_ip.index(ip) num+=1 check_server(conf, section,ip,num) # 實行完畢退出 cmd_f = ["rm -fr /tmp/backup/old/*"] ssh2(host_ip[0],"22","mwopr","XXX",cmd_f) os.system('exit') end = time.clock() print "version release SUCCESSFULL.......共耗時%f s" %(end - start) # 郵件群發到相關責任人 if section == "ONLINE": testReport = conf.get(section, 'test_report') text="%s版本發布通知郵件"% (VERSION) email_linux(receivers,"版本發布通知郵件",VERSION,testReport) elif section == "DEVTEST": email_send(section,VERSION) else: pass exit(0) except Exception as e: print traceback.format_exc() __err_exit_show_msg(str(e))
具體流程如下：
