wangjin0928
/
kafka_yolov5
Obserwuj 1
Polub 0
Forkuj 0
Kod Zgłoszenia 0 Oczekujące zmiany 0 Wydania 0 Wiki Aktywność
用kafka接收消息
Nie możesz wybrać więcej, niż 25 tematów Tematy muszą się zaczynać od litery lub cyfry, mogą zawierać myślniki ('-') i mogą mieć do 35 znaków.
2 Commity
2 Gałęzie
Drzewo: ccbb4ef24d
Gałęzie Tagi
${ item.name }
Utwórz gałąź ${ searchTerm }
z 'ccbb4ef24d'
${ noResults }
kafka_yolov5/master.py

579 lines
29KB
Czysty Wina Historia 

  1. import numpy as np

  2. import time,ast,copy

  3. from flask import request, Flask,jsonify

  4. import base64,cv2,os,sys,json

  5. #sys.path.extend(['../yolov5'])  

  6. #from Send_tranfer import b64encode_function,JsonSend,name_dic,nameID_dic,getLogFileFp 

  7. from segutils.segmodel import SegModel,get_largest_contours

  8. from models.experimental import attempt_load

  9. from utils.datasets import LoadStreams, LoadImages

  10. from utils.torch_utils import select_device, load_classifier, time_synchronized

  11. from queRiver import get_labelnames,get_label_arrays,post_process_,save_problem_images,time_str

  12. import subprocess as sp

  13. import matplotlib.pyplot as plt

  14. import torch,random,string

  15. import multiprocessing

  16. from multiprocessing import Process,Queue

  17. import traceback

  18. from kafka import KafkaProducer, KafkaConsumer,TopicPartition

  19. from kafka.errors import kafka_errors

  20. 

  21. #torch.multiprocessing.set_start_method('spawn')

  22. import utilsK

  23. from utilsK.GPUtils import *

  24. from utilsK.masterUtils import *

  25. from utilsK.sendUtils import create_status_msg,update_json

  26. 

  27. #from utilsK.modelEval import onlineModelProcsss

  28. import random,string

  29. from Send_tranfer_oss import msg_dict_on,msg_dict_off

  30. 

  31. process_id=0

  32. 

  33. def onlineModelProcess(parIn  ):

  34.     DEBUG=False

  35.     streamName = parIn['streamName']

  36.     childCallback=parIn['callback']

  37.     outStrList={}

  38.     channelIndex=parIn['channelIndex']

  39.     #try:

  40.     for wan in ['test']:

  41.         jsonfile=parIn['modelJson']

  42.         with open(jsonfile,'r') as fp:

  43.             parAll = json.load(fp)

  44.             

  45.         Detweights=parAll['gpu_process']['det_weights']

  46.         seg_nclass = parAll['gpu_process']['seg_nclass']

  47.         Segweights = parAll['gpu_process']['seg_weights']

  48.         StreamRecoveringTime=int(parAll['StreamRecoveringTime'])

  49.         videoSave =  parAll['AI_video_save']

  50.         imageTxtFile =  parAll['imageTxtFile']

  51.         taskId,msgId = streamName.split('-')[1:3]

  52.   

  53.         inSource,outSource=parIn['inSource'],parIn['outSource']

  54.         

  55.         ##构建日志文件

  56.         if outSource != 'NO':

  57.             logdir = parAll['logChildProcessOnline'] 

  58.             waitingTime=parAll['StreamWaitingTime']   

  59.         else:        

  60.             logdir = parAll['logChildProcessOffline']    

  61.             waitingTime=5  

  62.         logname='gpuprocess.log'  

  63.         fp_log=create_logFile(logdir=logdir,name=logname) 

  64.         logger=logdir.replace('/','.')+'.'+logname

  65.         kafka_par=parIn['kafka_par']

  66.         producer = KafkaProducer(bootstrap_servers=kafka_par['server'],value_serializer=lambda v: v.encode('utf-8'),metadata_max_age_ms=120000)

  67. 

  68.         ####要先检查视频的有效性

  69.         ###开始的时候,如果在线任务没有流，要发送的心跳消息，msg_h,

  70.         msg_h=   copy.deepcopy(msg_dict_off);

  71.         msg_h['status']='waiting';msg_h['msg_id']=msgId

  72.        

  73.         

  74.         thread='master:gpuprocess-%s'%(msgId) 

  75.         if outSource == 'NO': 

  76.             msg_h['type']=1

  77.             Stream_ok,_= get_fps_rtmp(inSource,video=True) 

  78.         else:

  79.             msg_h['type']=2         

  80.             msg_h_d = json.dumps(msg_h, ensure_ascii=False) 

  81.             outStrList=get_infos(taskId, msgId,msg_h_d,key_str='waiting stream or video, send heartbeat')          

  82.             Stream_ok=check_stream(inSource,producer,kafka_par,msg_h_d,outStrList,fp_log,logger,line=sys._getframe().f_lineno,thread=thread ,timeMs=waitingTime)

  83.         

  84.         if  Stream_ok:###发送开始信号

  85.             msg_h['status']='running'

  86.             msg_h_d = json.dumps(msg_h, ensure_ascii=False) 

  87.             outStrList=  get_infos(taskId, msgId,msg_h_d,key_str='informing stream/video is ok')          

  88.             send_kafka(producer,kafka_par,msg_h_d,outStrList,fp_log,line=sys._getframe().f_lineno,logger=logger,thread=thread      );

  89.             

  90.         else:

  91.             ####检测离线视频是否有效,无效要报错

  92.             outstr='offline vedio or live stream  Error:%s '%(inSource)

  93.             #outstr=wrtiteLog(fp_log,outstr);print( outstr);

  94.             writeELK_log(msg=outstr,fp=fp_log,level='ERROR',line=sys._getframe().f_lineno,logger=logger)   

  95.             

  96.             msg_h['error']='Stream or video ERROR';msg_h['status']='failed';

  97.             msg_h_d = json.dumps(msg_h, ensure_ascii=False);

  98.             outStrList=  get_infos(taskId, msgId,msg_h_d,key_str='informing invaid video or stream success')       

  99.             send_kafka(producer,kafka_par,msg_h_d,outStrList,fp_log ,line=sys._getframe().f_lineno,logger=logger,thread=thread     ); 

  100.             childCallback.send(' offline vedio or live stream  Error')  

  101.             continue

  102.             

  103.         

  104.        

  105.         if (inSource.endswith('.MP4')) or (inSource.endswith('.mp4')):

  106.             fps,outW,outH,totalcnt=get_fps_rtmp(inSource,video=True)[1][0:4];

  107.         else:

  108.             fps,outW,outH,totalcnt=get_fps_rtmp(inSource,video=False)[1][0:4]

  109.         fps = int(fps+0.5)       

  110.         if fps>30: fps=25  ###线下测试时候，有时候读帧率是9000，明显不符合实际，所以加这个判断。

  111.         if outSource != 'NO':

  112.             command=['/usr/bin/ffmpeg','-y','-f', 'rawvideo','-vcodec','rawvideo','-pix_fmt', 'bgr24',

  113.                 '-s', "{}x{}".format(outW,outH),# 图片分辨率           

  114.                 '-r', str(fps),# 视频帧率

  115.                 '-i', '-','-c:v', 

  116.                 'libx264',

  117.                 '-pix_fmt', 'yuv420p',

  118.                 '-f', 'flv',outSource

  119.                 ]

  120. 

  121. 

  122.             video_flag = videoSave['onLine']      

  123.             logdir = parAll['logChildProcessOnline'] 

  124.             waitingTime=parAll['StreamWaitingTime']   

  125.         else:        

  126.             video_flag = videoSave['offLine'] ;logdir = parAll['logChildProcessOffline']    

  127.             waitingTime=5  

  128. 

  129.         device = select_device(parIn['device'])

  130.         half = device.type != 'cpu'  # half precision only supported on CUDA

  131.         model = attempt_load(Detweights, map_location=device)  # load FP32 model

  132.         if half: model.half()

  133. 

  134.         segmodel = SegModel(nclass=seg_nclass,weights=Segweights,device=device)

  135. 

  136.        

  137.            

  138.     

  139.         ##后处理参数

  140.         par=parAll['post_process']

  141.         conf_thres,iou_thres,classes=par['conf_thres'],par['iou_thres'],par['classes']

  142.         outImaDir = par['outImaDir']

  143.         outVideoDir = par['outVideoDir']

  144.         labelnames=par['labelnames']

  145.         rainbows=par['rainbows']

  146.         fpsample = par['fpsample']

  147.         names=get_labelnames(labelnames)

  148.         label_arraylist = get_label_arrays(names,rainbows,outfontsize=40)

  149.         

  150.         #dataset = LoadStreams(inSource, img_size=640, stride=32)

  151.         

  152.     

  153.         childCallback.send('####model load success####')    

  154.         print('#####line153:',outVideoDir,video_flag)

  155.         if (outVideoDir!='NO')  :  ####2022.06.27新增在线任务也要传AI视频和原始视频

  156.             if video_flag:

  157.                 msg_id = streamName.split('-')[2]

  158.                 save_path = os.path.join(outVideoDir,msg_id+'.MP4')

  159.                 vid_writer = cv2.VideoWriter(save_path, cv2.VideoWriter_fourcc(*'mp4v'), fps, (outW,outH))

  160.                 if vid_writer.isOpened(): outstr='touch video success:%s'%(save_path);level='INFO'

  161.                 else:outstr='touch video failed:%s'%(save_path);level='ERROR'

  162.                 writeELK_log(msg=outstr,fp=fp_log,level=level,line=sys._getframe().f_lineno,logger=logger)   

  163.             else:

  164.                 msg_id = streamName.split('-')[2]

  165.                 save_path_OR = os.path.join(outVideoDir,msg_id+'_OR.MP4')

  166.                 vid_writer_OR = cv2.VideoWriter(save_path_OR, cv2.VideoWriter_fourcc(*'mp4v'), fps, (outW,outH))

  167.                 save_path_AI = os.path.join(outVideoDir,msg_id+'_AI.MP4')

  168.                 vid_writer_AI = cv2.VideoWriter(save_path_AI, cv2.VideoWriter_fourcc(*'mp4v'), fps, (outW,outH))

  169.                 if vid_writer_AI.isOpened() and vid_writer_OR.isOpened() :outstr='touch video success:%s,%s'%(save_path_OR,save_path_AI);level='INFO'

  170.                 else:outstr='touch video failed:%s,%s, fps:%d ,%d , %d'%(save_path_OR,save_path_AI,fps,outW,outH);level='ERROR'

  171.                 writeELK_log(msg=outstr,fp=fp_log,level=level,line=sys._getframe().f_lineno,logger=logger)  

  172.                 

  173.         iframe = 0;post_results=[];time_beg=time.time()

  174.         

  175.         t00=time.time()

  176.         time_kafka0=time.time()

  177.         

  178.         Pushed_Flag=False

  179.         while True:

  180.             try:   

  181.                 dataset = LoadStreams(inSource, img_size=640, stride=32)

  182.                  # 管道配置，其中用到管道

  183.                 if outSource !='NO'  and (not Pushed_Flag):

  184.                     ppipe = sp.Popen(command, stdin=sp.PIPE);Pushed_Flag = True

  185.                 for path, img, im0s, vid_cap in dataset:  

  186.                     t0= time_synchronized()

  187.                     if outSource == 'NO':###如果不推流，则显示进度条。离线不推流

  188.                         view_bar(iframe,totalcnt,time_beg ,parIn['process_uid'] )

  189.                     streamCheckCnt=0 

  190.                     ###直播和离线都是1分钟发一次消息

  191.                     time_kafka1 = time.time()

  192.                     if time_kafka1 - time_kafka0 >60:

  193.                         time_kafka0 = time_kafka1

  194.                         ###发送状态信息waiting

  195.                         msg = copy.deepcopy(msg_dict_off);

  196.                         msg['msg_id']= msgId; 

  197.                         if outSource == 'NO':

  198.                             msg['progressbar']= '%.4f'%(iframe*1.0/totalcnt)

  199.                             msg['type']=1

  200.                         else:

  201.                             msg['progressbarOn']= str(iframe)

  202.                             msg['type']=2

  203.                         

  204.                         msg = json.dumps(msg, ensure_ascii=False)    

  205.                         outStrList=  get_infos(taskId, msgId,msg,key_str='processing send progressbar or online heartbeat') 

  206.                         send_kafka(producer,kafka_par,msg,outStrList,fp_log,line=sys._getframe().f_lineno,logger=logger,thread=thread      ); 

  207.                     

  208.                     time0=time.time()

  209.                     iframe +=1

  210.                     time1=time.time()

  211.                     img = torch.from_numpy(img).to(device)

  212.                     img = img.half() if half else img.float()  # uint8 to fp16/32

  213.           

  214.                     img /= 255.0  # 0 - 255 to 0.0 - 1.0

  215.                     

  216.                     timeseg0 = time.time()

  217.                     seg_pred,segstr = segmodel.eval(im0s[0] )

  218.                     timeseg1 = time.time()

  219.                     

  220.                     t1= time_synchronized()

  221. 

  222.                     pred = model(img,augment=False)[0]

  223. 

  224.                     time4 = time.time()

  225.                     datas = [path, img, im0s, vid_cap,pred,seg_pred,iframe]

  226.                     

  227.                     p_result,timeOut = post_process_(datas,conf_thres, iou_thres,names,label_arraylist,rainbows,iframe)

  228.                     t2= time_synchronized()

  229.                     

  230.                     #print('###line138:',timeOut,outSource,outVideoDir)

  231.                     ##每隔 fpsample帧处理一次，如果有问题就保存图片       

  232.                     if (iframe % fpsample == 0) and (len(post_results)>0) :

  233.                         parImage=save_problem_images(post_results,iframe,names,streamName=streamName,outImaDir='problems/images_tmp',imageTxtFile=imageTxtFile)

  234.                         post_results=[]   

  235. 

  236.                     if len(p_result[2] )>0:  ##                

  237.                         post_results.append(p_result)

  238.                     t3= time_synchronized()

  239.                     image_array = p_result[1]

  240.                     if outSource!='NO':

  241.                         ppipe.stdin.write(image_array.tobytes())  

  242.           

  243.                     if (outVideoDir!='NO'):  

  244.                         if video_flag:    ret = vid_writer.write(image_array)  

  245.                         else:

  246.                             time_w0=time.time()

  247.                             ret = vid_writer_AI.write(image_array)    

  248.                             ret = vid_writer_OR.write(im0s[0]) 

  249.                             time_w1=time.time()   

  250.                             #if not ret:

  251.                             #    print('\n  write  two videos time:%f ms'%(time_w1-time_w0)*1000,ret)                    

  252.                     t4= time_synchronized()

  253.                     

  254.                     timestr2 = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime()) 

  255.                     if iframe%100==0:

  256.                         outstr='%s,,read:%.1f ms,copy:%.1f, infer:%.1f ms, detinfer:%.1f ms,draw:%.1f ms,  save:%.1f ms   total:%.1f ms \n'%(timestr2,(t0 - t00)*1000,(timeseg0-t0)*1000, (t1 - timeseg0)*1000,(t2-t1)*1000, (t3 - t2)*1000,(t4-t3)*1000, (t4-t00)*1000)

  257.                         #wrtiteLog(fp_log,outstr);

  258.                         writeELK_log(msg=outstr,fp=fp_log,line=sys._getframe().f_lineno,logger=logger,printFlag=False)   

  259.                     #print(outstr)

  260.                     t00 = t4;  

  261.                     

  262.             except Exception as e:

  263.                 #if outSource:###推流才有如下

  264.                 streamCheckCnt+=1;taskEnd=False

  265.                 if streamCheckCnt==1:timeBreak0=time.time();time_kafka0 = time.time()

  266.                 timeBreak1=time.time();

  267.                 if  timeBreak1-timeBreak0 >5 and  Pushed_Flag:###流断开5秒后，要关闭推流

  268.                     ppipe.kill();Pushed_Flag=False

  269.                     writeELK_log(msg='stream pip is killed ',fp=fp_log,line=sys._getframe().f_lineno,logger=logger) 

  270.                     

  271.                 ###读接口，看看任务有没有结束

  272.                 ChanellInfos,taskEnd=query_channel_status(channelIndex)

  273.                 ####taskEnd######################DEBUG

  274.                 #taskEnd=False

  275.                 

  276.                 if timeBreak1-timeBreak0 >StreamRecoveringTime : ##默认30分钟内，流没有恢复的话，就断开。

  277.                     taskEnd=True

  278.                 

  279.                 outstr_channel='%s ,taskEnd:%s'%(ChanellInfos,taskEnd)

  280.                 writeELK_log(msg=outstr_channel,fp=fp_log,line=sys._getframe().f_lineno,logger=logger)    

  281.                 if outSource == 'NO':#离线没有推流

  282.                     taskEnd=True            

  283.                 if  taskEnd:   

  284.                     if timeBreak1-timeBreak0 > 60:###超时结束

  285.                         writeTxtEndFlag(outImaDir,streamName,imageTxtFile,endFlag='超时结束')

  286.                     else:

  287.                         writeTxtEndFlag(outImaDir,streamName,imageTxtFile,endFlag='结束')

  288.                     if (outVideoDir!='NO'):  

  289.                         if video_flag:vid_writer.release()

  290.                         else:

  291.                             vid_writer_OR.release();

  292.                             vid_writer_AI.release();

  293.                     outstr='Task ends:%.1f , msgid:%s,taskID:%s '%(timeBreak1-timeBreak0,taskId,msgId)

  294.                     writeELK_log(msg=outstr,fp=fp_log,line=sys._getframe().f_lineno,logger=logger)         

  295.                     break

  296.                    

  297.                 ##执行到这里的一定是在线任务，在等待流的过程中要发送waiting

  298.                 time_kafka1 = time.time()

  299.                 if time_kafka1-time_kafka0>60:

  300.                     msg_res = copy.deepcopy(msg_dict_off);

  301.                     msg_res['msg_id']= msgId; msg_res['type']=2

  302.                     msg_res = json.dumps(msg_res, ensure_ascii=False)        

  303.                     outStrList=  get_infos(taskId, msgId,msg_res,key_str='Waiting stream restoring heartbeat') 

  304.                     send_kafka(producer,kafka_par,msg_res,outStrList,fp_log,line=sys._getframe().f_lineno,logger=logger,thread=thread      );

  305.                     

  306.                     outstr='Waiting stream recovering:%.1f s'%(timeBreak1-timeBreak0)

  307.                     writeELK_log(msg=outstr,fp=fp_log,line=sys._getframe().f_lineno,logger=logger) 

  308.                     writeELK_log(msg=outstr_channel,fp=fp_log,line=sys._getframe().f_lineno,logger=logger)  

  309.                     time_kafka0 = time_kafka1

  310.             

  311.                 #break###断流或者到终点

  312.                  

  313.                 time.sleep(5) 

  314.                 print('Waiting stream for ',e)

  315. def lauch_process(gpuid,inSource,outSource,taskId,msgId,modelJson,kafka_par,channelIndex='LC001'):

  316. 

  317.     if outSource=='NO':

  318.         streamName='off-%s-%s'%(taskId,msgId)

  319.     else:

  320.         streamName='live-%s-%s'%(taskId,msgId)    

  321.     dataPar ={

  322.             'imgData':'',

  323.             'imgName':'testW',

  324.              'streamName':streamName,

  325.              'taskId':taskId,

  326.              'msgId':msgId,

  327.              'channelIndex':channelIndex,

  328.              'device':str(gpuid),

  329.              'modelJson':modelJson,

  330.              'kafka_par':kafka_par,

  331.         

  332.              }   

  333.     #dataPar['inSource'] = 'http://images.5gai.taauav.com/video/8bc32984dd893930dabb2856eb92b4d1.mp4';dataPar['outSource'] = None

  334.     dataPar['inSource'] = inSource;dataPar['outSource'] = outSource    

  335.     process_uid=''.join(random.sample(string.ascii_letters + string.digits, 16));dataPar['process_uid']=process_uid

  336.     parent_conn, child_conn = multiprocessing.Pipe();dataPar['callback']=child_conn

  337.     gpuProcess=Process(target=onlineModelProcess,name='process:%s'%( process_uid ),args=(dataPar,))

  338.     gpuProcess.start()

  339.     #print(dir(gpuProcess))

  340.     #child_return = parent_conn.recv()

  341.     #timestr2=time.strftime("%Y-%m-%d %H:%M:%S  ", time.localtime())

  342.     #print(timestr2,'-'*20,'progress:%s ,msgId:%s , taskId:%s  return:'%(process_uid,msgId,taskId),child_return)

  343.     

  344.     return gpuProcess

  345. 

  346.          

  347. msg_dict_offline = {

  348.                     

  349.                     "biz_id":"hehuzhang",

  350.                     "mod_id":"ai",

  351.                     "msg_id":'bb'+''.join(random.sample(string.ascii_letters ,30)  ) ,

  352.                     "offering_id":"http://vod.play.t-aaron.com/customerTrans/c49a2c620795d124f2ae4b10197b8d0e/303b7a58-17f3ef4494e-0004-f90c-f2c-7ec68.mp4",

  353.                     "offering_type":"mp4",

  354.                     "results_base_dir": "XJRW202203171535"+str(random.randint(10,99)),

  355.                     

  356.                     'outSource':'NO'

  357.                     

  358.                     

  359.                     }

  360. 

  361. 

  362. taskStatus={} 

  363. taskStatus['onLine'] = Queue(100)

  364. taskStatus['offLine']= Queue(100)   

  365. taskStatus['pidInfos']= {}

  366.     

  367. def get_msg_from_kafka(par):

  368.     thread='master:readingKafka' 

  369.     outStrList={}

  370.     fp_log = par['fp_log']

  371.     logger=par['logger']

  372.     consumer = KafkaConsumer(bootstrap_servers=par['server'],client_id='AI_server',group_id=par['group_id'],auto_offset_reset='latest')           

  373.     consumer.subscribe( par['topic'][0:2]) 

  374.     outstr='reading kafka process starts'

  375.     writeELK_log(msg=outstr,fp=fp_log,thread=thread,line=sys._getframe().f_lineno,logger=logger)

  376.     kafka_par ={ 'server':par['server'],'topic':par['topic'][2]  }

  377.     producer = KafkaProducer(

  378.                     bootstrap_servers=par['server'],#tencent yun 

  379.                     value_serializer=lambda v: v.encode('utf-8'),

  380.                     metadata_max_age_ms=120000)

  381. 

  382.     

  383.     for ii,msg in enumerate(consumer):

  384.         ##读取消息

  385.         try:

  386.             taskInfos = eval(msg.value.decode('utf-8')  )   

  387.         except:  

  388.             outstr='%s   msg format error,value:%s,offset:%d partition:%s topic:%s'%('#'*20,msg.value,msg.offset,msg.topic,msg.topic)

  389.             continue

  390.                 

  391.         if msg.topic == par['topic'][0]: ##               

  392.             taskInfos['inSource']= taskInfos['pull_channel']; 

  393.             taskInfos['outSource']=  get_push_address(taskInfos['push_channel']) ;    

  394.                      

  395.             taskStatus['onLine'].put( taskInfos )   

  396.             save_message(par['kafka'],taskInfos)     

  397.             

  398.             ###发送状态信息waiting

  399.             msg = create_status_msg(msg_dict_on,taskInfos,sts='waiting')       

  400.             outStrList=get_infos(taskInfos['results_base_dir'], taskInfos['msg_id'],msg,key_str='read msgs from kafka online task and response to kafka') 

  401.             send_kafka(producer,kafka_par,msg,outStrList,fp_log,line=sys._getframe().f_lineno,logger=logger,thread=thread); 

  402.         

  403.         else:     

  404.             try:

  405.                 taskInfos['inSource']= taskInfos['offering_id']; 

  406.                 taskInfos['outSource']= 'NO'

  407.                

  408.                 taskStatus['offLine'].put( taskInfos )

  409.                 save_message(par['kafka'],taskInfos)   

  410. 

  411.                 ###发送状态信息waiting 

  412.                 msg = create_status_msg(msg_dict_off,taskInfos,sts='waiting')   

  413.                 outStrList=get_infos(taskInfos['results_base_dir'], taskInfos['msg_id'],msg,key_str='read msgs from kafka offline task and response to kafka') 

  414.                 send_kafka(producer,kafka_par,msg,outStrList,fp_log ,line=sys._getframe().f_lineno,logger=logger,thread=thread    ); 

  415.             except Exception as e:

  416.                 print('######msg Error######',msg,e)            

  417.                 

  418.         

  419. def detector(par):

  420. 

  421.     ####初始化信息列表

  422.     kafka_par ={ 'server':par['server'],'topic':par['topic'][2]  }

  423.     producer = KafkaProducer(

  424.                     bootstrap_servers=par['server'],#tencent yun 

  425.                     value_serializer=lambda v: v.encode('utf-8'),

  426.                     metadata_max_age_ms=120000)

  427.     time_interval=par['logPrintInterval']

  428.     logname='detector.log';thread='master:detector' 

  429.     fp_log=create_logFile(logdir=par['logDir'],name=logname)

  430.     ##准备日志函数所需参数

  431.     logger=par['logDir'].replace('/','.')+'.'+logname

  432.     #wrtiteLog(fp_log,'########### detector process starts  ######\n');

  433.     outstr='detector process starts';sys._getframe().f_lineno

  434.     writeELK_log(msg=outstr,fp=fp_log,thread=thread,line=sys._getframe().f_lineno,logger=logger)

  435.     

  436.     ###开启kafka consumer 进程##

  437.     parIn=copy.deepcopy(par);parIn['fp_log']=fp_log ;parIn['logger']=logger

  438.     HeartProcess=Process(target=get_msg_from_kafka,name='process-consumer-kafka',args=(parIn,))

  439.     HeartProcess.start()

  440.     

  441.     

  442.     timeSleep=1

  443.     

  444.     time0=time.time()

  445.     time0_kafQuery=time.time()

  446.     time0_taskQuery=time.time()

  447.     time0_sleep=time.time()

  448.     outStrList={}

  449.     while True:###每隔timeSleep秒，轮询一次   

  450.     

  451.         time0_taskQuery,printFlag = check_time_interval(time0_taskQuery,time_interval)

  452.         outstr_task=  ' task queue onLine cnt:%d    offLine:%d'%(taskStatus['onLine'].qsize(), taskStatus['offLine'].qsize())      

  453.         if (taskStatus['onLine'].qsize()>0) or (taskStatus['offLine'].qsize()>0):

  454.             #outstr_task=wrtiteLog(fp_log,outstr_task);print( outstr_task);

  455.             writeELK_log(msg=outstr_task,fp=fp_log,thread=thread,line=sys._getframe().f_lineno,logger=logger)

  456.         ##2-更新显卡信息

  457.         gpuStatus = getGPUInfos()

  458.         

  459.         ##3-优先考虑在线任务

  460.         if not taskStatus['onLine'].empty():

  461.             ###3.1-先判断有没有空闲显卡：

  462.             cuda = get_available_gpu(gpuStatus)

  463.              ###获取在线任务信息，并执行,lauch process

  464.             taskInfos =   taskStatus['onLine'].get() 

  465.             outstr='start to process onLine taskId:%s    msgId:%s'%( taskInfos['results_base_dir'],taskInfos['msg_id'] )

  466.             #outstr=wrtiteLog(fp_log,outstr);print( outstr);

  467.             writeELK_log(msg=outstr,fp=fp_log,thread=thread,line=sys._getframe().f_lineno,logger=logger)

  468.             if cuda: ###3.1.1 -有空余显卡

  469.                 #lauch process

  470.                 msg=   copy.deepcopy(msg_dict_on);

  471.                 gpuProcess=lauch_process(cuda,taskInfos['inSource'],taskInfos['outSource'],taskInfos['results_base_dir'],taskInfos['msg_id'],par['modelJson'],kafka_par,taskInfos['channel_code'])

  472.                 taskStatus['pidInfos'][gpuProcess.pid] = {'gpuProcess':gpuProcess,'type':'onLine','taskInfos':taskInfos}

  473.                           

  474.             else:###3.1.2-没有显卡

  475.                 ##判断有没有显卡上面都是离线进程的

  476.                 cuda_pid = get_potential_gpu(gpuStatus,taskStatus['pidInfos'])

  477.           

  478.                 if cuda_pid:#3.1.2.1 - ##如果有可以杀死的进程

  479.                     cuda = cuda_pid['cuda']

  480.                     pids = cuda_pid['pids']

  481.                     ##kill 离线进程，并更新离线任务表

  482.                     cnt_off_0 = taskStatus['offLine'].qsize()

  483.                     for pid in pids:

  484.                         ##kill 离线进程

  485.                         taskStatus['pidInfos'][pid]['gpuProcess'].kill()

  486.                         ##更新离线任务表

  487.                         taskStatus['offLine'].put( taskStatus['pidInfos'][pid]['taskInfos']    )

  488.                         taskInfos_off=taskStatus['pidInfos'][pid]['taskInfos'] 

  489.                         

  490.                         ##发送离线数据，说明状态变成waiting

  491.                         msg=   msg_dict_off;

  492.                         msg=update_json(taskInfos_off,msg,offkeys=["msg_id","biz_id" ,"mod_id"] )

  493.                         msg['results'][0]['original_url']=taskInfos_off['inSource']

  494.                         msg['results'][0]['sign_url']=get_boradcast_address(taskInfos_off['outSource']) 

  495.                         msg['status']='waiting'                        

  496.                         msg = json.dumps(msg, ensure_ascii=False)  

  497. 

  498.                         

  499.                         outStrList=get_infos(taskInfos_off['results_base_dir'], taskInfos_off['msg_id'],msg,key_str='start online task after kill offline tasks') 

  500.                         

  501.                         send_kafka(producer,kafka_par,msg,outStrList,fp_log ,line=sys._getframe().f_lineno,logger=logger,thread=thread      ); 

  502.                         

  503.                         

  504.                     cnt_off_1 = taskStatus['offLine'].qsize()

  505.                     outstr='before killing process, offtask cnt:%d  ,after killing, offtask cnt:%d '%(cnt_off_0,cnt_off_1)

  506.                     #outstr=wrtiteLog(fp_log,outstr);print( outstr);

  507.                     writeELK_log(msg=outstr,fp=fp_log,thread=thread,line=sys._getframe().f_lineno,logger=logger)

  508.                     gpuProcess=lauch_process(cuda,taskInfos['inSource'],taskInfos['outSource'],taskInfos['results_base_dir'],taskInfos['msg_id'],par['modelJson'],kafka_par,taskInfos['channel_code'])

  509.                              

  510.                     ###更新pidinfos，update pidInfos

  511.                     taskStatus['pidInfos'][gpuProcess.pid] = {'gpuProcess':gpuProcess,'type':'onLine','taskInfos':taskInfos}

  512.                 else:

  513.                     outstr='No available GPUs for onLine task'

  514.                     #outstr=wrtiteLog(fp_log,outstr);print(outstr);      

  515.                     writeELK_log(msg=outstr,fp=fp_log,level='ERROR',thread=thread,line=sys._getframe().f_lineno,logger=logger)

  516.                     

  517.         ##4-更新显卡信息

  518.         gpuStatus = getGPUInfos()        

  519.         ##5-考虑离线任务

  520.         if not taskStatus['offLine'].empty():

  521.             cudaArrange= arrange_offlineProcess(gpuStatus,taskStatus['pidInfos'],modelMemory=1500)

  522.             outstr='IN OFF LINE TASKS available cudas:%s'%(cudaArrange)

  523.             #outstr=wrtiteLog(fp_log,outstr);print( outstr);   

  524.             writeELK_log(msg=outstr,fp=fp_log,thread=thread,line=sys._getframe().f_lineno,logger=logger)

  525.                      

  526.             for cuda in cudaArrange:

  527.                 if  not taskStatus['offLine'].empty():

  528.                     taskInfos =   taskStatus['offLine'].get()

  529.                     outstr='start to process offLine taskId:%s    msgId:%s'%( taskInfos['results_base_dir'],taskInfos['msg_id'] )

  530.                     taskInfos['channel_code']='LC999'###离线消息没有这个参数

  531.                     #outstr=wrtiteLog(fp_log,outstr);print( outstr); 

  532.                     writeELK_log(msg=outstr,fp=fp_log,thread=thread,line=sys._getframe().f_lineno,logger=logger)                    

  533.                     gpuProcess=lauch_process(cuda,taskInfos['inSource'],taskInfos['outSource'],taskInfos['results_base_dir'],taskInfos['msg_id'],par['modelJson'],kafka_par,taskInfos['channel_code'])

  534.                     taskStatus['pidInfos'][gpuProcess.pid] = {'gpuProcess':gpuProcess,'type':'offLine','taskInfos':taskInfos}   

  535.                     

  536.                   

  537.                    

  538.                     

  539.         if get_whether_gpuProcess():                 

  540.             time0_sleep,printFlag = check_time_interval(time0_sleep,time_interval)

  541.             if printFlag:

  542.                outstr=  '*'*20 +'sleep '+'*'*20;  

  543.                #outstr=wrtiteLog(fp_log,outstr);print( outstr);

  544.                writeELK_log(msg=outstr,fp=fp_log,thread=thread,line=sys._getframe().f_lineno,logger=logger)   

  545.                

  546.         time.sleep(timeSleep)

  547.        

  548.     print('########Program End#####')

  549.     

  550. 

  551.                     

  552. if __name__ == '__main__':

  553.     par={};

  554.     ###topic0--在线，topic1--离线

  555. 

  556.     #par['server']='212.129.223.66:9092';par['topic']=('thsw','thsw2','testReturn');par['group_id']='test';

  557.     #101.132.127.1:19092  

  558.     

  559.     '''

  560.     par['server']='101.132.127.1:19092 ';par['topic']=('alg-online-tasks','alg-offline-tasks','alg-task-results');par['group_id']='test';

  561.     

  562.     par['kafka']='mintors/kafka'

  563.     par['modelJson']='conf/model.json'

  564.     '''

  565.     masterFile="conf/master.json"

  566.     assert os.path.exists(masterFile)

  567.     with open(masterFile,'r') as fp:

  568.         data=json.load(fp)   

  569.     par=data['par']        

  570.     print(par)   

  571.     detector(par)

  572. 

  573.     

  574.     

  575.     

  576.     

  577.