You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

207 lines
9.3KB

  1. from aliyunsdkvod.request.v20170321 import GetPlayInfoRequest
  2. import json
  3. import traceback
  4. from aliyunsdkcore.client import AcsClient
  5. from PIL import Image
  6. import numpy as np
  7. import cv2
  8. import base64
  9. import io,os,copy
  10. import requests
  11. import time,json
  12. import string,random
  13. import glob,string,sys
  14. from multiprocessing import Process,Queue
  15. import oss2
  16. from kafka import KafkaProducer, KafkaConsumer
  17. from voduploadsdk.UploadVideoRequest import UploadVideoRequest
  18. from voduploadsdk.AliyunVodUtils import *
  19. from voduploadsdk.AliyunVodUploader import AliyunVodUploader
  20. from datetime import datetime, date, timedelta
  21. def get_today():
  22. return date.today().strftime("%Y-%m-%d")
  23. def get_yesterday(beforeday=-1):
  24. return (date.today() + timedelta(days =beforeday)).strftime("%Y-%m-%d")
  25. def get_videoUurl(videoBakDir,filename):
  26. ###七天时间内
  27. potentialUrls=[ os.path.join( videoBakDir,get_yesterday(beforeday=-x),filename) for x in range(7) ]
  28. existsList=[os.path.exists(x ) for x in potentialUrls]
  29. for i,flag in enumerate(existsList):
  30. if flag: return potentialUrls[i]
  31. return potentialUrls[0]
  32. def getNamedic(jsonfile):
  33. with open(jsonfile) as fp:
  34. dataDic=json.load(fp)
  35. #"labelnames":["排口","排污口","水生植被","漂浮物","其它"],
  36. #"labelIndexs":["SL014","SL011","SL013","SL001","SL001" ]
  37. assert 'labelnames' in dataDic.keys() , 'labelnames is not the key in %s'%(jsonfile)
  38. assert 'labelIndexs' in dataDic.keys() , 'labelIndexs is not the key in %s'%(jsonfile)
  39. assert len(dataDic['labelnames'])==len(dataDic['labelIndexs'])
  40. nameDic={}
  41. for key,value in zip(dataDic['labelnames'],dataDic['labelIndexs']):
  42. nameDic[key]=value
  43. return nameDic
  44. def get_play_info(clt, videoId):
  45. request = GetPlayInfoRequest.GetPlayInfoRequest()
  46. request.set_accept_format('JSON')
  47. request.set_VideoId(videoId)
  48. request.set_AuthTimeout(3600*5)
  49. response = json.loads(clt.do_action_with_exception(request))
  50. return response
  51. def create_status_msg(msg_dict_off,taskInfos,sts='waiting'):
  52. msg= copy.deepcopy(msg_dict_off)
  53. msg=update_json(taskInfos,msg,offkeys=["request_id"] )
  54. msg['status']=sts
  55. msg = json.dumps(msg, ensure_ascii=False)
  56. return msg
  57. # 填入AccessKey信息
  58. def init_vod_client(accessKeyId, accessKeySecret):
  59. regionId = 'cn-shanghai' # 点播服务接入地域
  60. connectTimeout = 3 # 连接超时,单位为秒
  61. return AcsClient(accessKeyId, accessKeySecret, regionId, auto_retry=True, max_retry_time=3, timeout=connectTimeout)
  62. def update_json(jsonOri,jsonNew,offkeys=["request_id" ]):
  63. #{'biz_id': 'hehuzhang', 'mod_id': 'ai', 'request_id': 'bblvgyntTsZCamqjuLArkiSYIbKXEeWx', 'offering_id': 'http://vod.play.t-aaron.com/customerTrans/c49a2c620795d124f2ae4b10197b8d0e/303b7a58-17f3ef4494e-0004-f90c-f2c-7ec68.mp4', 'offering_type': 'mp4', 'results_base_dir': 'XJRW20220317153547', 'inSource': 'http://vod.play.t-aaron.com/customerTrans/c49a2c620795d124f2ae4b10197b8d0e/303b7a58-17f3ef4494e-0004-f90c-f2c-7ec68.mp4', 'outSource': 'NO'}
  64. for key in offkeys:
  65. jsonNew[key] = jsonOri[key]
  66. return jsonNew
  67. def get_time(filename):
  68. #2021-10-09-11-44-51_frame-598-720_type-水生植被.jpg
  69. sps=filename.strip().split('_')[0]
  70. tsps=sps.split('-')
  71. return '%s-%s-%s %s:%s:%s'%(tsps[0],tsps[1],tsps[2],tsps[3],tsps[4],tsps[5])
  72. def get_ms(time0,time1):
  73. str_time ='%.2f ms'%((time1-time0)*1000)
  74. return str_time
  75. def get_urls( platform_query_url,fp_log ):
  76. try:
  77. if os.path.exists(platform_query_url):
  78. #print('###line49')
  79. with open('SendLog/platformQuery.json','r') as fp:
  80. res = json.load(fp)
  81. else:
  82. res = requests.get(platform_query_url,timeout=10).json()
  83. #print('###line54')
  84. questionUrl = res['data']['questionUrl'] ###直播流时,问题图片的推送地址
  85. offlineUrl = res['data']['offlineUrl'] ###http离线视频时,问题图片的推送地址
  86. except Exception as ee:
  87. timestr=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
  88. print('###### %s: file:send_transfer: error %s ,url:%s #####'%(timestr,ee,platform_query_url))
  89. outstr = '\n %s ###### get url platform error : update error:%s , url:%s'%( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) ,ee,platform_query_url)
  90. fp_log.write(outstr);fp_log.flush()
  91. questionUrl="http://47.96.182.154:9040/api/taskFile/submitUAVKHQuestion"
  92. offlineUrl ="http://47.96.182.154:9040/api/taskFile/submitUAVKHQuestion"
  93. return questionUrl,offlineUrl
  94. def parse_filename(filename_base):
  95. #etc:2022-01-13-16-04-17_frame-823-1440_type-水生植被_hgYFEulc0dPIrG1S_s-off-XJRW20220113154959_AI.jpg
  96. uid =filename_base.split('.')[0].split('_')[3].strip()
  97. sourceType=filename_base.split('_')[4].split('-')[1]
  98. sourceId=filename_base.split('_')[4].split('-')[2]
  99. typename=filename_base.split('.')[0].split('_')[2].split('-')[1].strip()
  100. return uid,sourceType,sourceId,typename
  101. def b64encode_function(filename, filename_OR):
  102. if os.path.exists(filename):
  103. image_ori=cv2.imread(filename)
  104. image_ori_OR=cv2.imread(filename_OR)
  105. else:
  106. image_ori = filename.copy()
  107. image_ori_OR = image_ori_OR.copy()
  108. image_pngcode = cv2.imencode('.jpg',image_ori)[-1]
  109. image_pngcode_OR = cv2.imencode('.jpg',image_ori_OR)[-1]
  110. image_code = str(base64.b64encode(image_pngcode))[2:-1]
  111. image_code_OR = str(base64.b64encode(image_pngcode_OR))[2:-1]
  112. return image_code, image_code_OR
  113. def JsonSend(parIn):
  114. fp_log = parIn['fp_log']
  115. try:
  116. response=requests.post(parIn['api'],json=parIn['input_'],timeout=10).json()
  117. t3 = time.time()
  118. print('\n file:%s encodetime:%.5f request time:%.5f,send to %s ,return code:%s, size:%.2f M \n'%(parIn['filename_base'],parIn['t2']-parIn['t1'],t3-parIn['t2'],api,response['code'],parIn['sizeImage']))
  119. outstr = '%s file:%s encodetime:%.5f request time:%.5f,send to %s ,return code:%s,size:%.2f M ,%s\n'%( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),parIn['filename_base'],parIn['t2']-parIn['t1'],t3-parIn['t2'],parIn['api'],response['code'],parIn['sizeImage'],parIn['dic_str'])
  120. fp_log.write(outstr);fp_log.flush()
  121. except Exception as ee:
  122. print('\n ######file:%s: upload error:%s,size:%.2f M'%(parIn['filename_base'],ee, parIn['sizeImage']))
  123. outstr = '\n%s ###### file:%s: upload error:%s , size:%.2f M'%( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) ,parIn['filename_base'],ee,parIn['sizeImage'])
  124. fp_log.write(outstr);fp_log.flush()
  125. def dic2str(dic):
  126. st=''
  127. for key in dic.keys():
  128. st='%s %s:%s,'%(st,key,dic[key])
  129. return st
  130. def createJsonInput(filename,offlineUrl,questionUrl):
  131. flag = True
  132. filename_base = os.path.basename(filename)
  133. filename_OR=filename.replace('_AI.','_OR.')
  134. if not os.path.exists(filename_OR ):
  135. return False
  136. uid,sourceType, sourceId,typename = parse_filename(filename_base)
  137. if (typename not in name_dic.keys()) or (typename == '排口'):
  138. return False
  139. api = questionUrl if sourceType=='live' else offlineUrl
  140. time_str = get_time(filename_base)
  141. input_ ={
  142. 'imgName':os.path.basename(filename),
  143. 'imgNameOriginal':os.path.basename(filename_OR),
  144. 'time':time_str,
  145. 'fid':uid, ###随机16位字符
  146. 'type':name_dic[typename],###这次先采用 ["排口","污口","水生植被","漂浮物","其它"]
  147. 'typeId':nameID_dic[typename]
  148. }
  149. if sourceType!='live':
  150. input_['code']=sourceId;###只有离线视频才需要code,
  151. dic_str = dic2str(input_)
  152. t1 = time.time()
  153. image_code, image_code_OR = b64encode_function(filename, filename_OR)
  154. input_['imgData']=image_code
  155. input_['imgDataOriginal']=image_code_OR
  156. sizeImage = (len(image_code) + len(image_code_OR) )/1000000.0
  157. parOut={};parOut['flag']=True;parOut['input_']=input_;
  158. parOut['sizeImage']=sizeImage;parOut['dic_str']=dic_str;
  159. parOut['filename']=filename;parOut['filename_OR']=filename_OR;
  160. parOut['api']=api ; parOut['t1']=t1 ; parOut['filename_base']= filename_base
  161. return parOut
  162. def getLogFileFp(streamName):
  163. logname ='SendLog/'+ time.strftime("%Y-%m-%d", time.localtime())+'_%s.txt'%(streamName)
  164. if os.path.exists(logname):
  165. fp_log = open(logname,'a+')
  166. else:
  167. fp_log = open(logname,'w')
  168. return
  169. def lodaMsgInfos(jsonDir,msgId):
  170. jsonUrl = os.path.join(jsonDir,msgId+'.json')
  171. with open(jsonUrl,'r') as fp:
  172. data=json.load(fp)
  173. return data
  174. def parse_filename_for_oss(name):
  175. splts=name.split('_')
  176. typename=splts[2].split('-')[1].strip()
  177. msgId=splts[4].split('-')[3]
  178. onLineType=splts[4].split('-')[1]
  179. return typename,msgId,onLineType
  180. def percentage(consumed_bytes, total_bytes):
  181. if total_bytes:
  182. rate = int(100 * (float(consumed_bytes) / float(total_bytes)))
  183. print('\r{0}% '.format(rate), end='')
  184. sys.stdout.flush()