208 lines
9.4 KiB
Python
208 lines
9.4 KiB
Python
|
from aliyunsdkvod.request.v20170321 import GetPlayInfoRequest
|
|||
|
|
import json
|
|||
|
|
import traceback
|
|||
|
|
from aliyunsdkcore.client import AcsClient
|
|||
|
|
|
|||
|
|
from PIL import Image
|
|||
|
|
import numpy as np
|
|||
|
|
import cv2
|
|||
|
|
import base64
|
|||
|
|
import io,os,copy
|
|||
|
|
import requests
|
|||
|
|
import time,json
|
|||
|
|
import string,random
|
|||
|
|
import glob,string,sys
|
|||
|
|
from multiprocessing import Process,Queue
|
|||
|
|
import oss2
|
|||
|
|
from kafka import KafkaProducer, KafkaConsumer
|
|||
|
|
from voduploadsdk.UploadVideoRequest import UploadVideoRequest
|
|||
|
|
from voduploadsdk.AliyunVodUtils import *
|
|||
|
|
from voduploadsdk.AliyunVodUploader import AliyunVodUploader
|
|||
|
|
from datetime import datetime, date, timedelta
|
|||
|
|
def get_today():
|
|||
|
|
return date.today().strftime("%Y-%m-%d")
|
|||
|
|
def get_yesterday(beforeday=-1):
|
|||
|
|
return (date.today() + timedelta(days =beforeday)).strftime("%Y-%m-%d")
|
|||
|
|
|
|||
|
|
def get_videoUurl(videoBakDir,filename):
|
|||
|
|
###七天时间内
|
|||
|
|
potentialUrls=[ os.path.join( videoBakDir,get_yesterday(beforeday=-x),filename) for x in range(7) ]
|
|||
|
|
existsList=[os.path.exists(x ) for x in potentialUrls]
|
|||
|
|
for i,flag in enumerate(existsList):
|
|||
|
|
if flag: return potentialUrls[i]
|
|||
|
|
return potentialUrls[0]
|
|||
|
|
|
|||
|
|
def getNamedic(jsonfile):
|
|||
|
|
with open(jsonfile) as fp:
|
|||
|
|
dataDic=json.load(fp)
|
|||
|
|
#"labelnames":["排口","排污口","水生植被","漂浮物","其它"],
|
|||
|
|
#"labelIndexs":["SL014","SL011","SL013","SL001","SL001" ]
|
|||
|
|
|
|||
|
|
assert 'labelnames' in dataDic.keys() , 'labelnames is not the key in %s'%(jsonfile)
|
|||
|
|
assert 'labelIndexs' in dataDic.keys() , 'labelIndexs is not the key in %s'%(jsonfile)
|
|||
|
|
assert len(dataDic['labelnames'])==len(dataDic['labelIndexs'])
|
|||
|
|
nameDic={}
|
|||
|
|
for key,value in zip(dataDic['labelnames'],dataDic['labelIndexs']):
|
|||
|
|
nameDic[key]=value
|
|||
|
|
return nameDic
|
|||
|
|
|
|||
|
|
|
|||
|
|
def get_play_info(clt, videoId):
|
|||
|
|
request = GetPlayInfoRequest.GetPlayInfoRequest()
|
|||
|
|
request.set_accept_format('JSON')
|
|||
|
|
request.set_VideoId(videoId)
|
|||
|
|
request.set_AuthTimeout(3600*5)
|
|||
|
|
response = json.loads(clt.do_action_with_exception(request))
|
|||
|
|
return response
|
|||
|
|
|
|||
|
|
def create_status_msg(msg_dict_off,taskInfos,sts='waiting'):
|
|||
|
|
msg= copy.deepcopy(msg_dict_off)
|
|||
|
|
msg=update_json(taskInfos,msg,offkeys=["request_id"] )
|
|||
|
|
msg['status']=sts
|
|||
|
|
msg = json.dumps(msg, ensure_ascii=False)
|
|||
|
|
return msg
|
|||
|
|
# 填入AccessKey信息
|
|||
|
|
def init_vod_client(accessKeyId, accessKeySecret):
|
|||
|
|
regionId = 'cn-shanghai' # 点播服务接入地域
|
|||
|
|
connectTimeout = 3 # 连接超时，单位为秒
|
|||
|
|
return AcsClient(accessKeyId, accessKeySecret, regionId, auto_retry=True, max_retry_time=3, timeout=connectTimeout)
|
|||
|
|
def update_json(jsonOri,jsonNew,offkeys=["request_id" ]):
|
|||
|
|
#{'biz_id': 'hehuzhang', 'mod_id': 'ai', 'request_id': 'bblvgyntTsZCamqjuLArkiSYIbKXEeWx', 'offering_id': 'http://vod.play.t-aaron.com/customerTrans/c49a2c620795d124f2ae4b10197b8d0e/303b7a58-17f3ef4494e-0004-f90c-f2c-7ec68.mp4', 'offering_type': 'mp4', 'results_base_dir': 'XJRW20220317153547', 'inSource': 'http://vod.play.t-aaron.com/customerTrans/c49a2c620795d124f2ae4b10197b8d0e/303b7a58-17f3ef4494e-0004-f90c-f2c-7ec68.mp4', 'outSource': 'NO'}
|
|||
|
|
for key in offkeys:
|
|||
|
|
jsonNew[key] = jsonOri[key]
|
|||
|
|
return jsonNew
|
|||
|
|
def get_time(filename):
|
|||
|
|
#2021-10-09-11-44-51_frame-598-720_type-水生植被.jpg
|
|||
|
|
sps=filename.strip().split('_')[0]
|
|||
|
|
tsps=sps.split('-')
|
|||
|
|
return '%s-%s-%s %s:%s:%s'%(tsps[0],tsps[1],tsps[2],tsps[3],tsps[4],tsps[5])
|
|||
|
|
def get_ms(time0,time1):
|
|||
|
|
str_time ='%.2f ms'%((time1-time0)*1000)
|
|||
|
|
return str_time
|
|||
|
|
|
|||
|
|
def get_urls( platform_query_url,fp_log ):
|
|||
|
|
try:
|
|||
|
|
if os.path.exists(platform_query_url):
|
|||
|
|
#print('###line49')
|
|||
|
|
with open('SendLog/platformQuery.json','r') as fp:
|
|||
|
|
res = json.load(fp)
|
|||
|
|
else:
|
|||
|
|
res = requests.get(platform_query_url,timeout=10).json()
|
|||
|
|
#print('###line54')
|
|||
|
|
questionUrl = res['data']['questionUrl'] ###直播流时，问题图片的推送地址
|
|||
|
|
offlineUrl = res['data']['offlineUrl'] ###http离线视频时，问题图片的推送地址
|
|||
|
|
except Exception as ee:
|
|||
|
|
timestr=time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
|||
|
|
print('###### %s: file:send_transfer: error %s ,url:%s #####'%(timestr,ee,platform_query_url))
|
|||
|
|
outstr = '\n %s ###### get url platform error : update error:%s , url:%s'%( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) ,ee,platform_query_url)
|
|||
|
|
fp_log.write(outstr);fp_log.flush()
|
|||
|
|
questionUrl="http://47.96.182.154:9040/api/taskFile/submitUAVKHQuestion"
|
|||
|
|
offlineUrl ="http://47.96.182.154:9040/api/taskFile/submitUAVKHQuestion"
|
|||
|
|
return questionUrl,offlineUrl
|
|||
|
|
def parse_filename(filename_base):
|
|||
|
|
#etc:2022-01-13-16-04-17_frame-823-1440_type-水生植被_hgYFEulc0dPIrG1S_s-off-XJRW20220113154959_AI.jpg
|
|||
|
|
uid =filename_base.split('.')[0].split('_')[3].strip()
|
|||
|
|
sourceType=filename_base.split('_')[4].split('-')[1]
|
|||
|
|
sourceId=filename_base.split('_')[4].split('-')[2]
|
|||
|
|
typename=filename_base.split('.')[0].split('_')[2].split('-')[1].strip()
|
|||
|
|
return uid,sourceType,sourceId,typename
|
|||
|
|
def b64encode_function(filename, filename_OR):
|
|||
|
|
if os.path.exists(filename):
|
|||
|
|
image_ori=cv2.imread(filename)
|
|||
|
|
image_ori_OR=cv2.imread(filename_OR)
|
|||
|
|
else:
|
|||
|
|
image_ori = filename.copy()
|
|||
|
|
image_ori_OR = image_ori_OR.copy()
|
|||
|
|
image_pngcode = cv2.imencode('.jpg',image_ori)[-1]
|
|||
|
|
image_pngcode_OR = cv2.imencode('.jpg',image_ori_OR)[-1]
|
|||
|
|
image_code = str(base64.b64encode(image_pngcode))[2:-1]
|
|||
|
|
image_code_OR = str(base64.b64encode(image_pngcode_OR))[2:-1]
|
|||
|
|
return image_code, image_code_OR
|
|||
|
|
def JsonSend(parIn):
|
|||
|
|
|
|||
|
|
fp_log = parIn['fp_log']
|
|||
|
|
try:
|
|||
|
|
response=requests.post(parIn['api'],json=parIn['input_'],timeout=10).json()
|
|||
|
|
t3 = time.time()
|
|||
|
|
print('\n file:%s encodetime:%.5f request time:%.5f,send to %s ,return code:%s, size:%.2f M \n'%(parIn['filename_base'],parIn['t2']-parIn['t1'],t3-parIn['t2'],api,response['code'],parIn['sizeImage']))
|
|||
|
|
outstr = '%s file:%s encodetime:%.5f request time:%.5f,send to %s ,return code:%s,size:%.2f M ,%s\n'%( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),parIn['filename_base'],parIn['t2']-parIn['t1'],t3-parIn['t2'],parIn['api'],response['code'],parIn['sizeImage'],parIn['dic_str'])
|
|||
|
|
fp_log.write(outstr);fp_log.flush()
|
|||
|
|
|
|||
|
|
except Exception as ee:
|
|||
|
|
print('\n ######file:%s: upload error:%s,size:%.2f M'%(parIn['filename_base'],ee, parIn['sizeImage']))
|
|||
|
|
outstr = '\n%s ###### file:%s: upload error:%s , size:%.2f M'%( time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) ,parIn['filename_base'],ee,parIn['sizeImage'])
|
|||
|
|
fp_log.write(outstr);fp_log.flush()
|
|||
|
|
|
|||
|
|
|
|||
|
|
def dic2str(dic):
|
|||
|
|
st=''
|
|||
|
|
for key in dic.keys():
|
|||
|
|
st='%s %s:%s,'%(st,key,dic[key])
|
|||
|
|
return st
|
|||
|
|
def createJsonInput(filename,offlineUrl,questionUrl):
|
|||
|
|
flag = True
|
|||
|
|
filename_base = os.path.basename(filename)
|
|||
|
|
filename_OR=filename.replace('_AI.','_OR.')
|
|||
|
|
if not os.path.exists(filename_OR ):
|
|||
|
|
return False
|
|||
|
|
|
|||
|
|
uid,sourceType, sourceId,typename = parse_filename(filename_base)
|
|||
|
|
if (typename not in name_dic.keys()) or (typename == '排口'):
|
|||
|
|
return False
|
|||
|
|
api = questionUrl if sourceType=='live' else offlineUrl
|
|||
|
|
|
|||
|
|
time_str = get_time(filename_base)
|
|||
|
|
input_ ={
|
|||
|
|
'imgName':os.path.basename(filename),
|
|||
|
|
'imgNameOriginal':os.path.basename(filename_OR),
|
|||
|
|
'time':time_str,
|
|||
|
|
'fid':uid, ###随机16位字符
|
|||
|
|
'type':name_dic[typename],###这次先采用 ["排口","污口","水生植被","漂浮物","其它"]
|
|||
|
|
'typeId':nameID_dic[typename]
|
|||
|
|
|
|||
|
|
}
|
|||
|
|
if sourceType!='live':
|
|||
|
|
input_['code']=sourceId;###只有离线视频才需要code，
|
|||
|
|
|
|||
|
|
dic_str = dic2str(input_)
|
|||
|
|
t1 = time.time()
|
|||
|
|
|
|||
|
|
image_code, image_code_OR = b64encode_function(filename, filename_OR)
|
|||
|
|
input_['imgData']=image_code
|
|||
|
|
input_['imgDataOriginal']=image_code_OR
|
|||
|
|
|
|||
|
|
sizeImage = (len(image_code) + len(image_code_OR) )/1000000.0
|
|||
|
|
|
|||
|
|
parOut={};parOut['flag']=True;parOut['input_']=input_;
|
|||
|
|
parOut['sizeImage']=sizeImage;parOut['dic_str']=dic_str;
|
|||
|
|
parOut['filename']=filename;parOut['filename_OR']=filename_OR;
|
|||
|
|
parOut['api']=api ; parOut['t1']=t1 ; parOut['filename_base']= filename_base
|
|||
|
|
return parOut
|
|||
|
|
|
|||
|
|
def getLogFileFp(streamName):
|
|||
|
|
logname ='SendLog/'+ time.strftime("%Y-%m-%d", time.localtime())+'_%s.txt'%(streamName)
|
|||
|
|
if os.path.exists(logname):
|
|||
|
|
fp_log = open(logname,'a+')
|
|||
|
|
else:
|
|||
|
|
fp_log = open(logname,'w')
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
def lodaMsgInfos(jsonDir,msgId):
|
|||
|
|
jsonUrl = os.path.join(jsonDir,msgId+'.json')
|
|||
|
|
with open(jsonUrl,'r') as fp:
|
|||
|
|
data=json.load(fp)
|
|||
|
|
return data
|
|||
|
|
|
|||
|
|
def parse_filename_for_oss(name):
|
|||
|
|
splts=name.split('_')
|
|||
|
|
typename=splts[2].split('-')[1].strip()
|
|||
|
|
msgId=splts[4].split('-')[3]
|
|||
|
|
onLineType=splts[4].split('-')[1]
|
|||
|
|
return typename,msgId,onLineType
|
|||
|
|
def percentage(consumed_bytes, total_bytes):
|
|||
|
|
if total_bytes:
|
|||
|
|
rate = int(100 * (float(consumed_bytes) / float(total_bytes)))
|
|||
|
|
print('\r{0}% '.format(rate), end='')
|
|||
|
|
sys.stdout.flush()
|
|||
|
|
|