|
| 1 | +import time |
1 | 2 | import os |
2 | | -from glob import glob |
| 3 | +import re |
| 4 | +import argparse |
| 5 | +import smtplib |
| 6 | +from email.mime.text import MIMEText |
| 7 | +from email.header import Header |
| 8 | +import smtplib |
3 | 9 |
|
4 | | -work_dir='/home/mic_dachuang/B/mmdetection/work_dir_new_4_14/ssd512_default' |
5 | | -config='/home/mic_dachuang/B/mmdetection/modified_configs/ssd512_coco.py' |
6 | 10 |
|
7 | | -f=open(work_dir+'testall.txt','w+') |
8 | | -content='' |
9 | | -for epoch in glob(work_dir+'/*.pth'): |
10 | | - print("Testing "+epoch+'...') |
11 | | - raw=os.popen('pyhton tools/test.py '+config+' '+epoch+' --eval bbox').readlines() |
12 | | - result=raw[-12:0] |
| 11 | +#log_path='/home/mic_dachuang/B/mmdetection/work_dir_new_4_14/faster_adagrad/20200421_015049.log' |
| 12 | +log_path='20200409_202738.log' |
| 13 | +model_path='' |
| 14 | +config_path='' |
| 15 | +model_name=input('自定义模型名称,则为邮件发送时的送信人名称:') |
| 16 | +mail_addr=input('想接收的邮件地址,若不输入则为默认:') |
| 17 | + |
| 18 | +current_epoch=0 |
| 19 | + |
| 20 | +def parse_args(): |
| 21 | + parser = argparse.ArgumentParser() |
| 22 | + parser.add_argument('log_path') |
| 23 | + parser.add_argument('model_path') |
| 24 | + parser.add_argument('config_path') |
| 25 | + args = parser.parse_args() |
| 26 | + |
| 27 | + return args |
| 28 | + |
| 29 | +def send_mail(log): |
| 30 | + global model_name, mail_addr |
| 31 | + msg=MIMEText(str(log),'plain','utf-8') |
| 32 | + usrn='smtpservicetest@163.com' |
| 33 | + pswd='BBSKRKQDYQFJIXKG' |
| 34 | + if not mail_addr: sendto='random_name@sjtu.edu.cn' |
| 35 | + else: send_to=mail_addr |
| 36 | + msg['From']='Training Brief' |
| 37 | + msg['To']=sendto |
| 38 | + msg['Subject']='Status update of model: '+model_name |
| 39 | + smtp=smtplib.SMTP() |
| 40 | + #smtp.set_debuglevel(1) |
| 41 | + smtp.connect('smtp.163.com',25) |
| 42 | + smtp.login(usrn,pswd) |
| 43 | + smtp.sendmail(usrn,[sendto],msg.as_string()) |
| 44 | + |
| 45 | + |
| 46 | +def current_log(): |
| 47 | + with open(log_path,'r') as f: |
| 48 | + log=f.readlines() |
| 49 | + return log[-1] |
| 50 | + |
| 51 | +def parse(log): |
| 52 | + global current_epoch |
| 53 | + if re.search('workflow:',log)==None: |
| 54 | + if re.search('Epoch',log)==None: |
| 55 | + return 0 |
| 56 | + else: |
| 57 | + e=int(re.search('Epoch.+?\d\d?',log).group()[7:]) |
| 58 | + if not e==current_epoch: |
| 59 | + current_epoch=e |
| 60 | + return 3 |
| 61 | + else: return 2 |
| 62 | + else: |
| 63 | + print('Watched:',log) |
| 64 | + return 1 |
| 65 | +def check(): |
| 66 | + started=0 |
| 67 | + log=current_log() |
| 68 | + status=parse(log) |
| 69 | + if not status: |
| 70 | + print('Not started.') |
| 71 | + elif status==1: |
| 72 | + if not started: |
| 73 | + print('Started.') |
| 74 | + send_mail('training started, \n'+log) |
| 75 | + started=1 |
| 76 | + elif status==3: |
| 77 | + print('Epoch ',current_epoch,' Started, more info:\n',log) |
| 78 | + send_mail('Epoch '+str(current_epoch)+' Started,\n'+log) |
| 79 | + test(current_epoch) |
| 80 | + |
| 81 | + |
| 82 | +def test(epoch): |
| 83 | + print('Testing epoch ',str(epoch-1)) |
| 84 | + model=model_path+'/epoch_'+str(epoch-1)+'.pth' |
| 85 | + raw=os.popen('python tools/test.py '+config_path+' '+model_path+'/epoch_'+str(epoch-1)+'.pth --eval bbox').readlines() |
| 86 | + result=raw[-12:] |
13 | 87 | print(result) |
14 | | - content=content+result+'\n' |
15 | | -f.write(content) |
16 | | -f.flush() |
| 88 | + send_mail('The result of '+model+':\n'+str(result)) |
| 89 | + |
| 90 | + |
| 91 | + |
| 92 | +def timer(): |
| 93 | + global log_path, model_path, config_path |
| 94 | + args=parse_args() |
| 95 | + log_path=args.log_path |
| 96 | + model_path=args.model_path |
| 97 | + config_path=args.config_path |
| 98 | + |
| 99 | + while(True): |
| 100 | + check() |
| 101 | + time.sleep(10) |
| 102 | +timer() |
| 103 | + |
| 104 | + |
| 105 | + |
0 commit comments