# !/usr/bin/env python# encoding:utf-8# from selenium import webdriverimport timeimport reimport jsonfrom bs4 import BeautifulSoupimport requestsimport pymsteamsdef _get_table():    while True:        try:            html = requests.get('https://builds.wgti.net', verify=False)            body = html.text            if html.status_code == 200:                break        except Exception as e:            print e            time.sleep(60)    soup = BeautifulSoup(body, 'lxml')    tables = soup.findAll('table')    table2 = tables[1]    return table2def _get_rows():    content = ''    row = []    table = _get_table()    for tr in table.findAll('tr'):        for td in tr.findAll('td'):            # print td.getText()            content = content + td.getText() + ' '        row.append(content)        content = ''    row.pop(0)    row.pop()    new_rows = []    for item in row:        new_rows.append(item.strip())    print new_rows    return new_rowsdef _get_date(row_list):    date = []    for item in row_list:        # print item        dt = re.search(r'\d+/\d+/\d+ \d+:\d+', item).group(0)        date.append(dt)    return datedef _get_arr_no_date(row_list):    new_arr = []    for item in row_list:        new_item = re.sub('\d+/\d+/\d+ \d+:\d+', '', item)        new_arr.append(new_item)    return new_arrdef _get_project(row_list):    project = []    new_arr_list = _get_arr_no_date(row_list)    for item in new_arr_list:        project.append(item.split()[0])    return projectdef _get_status(row_list):    status = []    new_arr_list = _get_arr_no_date(row_list)    for item in new_arr_list:        status.append(item.split()[1])    return statusdef _get_build(row_list):    build = []    new_arr_list = _get_arr_no_date(row_list)    for item in new_arr_list:        build.append(item.split()[2])    return builddef check_status(sta):    if sta == 'Idle':        return True    else:        return Falsedef _post_data(data):    data = json.dumps(data)    new_data = re.sub('"true"', 'true', data)    print new_data    # response = requests.post('http://watsng-staging.wgti.net/job/', new_data)    # response = requests.post('http://10.138.110.92/job/', new_data)    response = requests.post('http://web/job/', new_data)  # inside url in docker    print response.textdef send_job_request(job_list):    data = dict(library_branch='dev', suite_name='nightly-regression',                build='', branch='', submitter='wats-ng@watchguard.com',                script_branch='dev', private_rig='RIG-Nightly-regression',                nightly_build='true')    if job_list == success:        for item in job_list:            branch = item.split()[0]            build = item.split()[2]            if branch in white_list:                data['suite_name'] = 'Nightly-' + branch                data['branch'] = branch                data['build'] = build                _post_data(data)                time.sleep(3)                print 'Success job request send completed!'            else:                print 'branch: %s is success, but it is not in white list, do NOT send success job request.' % branch    else:        for item in job_list:            if item == 'utm-inc':                print 'build: \'utm-inc\' is failed, but do NOT send fail job request.'            else:                data['suite_name'] = 'Nightly-' + item                data['branch'] = item                data['build'] = 'failed'                _post_data(data)                time.sleep(3)                print 'Fail job request send completed!'def get_diff_indx(list1, list2):    id = []    for i in xrange(len(list1)):        item_a = list1[i]        item_b = list2[i]        if item_a != item_b:            id.append(i)    return iddef get_equal_index(list1, list2):    id = []    for i in xrange(len(list1)):        item_a = list1[i]        item_b = list2[i]        if item_a == item_b:            id.append(i)    return iddef get_unfinished_and_failed(old, new, content):    diff_id = get_diff_indx(old, new)    equal_id = get_equal_index(old, new)    for i in diff_id:        if old[i] != 'Idle' and new[i] == 'Idle':            fail.append(content[i])   # send notification-------------->>>>>            reason = 'This branch: %s\'s status change from \'non-Idle\' to \'Idle\' ' \                     'but there is no new image generation' % content[i]            fail_dict['fail_reason'] = reason            fail_dict['Branch'] = content[i]            fail_dict_list.append(fail_dict)        else:            unfinished.append(content[i])    for i in equal_id:        if old[i] == 'Idle' and new[i] == 'Idle':            pass        else:            unfinished.append(content[i])    return unfinished, fail_dict_listdef get_unfinished_success_fail(idx, status_list, arr_list, record_unfinished):    success_branch = []    for i in xrange(idx):        if check_status(status_list[i]):            success.append(arr_list[i])            success_branch.append(arr_list[i].split()[0])        # else:        #     send_notification('Unknown error occurred!')   # send notification-------------->>>>>    for m in range(idx, len(status_list)):        project_name = arr_list[m].split()[0]        if not status_list[m] == 'Idle':            unfinished.append(project_name)    for un_job in record_unfinished:        if un_job not in unfinished and un_job not in success_branch:            fail.append(un_job)   # send notification-------------->>>>>            reason = 'This project: %s build failed when some other branch build success' % un_job            fail_dict['fail_reason'] = reason            fail_dict['Branch'] = un_job            fail_dict_list.append(fail_dict)    return unfinished, success, fail_dict_listdef init_timer_list(n):    timer = []    for i in xrange(n):        # timer.append(0)        timer.append(100)     # testing    return timerdef _send_hipchat_msg(msg, good=True):    url = '%s/v2/room/%s/notification?auth_token=%s' % (HIPCHAT_SERVER, HIPCHAT_ROOM, HIPCHAT_TOKEN)    headers = {
   'content-type': 'application/json'}    if good:        data = {
   'message': msg, 'from': 'West', 'color': 'green', 'style': 'application'}    else:        data = {
   'message': msg, 'from': 'West', 'color': 'red', 'style': 'application'}        print 'Something went wrong, send message to hipchat, message: %s' % msg    resp = requests.post(url, data=json.dumps(data), verify=False, headers=headers)    if resp.status_code == 204:        print 'Send notification succeed'    else:        print 'Send notification failed caused by %s', resp.textdef send_success_notification():    for item in success_dict_list:        branch = item['branch']        build = item['build']        if 'error_info' in item.keys():            error_info = item['error_info']            msg = 'Branch          : %s\n\r' \                  'Build           : %s\n\r' \                  'Status          : Build Success\n\r' \                  'Error_info      : %s' % (branch, build, error_info)        else:            start_time = item['start_time']            finish_time = item['finish_time']            msg = 'Branch          : %s\n\r' \                  'Build           : %s\n\r' \                  'Status          : Build Success\n\r' \                  'Start Time      : %s\n\r' \                  'Finish Time     : %s' \                  % (branch, build, start_time, finish_time)        if branch in white_list:            msg = msg + '\n\rWhite List      : Yes' \                        '\n\rSend Job Request: Yes'        else:            msg = msg + '\n\rWhite List      : No' \                        '\n\rSend Job Request: No'        _send_hipchat_msg(msg)        print 'hipchat notification send completed!'        _send_msteams_msg(msg)        print 'msteams notification send completed!'def send_fail_notification():    for item in fail_dict_list:        branch = item['Branch']        reason = fail_dict['fail_reason']        msg = 'Branch          : %s\n\r' \              'Status          : Build Failed\n\r' \              'Fail Reason     : %s' % (branch, reason)        if branch in white_list:            msg = msg + '\n\rWhite List      : Yes' \                        '\n\rSend Job Request: Yes'        else:            msg = msg + '\n\rWhite List      : No' \                        '\n\rSend Job Request: Yes'        _send_hipchat_msg(msg, good=False)        print 'hipchat_notification send completed!'        _send_msteams_msg(msg)        print 'msteams_notification send completed!'def send_messages(msg):    _send_hipchat_msg(msg)    _send_msteams_msg(msg)def _send_msteams_msg(msg, good=True):    myteamsmessage = pymsteams.connectorcard(        'https://outlook.office.com/webhook/'        '0f7a6329-9345-4c6e-a313-dc7a48cfdd6b@2563c132-88f5-466f-bbb2-e83153b3c808/'        'IncomingWebhook/68ac4a1564224e80ad1b3e4d32a88e22/176e0d36-e270-40bb-8b1e-4bfba2c98d82')    title = 'Check Build Report'    if good:        pass    else:        myteamsmessage.color('red')    myteamsmessage.title(title)    myteamsmessage.text(msg)    myteamsmessage.send()def get_white_list():    with open('whitelist_for_buildcheck', 'r') as f:        white_jobs = f.readlines()    new_white_jobs = []    for it in white_jobs:        new_it = it.split('\n')[0]        new_white_jobs.append(new_it)    return new_white_jobsdef get_timeout_fail():    # init a timer list and get job_timer_dict    timer_list = init_timer_list(len(record_unfinished))    print 'fail_dict_list-INININ-: ', fail_dict_list    for job_id in xrange(len(record_unfinished)):        job = record_unfinished[job_id]        if job in unfinished:            timer_list[job_id] = 1            job_timer_dict[job] = timer_list[job_id]    print 'init job_timer_dict: ', job_timer_dict    print 'old record_timer_dict: ', record_timer_dict    for key in record_timer_dict.keys():        if key in job_timer_dict.keys():            job_timer_dict[key] = record_timer_dict[key] + job_timer_dict[key]    print 'after merge from record_timer_dict, job_timer_dict: ', job_timer_dict    for key in job_timer_dict.keys():        job_name = key        # job_name = job_timer_dict.keys()[job_timer_dict.values().index(job_timer_dict[key])]        if job_timer_dict[key] > 720:  # 720*20sec/3600=4 hours            print "Task: \'%s\' did not finished in 4 hours, " \                  "send it in failed list" % job_name            reason = "\'%s\' did not finished in 4 hours, " \                     "send it in failed list" % job_name   # send notification-------------->>>>>            fail.append(job_name)            print 'fail-----', fail            print 'fail_dict_list-ououout-: ', fail_dict_list            fail_dict['fail_reason'] = reason            fail_dict['Branch'] = job_name            fail_dict_list.append(fail_dict)            print 'fail_dict_list--: ', fail_dict_list        else:            print "Task: \'%s\' is unfinished, spent time: %s seconds" % \                  (job_name, job_timer_dict[key] * 20)def get_cm_start_finish_time(branch, build):    url_list, page_url, image_url = [], [], []    start_t, finish_t, msg = '', '', ''    project_show_url = 'https://builds.wgti.net/project/show/%s' % branch    body = requests.get(project_show_url, verify=False).text    soup = BeautifulSoup(body, 'lxml')    soup.prettify()    for anchor in soup.findAll('a', href=True):        url_list.append(anchor.get('href'))    for url in url_list:        if re.match('http://cm02se.wgti.net/.*', url):            page_url.append(url)        if re.match(r'ftp://cmimages.wgti.net/images/%s/.*' % branch, url):            image_url.append(url)    for idx in xrange(len(image_url)):        if image_url[idx].split('/')[-1] == build:            wanna_page = page_url[idx]            body = requests.get(wanna_page, verify=False).text            print body            started = re.findall('Build started : \d+-\d+-\d+ \d+:\d+.*', body)            finished = re.findall('Build finished: \d+-\d+-\d+ \d+:\d+.*', body)            if not started or not finished:                started = re.findall('start time.*\d+:\d+:\d+.*', body)                finished = re.findall('end time.*\d+:\d+:\d+.*', body)            print 'started, finished: ', started, finished            start_t = re.findall('\d+.*', started[0])[0]            finish_t = re.findall('\d+.*', finished[0])[0]            break    if not start_t and not finish_t:        msg = 'the image: %s do not exist in image list of branch: %s, ' \              'please check on web: %s' % (build, branch, project_show_url)    return start_t, finish_t, msgif __name__ == '__main__':    # hipchat room information    HIPCHAT_SERVER = 'https://watchguard.hipchat.com'    HIPCHAT_TOKEN = 'psw9IfRRhVzaHmaJ80JUpQDoJN9h6WtgtSSO8YWx'    HIPCHAT_ROOM = '3280364'    # parameters initial    unfinished, success, fail = [], [], []  # store result: unfinished, success, fail    record_unfinished = []  # record unfinished    record_timer_dict = {}  # record total timer of unfinished    job_timer_dict = {}  # store current timer of unfinished    success_dict, fail_dict = {}, {}  # for send_notification    success_dict_list, fail_dict_list = [], []  # for send_notification    # get white list    white_list = get_white_list()    # get all old rows    old_rows = _get_rows()    #--------local debug------------    testfile = r'D:\Project_TCS\git_projects\TCS\farmacias\test_script\dashboard\testfile.txt'    with open(testfile, 'wb') as f:        for row in old_rows:            f.write(row + '\n')    #exit(0)    #--------local debug------------    # get old latest date    date_list = _get_date(old_rows)    old_latest_build_date = date_list[0]    print 'old_latest_build_date: ', old_latest_build_date    # get old latest row with no date    row_list = _get_arr_no_date(old_rows)    old_row1 = row_list[0]    print 'old_row1: ', old_row1    # get old status list    old_status_list = _get_status(old_rows)    while True:        # check interval        time.sleep(20)        # request website and get new rows of table        #new_rows = _get_rows()        #--------local debug------------        a = []        with open(testfile, 'r') as f:            for line in f.readlines():                #print line                a.append(line)        new_rows = []        for item in a:            new_rows.append(item.split('\n')[0])        print new_rows        # -------local debug-------------        # get new latest row with no date        arr_list = _get_arr_no_date(new_rows)        new_row1 = row_list[0]        print 'new_row1: ', new_row1        date_list = _get_date(new_rows)        status_list = _get_status(new_rows)        project_list = _get_project(new_rows)        # get new_latest_build_date        new_latest_build_date = date_list[0]        # key point        try:            idx = date_list.index(old_latest_build_date.strip())        except:            idx = None        print 'idx: ', idx        # judge idx        if idx is None:            if check_status(status_list[0]):                success.append(arr_list[0])            print 'success: ', success        elif idx == 0:            unfinished, fail = get_unfinished_and_failed(old_status_list, status_list, project_list)            print 'unfinished, fail: ', unfinished, fail        else:            unfinished, success, fail = get_unfinished_success_fail(idx, status_list, arr_list, record_unfinished)            print 'unfinished, success, fail: ', unfinished, success, fail        if success:            for suc_job in success:                bran = suc_job.split()[0]                buil = suc_job.split()[2]                success_dict['branch'] = bran                success_dict['build'] = buil                start_time, finish_time, error = get_cm_start_finish_time(bran, buil)                if start_time and finish_time:                    success_dict['start_time'] = start_time                    success_dict['finish_time'] = finish_time                else:                    success_dict['error_info'] = error                success_dict_list.append(success_dict)            print 'success_dict_list: ', success_dict_list            # send job request and notification -- success            #send_job_request(success)            send_success_notification()        # update old latest row and date        old_row1 = new_row1        old_latest_build_date = new_latest_build_date        # add new to record_unfinished and calculator timer        for job_un in unfinished:            if job_un not in record_unfinished:                record_unfinished.append(job_un)        print 'fail_dict_list: ', fail_dict_list        # calculate timer and put timeout job into fail list        get_timeout_fail()        print 'fail_dict_list: ', fail_dict_list        # send job request and notification -- fail        if fail:            #send_job_request(fail)            send_fail_notification()        # update job_timer_dict to record_timer_dict        # record_timer_dict = job_timer_dict        record_timer_dict = {
   'utm-inc': 720}        print 'record_timer_dict: job_timer_dict', record_timer_dict, job_timer_dict        print 'unfinished: ', unfinished        print 'success: ', success        print 'fail: ', fail        # update record_unfinished        record_unfinished = unfinished        # update old_status_list        old_status_list = status_list        # flush parameters        unfinished, success, fail = [], [], []  # store result: unfinished, success, fail        job_timer_dict = {}        success_dict, fail_dict = {}, {}  # for send_notification        success_dict_list, fail_dict_list = [], []  # for send_notification        print '-------------------------------------------'