import os import sys from log import Logger from collections import Counter from tqdm import tqdm import numpy as np import pandas as pd ROOT_DIR = os.path.join(sys.path[0], '../') LOG_DIR = os.path.join(ROOT_DIR, 'log') DATA_DIR = os.path.join(ROOT_DIR, 'data') TRAIN_DIR = os.path.join(DATA_DIR, 'preliminary_train') # 提交docker时 需要打开更换 MODEL_PATH = os.path.join(ROOT_DIR, './model/deberta-base') MODEL_1_PATH = os.path.join(ROOT_DIR, './model') TEST_A_DIR = os.path.join(ROOT_DIR, './tcdata') # TEST_A_DIR = os.path.join(ROOT_DIR, './tcdata_test') PSEUDO_FALG = True TEST_B_DIR = os.path.join(ROOT_DIR, 'tcdata') RESULT_DIR = os.path.join(ROOT_DIR, 'prediction_result') FEATURE_DIR = os.path.join(ROOT_DIR, 'feature') GENERATION_DIR = os.path.join(FEATURE_DIR, 'generation') CORRELATION_DIR = os.path.join(FEATURE_DIR, 'correlation') USER_DATA_DIR = os.path.join(ROOT_DIR, 'user_data') USER_MODEL_DIR = os.path.join(USER_DATA_DIR, 'model_data') TMP_DIR = os.path.join(USER_DATA_DIR, 'tmp_data') N_ROUNDS = 10000 TIME_INTERVAL = 60 KEY_1 = ['OEM record c2', 'Processor CPU_Core_Error', '001c4c', 'System Event Sys_Event', 'Power Supply PS0_Status', 'Temperature CPU0_Margin_Temp', 'Reading 51 > Threshold 85 degrees C', 'Lower Non-critical going low', 'Temperature CPU1_Margin_Temp', 'System ACPI Power State #0x7d', 'Lower Critical going low'] KEY_2 = ['OEM CPU0 MCERR', 'OEM CPU0 CATERR', 'Reading 0 < Threshold 2 degrees C', '0203c0a80101', 'Unknown CPU0 MCERR', 'Unknown CPU0 CATERR', 'Microcontroller #0x3b', 'System Boot Initiated', 'Processor #0xfa', 'Power Unit Pwr Unit Status', 'Hard reset', 'Power off/down', 'System Event #0xff', 'Memory CPU1A1_DIMM_Stat', '000000', 'Power cycle', 'OEM record c3', 'Memory CPU1C0_DIMM_Stat', 'Reading 0 < Threshold 1 degrees C', 'IERR'] KEY_3 = ['Memory', 'Correctable ECC logging limit reached', 'Memory MEM_CHE0_Status', 'Memory Memory_Status', 'Memory #0x87', 'Memory CPU0F0_DIMM_Stat', 'Memory Device Disabled', 'Memory #0xe2', 'OS Stop/Shutdown OS Status', 'System Boot Initiated System Restart', 'OS Boot BIOS_Boot_Up', 'System Boot Initiated BIOS_Boot_UP', 'Memory DIMM101', 'OS graceful shutdown', 'OS Critical Stop OS Status', 'Memory #0xf9', 'Memory CPU0C0_DIMM_Stat', 'Memory DIMM111', 'Memory DIMM021', ] KEY_4 = ['Drive Fault', 'NMI/Diag Interrupt', 'Failure detected', 'Power Supply AC lost', 'Power Supply PSU0_Supply', 'AC out-of-range, but present', 'Predictive failure', 'Drive Present', 'Temperature Temp_DIMM_KLM', 'Temperature Temp_DIMM_DEF', 'Power Supply PS1_Status', 'Identify Status', 'Power Supply PS2_Status', 'Temperature DIMMG1_Temp', 'Upper Non-critical going high', 'Temperature DIMMG0_Temp', 'Upper Critical going high', 'Power Button pressed', 'System Boot Initiated #0xb8', 'Deasserted'] TOP_KEY_WORDS = ['0203c0a80101', 'Configuration Error', 'Correctable ECC', 'Deasserted', 'Device Enabled', 'Drive Present', 'Event Logging Disabled SEL', 'Failure detected', 'IERR', 'Initiated by hard reset', 'Initiated by power up', 'Initiated by warm reset', 'Log area reset/cleared', 'Memory', 'Memory #0xe2', 'Memory CPU0C0', 'Microcontroller/Coprocessor BMC', 'OEM CPU0 CATERR', 'OEM CPU0 MCERR', 'OS Boot BIOS', 'OS Critical Stop OS Status', 'Power Supply PS1', 'Power Supply PS2', 'Presence detected', 'Processor', 'Processor CPU', 'Processor CPU0', 'Processor CPU1', 'S0/G0: working', 'S4/S5: soft-off', 'Slot / Connector PCIE', 'State Asserted', 'State Deasserted', 'System ACPI Power State ACPI', 'System Boot Initiated', 'System Boot Initiated #0xe0', 'System Boot Initiated BIOS', 'System Event', 'System Event #0x10', 'System Event #0xff', 'Timestamp Clock Sync', 'Transition to Running', 'Uncorrectable ECC', 'Uncorrectable machine check exception', 'Unknown CPU0 CATERR', 'Unknown CPU0 MCERR', 'Unknown Chassis', 'Watchdog2 IPMI', ] TOP_KEY_WORDS_2 = ['Processor CPU0 Status', 'System Boot Initiated BIOS Boot Up', 'Uncorrectable ECC', 'Initiated by power up', 'Configuration Error', 'Processor CPU CATERR', 'Processor CPU1 Status', 'Memory #0xe2', 'IERR', 'Initiated by warm reset', 'State Asserted', 'S4/S5: soft-off', 'Memory #0xf9', 'S0/G0: working', 'boot completed - device not specified', 'Timestamp Clock Sync', 'Presence detected', 'System Boot Initiated #0xe0', 'Drive Fault', 'Power Supply PS1 Status', 'Power off/down', 'OS Boot #0xe9', 'Failure detected', 'Uncorrectable machine check exception', 'Transition to Running', 'Power Supply PS2 Status', 'Memory Device Disabled', 'System Restart', 'System Event #0x10', 'Sensor access degraded or unavailable', 'Unknown #0x17', 'Drive Present', 'Management Subsys Health System Health', 'Power Supply AC lost', 'Microcontroller #0x16'] CHARATERS = ['#', '&', ] # KEY_WORDS = KEY_1 + KEY_2 + KEY_3 + KEY_4 + CHARATERS KEY_WORDS = KEY_1 + KEY_2 + KEY_3 + KEY_4 + CHARATERS + TOP_KEY_WORDS KEY_WORDS = list(set(KEY_WORDS)) # cnt_1_0_diff_key_words = ['State Asserted','Processor CPU_CATERR','Unknown #0x17','Microcontroller #0x16','Transition to Running','State Deasserted','Processor #0xfa','Temperature CPU1_Margin_Temp','Temperature CPU0_Margin_Temp','Power cycle','Management Subsys Health System_Health','Sensor access degraded or unavailable','Power off/down','System ACPI Power State #0x7d'] # key_words_0 = ['Temperature CPU0_Margin_Temp','Lower Critical going low','System ACPI Power State #0x7d','Temperature CPU1_Margin_Temp','Lower Non-critical going low','Uncorrectable machine check exception','Reading 0 < Threshold 1 degrees C','000000','Unknown #0x19','Temperature DIMMG1_Temp','Reading 0 < Threshold 0 degrees C','001c4c','IERR','Upper Critical going high','Unknown Chassis_control','Temperature DIMMG0_Temp','Upper Non-critical going high','Temperature Temp_DIMM_DEF','Power cycle','Processor CPU0_Status','Temperature Temp_DIMM_KLM','Processor CPU1_Status','Management Subsys Health System_Health'] # key_words_1 = ['Processor #0xfa','State Deasserted','Power off/down','Power cycle','IERR','Unknown #0x17','Management Subsys Health System_Health','Processor CPU_CATERR','Reading 0 < Threshold 1 degrees C','','Sensor access degraded or unavailable','Transition to Running','State Asserted','Microcontroller #0x16','Processor CPU0_Status','Processor CPU1_Status','Slot / Connector PCIE_Status','Fault Status','System ACPI Power State ACPI_PWR_Status','Management Subsystem Health System_Health','Configuration Error','Uncorrectable machine check exception','Timestamp Clock Sync'] # key_words_2 = ['Memory #0xe2','Memory Device Disabled','Memory #0x87','Memory #0xf9','Correctable ECC','Memory CPU0D0_DIMM_Stat','Uncorrectable ECC','Memory CPU1B0_DIMM_Stat','System Boot Initiated BIOS_Boot_UP','System Restart','Presence Detected','Temperature CPU0_Temp','boot completed - device not specified','Log almost full','Device Present','Legacy OFF state','System Boot Initiated #0xe0','System Event #0x10','Legacy ON state','OS Boot #0xe0','Unknown #0xc5','System Boot Initiated #0xb8','Event Logging Disabled SEL_Status'] # key_words_3 = ['Drive Fault','Failure detected','Drive Present','Temperature Temp_DIMM_KLM','Temperature Temp_DIMM_DEF','Power Supply PS4_Status','Upper Non-critical going high','Temperature DIMMG0_Temp','Temperature DIMMG1_Temp','Power Supply PS3_Status','Upper Critical going high','Predictive failure','Power Supply AC lost','Unknown #0x19','Power Unit Power Unit','AC out-of-range, but present','Power Supply PS1_Status','Power Supply PS2_Status','Log area reset/cleared','Microcontroller/Coprocessor BMC_Boot_Up','System Boot Initiated #0xb8','Power Button pressed','Device Present'] # top_key_words = [ 'Configuration Error','Uncorrectable ECC','Processor CPU0_Status','Initiated by power up','','Presence Detected','Processor CPU1_Status','S0/G0: working','Processor CPU_CATERR','Presence detected','S4/S5: soft-off','Upper Critical going high','Memory #0xe2','IERR','Initiated by warm reset','State Asserted','Upper Non-critical going high','boot completed - device not specified','Memory Device Disabled','Timestamp Clock Sync','Lower Critical going low','Transition to Running','Memory #0xf9','Power Supply PS1_Status'] # key_words_1_desc = ['#0xfa', '#0x','#0xff','CATERR','cycle','Unit','IERR','IPMI','#0x17', 'Running','#0x7c','Unknown','CPU', 'Sensor','CPU0','CPU1','Subsys'] # # key_words = cnt_1_0_diff_key_words +key_words_0+key_words_1+key_words_2+key_words_3+top_key_words+key_words_1_desc # key_words = list(set(key_words)) # KEY_WORDS = key_words+CHARATERS def create_dir(dir): """ 创建目录 :param dir: 目录名 :return: """ if not os.path.exists(dir): os.mkdir(dir) print(f'{dir}目录不存在,创建{dir}目录成功.') else: print(f'{dir}目录已存在.') def create_all_dir(): """ 创建所有需要的目录 :return: """ create_dir(ROOT_DIR) create_dir(LOG_DIR) # create_dir(MODEL_DIR) create_dir(RESULT_DIR) create_dir(FEATURE_DIR) create_dir(GENERATION_DIR) create_dir(CORRELATION_DIR) create_dir(DATA_DIR) create_dir(TRAIN_DIR) create_dir(TEST_A_DIR) # create_dir(TEST_B_DIR) create_dir(USER_DATA_DIR) create_dir(USER_MODEL_DIR) create_dir(TMP_DIR) def clean_str(string): return string def my_tokenizer(s): return s.split(' | ') def get_word_counter(data): print('获取异常日志计数字典') counter = Counter() for string_ in tqdm(data['msg']): string_ = string_.strip() counter.update(my_tokenizer(clean_str(string_))) return counter def macro_f1(target_df: pd.DataFrame, submit_df: pd.DataFrame): """ 计算得分 :param target_df: [sn,fault_time,label] :param submit_df: [sn,fault_time,label] :return: """ weights = [5 / 11, 4 / 11, 1 / 11, 1 / 11] # weights = [3 / 7, 2 / 7, 1 / 7, 1 / 7] overall_df = target_df.merge( submit_df, how='left', on=[ 'sn', 'fault_time'], suffixes=[ '_gt', '_pr']) overall_df.fillna(-1) macro_F1 = 0. for i in range(len(weights)): TP = len(overall_df[(overall_df['label_gt'] == i) & (overall_df['label_pr'] == i)]) FP = len(overall_df[(overall_df['label_gt'] != i) & (overall_df['label_pr'] == i)]) FN = len(overall_df[(overall_df['label_gt'] == i) & (overall_df['label_pr'] != i)]) precision = TP / (TP + FP) if (TP + FP) > 0 else 0 recall = TP / (TP + FN) if (TP + FN) > 0 else 0 F1 = 2 * precision * recall / \ (precision + recall) if (precision + recall) > 0 else 0 macro_F1 += weights[i] * F1 return macro_F1 def search_weight(train, valid_y, raw_prob, init_weight=[ 1.0], class_num=4, step=0.001): weight = init_weight.copy() * class_num oof = train[['sn', 'fault_time']] oof['label'] = raw_prob.argmax(axis=1) f_best = macro_f1(train[['sn', 'fault_time', 'label']], oof) print("Inint Score:", f_best) # f_best = f1_score(y_true=valid_y, y_pred=raw_prob.argmax(axis=1),average='macro') flag_score = 0 round_num = 1 while (flag_score != f_best): print("round: ", round_num) round_num += 1 flag_score = f_best for c in range(class_num): for n_w in range(0, 2000, 10): num = n_w * step new_weight = weight.copy() new_weight[c] = num prob_df = raw_prob.copy() prob_df = prob_df * np.array(new_weight) oof['label'] = prob_df.argmax(axis=1) f = macro_f1(train[['sn', 'fault_time', 'label']], oof) # f = f1_score(y_true=valid_y, y_pred=prob_df.argmax(axis=1),average='macro') if f > f_best: weight = new_weight.copy() f_best = f print(f"class:{c}, new_weight:{num}, f1 score: {f}") print( f'********************** SEARCH BEST WEIGHT : {weight} **********************') return weight def get_new_cols(df, key=['sn', 'fault_time']): if isinstance(df.columns[0], tuple): new_cols = [] for i in df.columns: if i[0] in key: new_cols.append(i[0]) else: new_cols.append(f'{i[0]}_{i[1]}') df.columns = new_cols return df else: print('当前的DataFrame没有二级列名，请检查。') return df if __name__ == '__main__': # create_all_dir() logger = Logger(name=os.path.basename(__file__).split( '.py')[0], log_path=LOG_DIR, mode="w").get_log print(len(KEY_WORDS))