import datetime import os import pickle from collections import Counter from utils import get_new_cols import numpy as np import pandas as pd from tqdm import tqdm from gensim.models import Word2Vec from utils import GENERATION_DIR from utils import KEY_1, KEY_2, KEY_3, KEY_4 from tqdm import tqdm from scipy import stats def cat2num(df, cat_cols, Transfer2num=True): ''' :param df: :param cat_cols: 类别特征列表 :param Transfer2num: 类别特征转换为数值特征 :return: ''' if Transfer2num: print('Transfer category feature to num feature ') for col in cat_cols: if not os.path.exists(os.path.join(GENERATION_DIR, f'{col}_map.pkl')): print(f'Transfer : {col}') tmp_map = dict(zip(df[col].unique(), range(df[col].nunique()))) with open(os.path.join(GENERATION_DIR, f'{col}_map.pkl'), 'wb') as f: pickle.dump(tmp_map, f) else: with open(os.path.join(GENERATION_DIR, f'{col}_map.pkl'), 'rb') as f: tmp_map = pickle.load(f) df[f'{col}_LabelEnc'] = df[col].map(tmp_map).fillna(-1).astype(int) else: print('Transfer category feature to category feature ') for col in cat_cols: df[col] = df[col].astype('category') print('Transfer category feature to num feature Down...') return df def add_minutes(x, minutes=5): dt = datetime.datetime.strptime(x, '%Y-%m-%d %H:%M:%S') out_date = (dt + datetime.timedelta(minutes=minutes) ).strftime('%Y-%m-%d %H:%M:%S') return out_date def time_process(df, time_cols, minutes_): df[f'time_{minutes_}'] = df[time_cols].apply( lambda x: add_minutes(str(x), minutes_)) return df def get_fea(x, fea): if fea in x: return 1 else: return 0 def get_last_msg_cnt(x): last_msg = x[-1] cnt = x.count(last_msg) return cnt def get_first_msg_cnt(x): first_msg = x[0] cnt = x.count(first_msg) return cnt def add_last_next_time4fault(label, preliminary_submit_dataset_a, time_interval, next_time_list): print(f'添加自定义异常出现的时间间隔{time_interval}的前后的时间点') for i in tqdm([-i for i in next_time_list] + next_time_list): label = time_process(label, 'fault_time', i * time_interval) preliminary_submit_dataset_a = time_process( preliminary_submit_dataset_a, 'fault_time', i * time_interval) return label, preliminary_submit_dataset_a def get_msg_text_fea(df, msg_type='last'): print(f'获取 msg text {msg_type}特征') df_fea = df.groupby(['sn', 'fault_time']).agg( {'msg_list': 'sum', 'msg_0': 'sum', 'msg_1': 'sum', 'msg_2': 'sum'}).reset_index() df_fea['msg_list_unique'] = df_fea['msg_list'].apply(lambda x: str(set(x))) df_fea['msg_0_unique'] = df_fea['msg_0'].apply(lambda x: str(set(x))) df_fea['msg_1_unique'] = df_fea['msg_1'].apply(lambda x: str(set(x))) df_fea['msg_2_unique'] = df_fea['msg_2'].apply(lambda x: str(set(x))) df_fea['msg_list_list'] = df_fea['msg_list'].apply(lambda x: str(x)) df_fea['msg_0_list'] = df_fea['msg_0'].apply(lambda x: str(x)) df_fea['msg_1_list'] = df_fea['msg_1'].apply(lambda x: str(x)) df_fea['msg_2_list'] = df_fea['msg_2'].apply(lambda x: str(x)) df_fea['msg_0_first'] = df_fea['msg_0'].apply(lambda x: x[0]) df_fea['msg_1_first'] = df_fea['msg_1'].apply(lambda x: x[0]) df_fea['msg_2_first'] = df_fea['msg_2'].apply(lambda x: x[0]) df_fea['msg_0_last'] = df_fea['msg_0'].apply(lambda x: x[-1]) df_fea['msg_1_last'] = df_fea['msg_1'].apply(lambda x: x[-1]) df_fea['msg_2_last'] = df_fea['msg_2'].apply(lambda x: x[-1]) df_fea['msg_last'] = df.groupby(['sn', 'fault_time']).apply( lambda x: x['msg'].to_list()[-1]).values df_fea['msg_first'] = df.groupby(['sn', 'fault_time']).apply( lambda x: x['msg'].to_list()[0]).values df_fea['last_msg_cnt'] = df_fea['msg_list'].apply( lambda x: get_last_msg_cnt(x)) df_fea['first_msg_cnt'] = df_fea['msg_list'].apply( lambda x: get_first_msg_cnt(x)) cat_cols = ['msg_list', 'msg_0', 'msg_1', 'msg_2', 'msg_list_unique', 'msg_0_unique', 'msg_1_unique', 'msg_2_unique', 'msg_list_list', 'msg_0_list', 'msg_1_list', 'msg_2_list', 'msg_0_first', 'msg_1_first', 'msg_2_first', 'msg_0_last', 'msg_1_last', 'msg_2_last', 'msg_last', 'msg_first'] num_cols = ['last_msg_cnt', 'first_msg_cnt'] id_cols = ['sn', 'fault_time'] df_fea = df_fea.rename( columns={ i: f'{msg_type}_{i}' for i in (cat_cols + num_cols)}) cat_cols = [f'{msg_type}_{i}' for i in cat_cols] for cat_col in cat_cols: df_fea[cat_col] = df_fea[cat_col].astype(str) df_fea = cat2num(df_fea, cat_cols, Transfer2num=True) for i in cat_cols: del df_fea[i] return df_fea def add_w2v_feats(all_data,w2v_feats_df,f1,f2,emb_size = 32,window = 5,min_count =5,): print(f'生成 {f1}_{f2}_w2v 特征') df_fea = all_data.groupby(f1).agg({f2:'sum'}).reset_index() df_emb = df_fea[[f1 ]] sencences = df_fea[f2].to_list() if not os.path.exists(os.path.join(GENERATION_DIR, f'{f1}_{f2}_w2v_model.pkl')): print(f'{f1}_{f2}_w2v_model 不存在，开始训练......') model = Word2Vec(sencences, vector_size=emb_size, window=window, min_count=min_count, sg=0, hs=1, seed=42) with open(os.path.join(GENERATION_DIR, f'{f1}_{f2}_w2v_model.pkl'), 'wb') as f: pickle.dump(model, f) else: print(f'{f1}_{f2}_w2v_model 已存在，开始读取......') with open(os.path.join(GENERATION_DIR, f'{f1}_{f2}_w2v_model.pkl'), 'rb') as f: model = pickle.load(f) emb_matrix_mean = [] for sent in sencences: vec = [] for w in sent: if w in model.wv: vec.append(model.wv[w]) if len(vec) >0: emb_matrix_mean.append(np.mean(vec,axis = 0)) else: emb_matrix_mean.append([0]*emb_size) df_emb_mean = pd.DataFrame(emb_matrix_mean).add_prefix(f'{f1}_{f2}_w2v_') df_emb = pd.concat([df_emb,df_emb_mean],axis = 1) w2v_feats_df = w2v_feats_df.merge(df_emb,on = f1,how ='left') return w2v_feats_df def get_w2v_feats(all_data,f1_list,f2_list): all_data['msg_list'] = all_data['msg'].apply(lambda x: [i.strip() for i in x.split(' | ')]) all_data['msg_0'] = all_data['msg'].apply(lambda x: [get_msg_location(x.split(' | '), 0)]) all_data['msg_1'] = all_data['msg'].apply(lambda x: [get_msg_location(x.split(' | '), 1)]) all_data['msg_2'] = all_data['msg'].apply(lambda x: [get_msg_location(x.split(' | '), 2)]) w2v_feats_df = all_data[f1_list].drop_duplicates() for f1 in f1_list: for f2 in f2_list: w2v_feats_df = add_w2v_feats(all_data,w2v_feats_df,f1,f2,emb_size = 10,window = 5,min_count =5,) print(f'w2v_feats 的特征维度: {w2v_feats_df.shape}') return w2v_feats_df def get_time_diff_feats_v2(all_data): print('生成时间差特征 time_diff_feats_v2') all_data['duration_seconds'] = all_data['time_interval'] all_data['duration_minutes'] = all_data['time_interval'] / 60 df_merge_log = all_data[['sn', 'fault_time', 'label', 'time', 'msg', 'server_model', 'time_interval', 'duration_seconds', 'duration_minutes']] df_merge_log['fault_id'] = df_merge_log['sn'] + '_' + df_merge_log['fault_time'] + '_' + df_merge_log[ 'server_model'] f1_list = ['fault_id', 'sn', 'server_model'] f2_list = ['duration_minutes', 'duration_seconds'] time_diff_feats_v2 = df_merge_log[['sn', 'fault_time', 'fault_id', 'server_model']].drop_duplicates().reset_index( drop=True) for f1 in f1_list: for f2 in f2_list: func_opt = ['count', 'nunique', 'min', 'max', 'median', 'sum'] for opt in func_opt: tmp = df_merge_log.groupby([f1])[f2].agg([(f'{f2}_in_{f1}_' + opt, opt)]).reset_index() # print(f'{f1}_in_{f2}_{opt}:{tmp.shape}' ) time_diff_feats_v2 = time_diff_feats_v2.merge(tmp, on=f1, how='left') temp = df_merge_log.groupby([f1])[f2].apply(lambda x: stats.mode(x)[0][0]) time_diff_feats_v2[f'{f2}_in_{f1}_mode'] = time_diff_feats_v2[f1].map(temp).fillna(np.nan) secs = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9] for sec in secs: temp = df_merge_log.groupby([f1])[f2].quantile(sec).reset_index( name=f'log_{f2}_in_{f1}_quantile_' + str(sec * 100)) # print(f'log_{f1}_in_{f2}_quantile_{str(sec * 100)}:{tmp.shape}' ) time_diff_feats_v2 = pd.merge(time_diff_feats_v2, temp, on=f1, how='left') del time_diff_feats_v2['fault_id'] return time_diff_feats_v2 def get_feature(data, time_list, log_fea, fea_num, key): print(f'当前特征维度{data.shape}') fea_df_list = [] fea_cnt_list = ['OEM record c2', 'Processor CPU_Core_Error', '001c4c', 'System Event Sys_Event','OEM CPU0 MCERR', 'OEM CPU0 CATERR', 'Reading 0 < Threshold 2 degrees C', '0203c0a80101', 'Unknown CPU0 MCERR', 'Unknown CPU0 CATERR','Memory', 'Correctable ECC logging limit reached', 'Memory MEM_CHE0_Status', 'Memory Memory_Status', 'Memory #0x87', 'Memory CPU0F0_DIMM_Stat', 'Drive Fault', 'NMI/Diag Interrupt', 'Failure detected', 'Power Supply AC lost', ] for time_tmp in tqdm(time_list): print(f'获取异常前后 {time_tmp} min的数据进行聚合') tmp1 = data[(pd.to_datetime(data['time']) < pd.to_datetime(data[f'time_{time_tmp}'])) & (pd.to_datetime(data['time']) > pd.to_datetime(data[f'time_-{time_tmp}']))].sort_values( ['sn', 'fault_time']) tmp1 = tmp1.groupby(key).apply( lambda x: ' | '.join(x['msg'].to_list())).reset_index().rename(columns={0: 'msg'}) tmp1[f'msg_len'] = tmp1['msg'].apply(lambda x: len(x.split(' | '))) # tmp1[f'msg_len_two'] = tmp1['msg'].apply(lambda x: len(x)) # 添加数字个数 # tmp1[f'msg_num_two'] = tmp1['msg'].apply( # lambda x: len([int(s) for s in re.findall(r'\b\d+\b', x)])) print(f'根据异常前后 {time_tmp} min的数据的日志数据提取 {fea_num} 个稀疏特征') feature = log_fea + ['msg_len'] for fea in feature: tmp1[fea] = tmp1['msg'].apply(lambda x: get_fea(x, fea)) # 添加计数特征 if fea in fea_cnt_list: tmp1[f'{fea}_cnt'] = tmp1['msg'].apply(lambda x:x.replace('|',' ').replace('_',' ').split(' ').count(fea)) feature.append(f'{fea}_cnt') tmp1_new_col_map = {i: i + '_' + str(int(time_tmp)) for i in feature} tmp1 = tmp1.rename(columns=tmp1_new_col_map) del tmp1['msg'] fea_df_list.append(tmp1) fea_df = fea_df_list[-1] print(fea_df.shape) for i in fea_df_list[:-1]: fea_df = fea_df.merge(i, on=key, how='left') print(fea_df.shape) return fea_df def get_msg_location(x, num): try: return x[num] except BaseException: return '其它' def get_nearest_msg_fea(train, test): print('生成 nearest_msg 特征') df = pd.concat([train, test], axis=0, ignore_index=True) df['duration_minutes'] = (pd.to_datetime(df['fault_time']) - pd.to_datetime(df['time'])).apply( lambda x: x.total_seconds()) df = df.sort_values( ['sn', 'server_model', 'fault_time', 'time']).reset_index(drop=True) df['duration_minutes_abs'] = np.abs(df['duration_minutes']) df['duration_minutes_abs_rank'] = df.groupby(['sn', 'server_model', 'fault_time'])['duration_minutes_abs'].rank( method='first', ascending=False) key = ['sn', 'server_model', 'fault_time', 'duration_minutes_abs'] df = df.sort_values(key, ascending=False) df = df.drop_duplicates( ['sn', 'server_model', 'fault_time', ], keep='first') df.loc[df['duration_minutes'] == df['duration_minutes_abs'], 'last_or_next'] = 1 df.loc[df['duration_minutes'] != df['duration_minutes_abs'], 'last_or_next'] = 0 df['msg_cnt'] = df['msg'].map(df['msg'].value_counts()) df['msg_0'] = df['msg'].apply( lambda x: get_msg_location( x.split(' | '), 0)) df['msg_0_cnt'] = df['msg_0'].map(df['msg_0'].value_counts()) df['msg_1'] = df['msg'].apply( lambda x: get_msg_location( x.split(' | '), 1)) df['msg_1_cnt'] = df['msg_1'].map(df['msg_1'].value_counts()) df['msg_2'] = df['msg'].apply( lambda x: get_msg_location( x.split(' | '), 2)) df['msg_2_cnt'] = df['msg_2'].map(df['msg_2'].value_counts()) cat_feats = ['msg', 'msg_0', 'msg_1', 'msg_2'] # ,'server_model_day_date','server_model_dayofmonth','server_model_dayofweek','server_model_hour'] # for name in cat_feats: # le = LabelEncoder() # df[f'{name}_LabelEnc'] = le.fit_transform(df[name]) df = cat2num(df,cat_feats) df = df.drop_duplicates().reset_index(drop=True) df = df[['sn', 'server_model', 'fault_time', 'msg_cnt', 'msg_0_cnt', 'msg_1_cnt', 'msg_2_cnt', # 'duration_minutes_abs','duration_minutes', 'duration_minutes_abs_rank', 'last_or_next', 'msg_LabelEnc', 'msg_0_LabelEnc', 'msg_1_LabelEnc', 'msg_2_LabelEnc']] print(f'生成 nearest_msg 特征完毕，特征维度{df.shape}') return df def get_server_model_time_interval_stat_fea(all_data): server_model_time_interval_stat_fea = all_data.groupby('server_model').agg({'time_interval':['min','max','mean','median']}).reset_index() server_model_time_interval_stat_fea = get_new_cols(server_model_time_interval_stat_fea,key = ['server_model' ]) server_model_time_interval_stat_fea.columns = ['server_model', 'sm_time_interval_min', 'sm_ttime_interval_max', 'sm_ttime_interval_mean', 'sm_ttime_interval_median'] return server_model_time_interval_stat_fea def get_server_model_sn_fea_2(train, test): df = pd.concat([train[['sn', 'server_model']], test[['sn', 'server_model']]], ignore_index=True) df['server_model_count_sn_2'] = df.groupby( ['server_model'])['sn'].transform('count') df['server_model_nunique_sn_2'] = df.groupby( ['server_model'])['sn'].transform('nunique') df['sn_cnt_2'] = df['sn'].map(df['sn'].value_counts()) return df.drop_duplicates().reset_index(drop=True) def get_4_time_stat_fea(df): print(' 生成时间统计特征') time_stat_fea_df = df.groupby(['sn', 'fault_time', 'server_model']).agg( {'duration_minutes': ['min', 'max', 'mean', 'median', 'skew', 'sum', 'std', 'count'], 'log_duration_minutes': ['min', 'max', 'mean', 'median', 'skew', 'sum', 'std'], 'time_diff_1': ['min', 'max', 'mean', 'median', 'skew', 'sum', 'std'], 'log_time_diff_1': ['min', 'max', 'median'], }).reset_index() new_time_stat_cols = [] for i in time_stat_fea_df.columns: if i[0] in ['sn', 'fault_time', 'server_model']: new_time_stat_cols.append(i[0]) else: new_time_stat_cols.append(f'{i[0]}_{i[1]}') # print(f'{i[0]}_{i[1]}') time_stat_fea_df.loc[time_stat_fea_df[i[0]] [i[1]] == -np.inf, (i[0], i[1])] = -20 time_stat_fea_df.loc[time_stat_fea_df[i[0]] [i[1]] == np.inf, (i[0], i[1])] = 30 time_stat_fea_df.columns = new_time_stat_cols time_stat_fea_df['duration_minutes_range'] = time_stat_fea_df['duration_minutes_max'] - time_stat_fea_df[ 'duration_minutes_min'] time_stat_fea_df['log_duration_minutes_range'] = time_stat_fea_df['log_duration_minutes_max'] - time_stat_fea_df[ 'log_duration_minutes_min'] time_stat_fea_df['time_diff_1_range'] = time_stat_fea_df['time_diff_1_max'] - \ time_stat_fea_df['time_diff_1_min'] time_stat_fea_df['log_time_diff_1_range'] = time_stat_fea_df['log_time_diff_1_max'] - time_stat_fea_df[ 'log_time_diff_1_min'] time_stat_fea_df['duration_minutes_freq'] = time_stat_fea_df['duration_minutes_range'] / time_stat_fea_df[ 'duration_minutes_count'] print(f' 生成时间统计特征完毕，特征维度:{time_stat_fea_df.shape}') return time_stat_fea_df def get_time_std_fea(train, test): print('生成 server_model 特征') df = pd.concat([train, test], axis=0, ignore_index=True) # df['year'] = df['time'].dt.year # df['month'] = df['time'].dt.month df['hour'] = df['time'].dt.hour # df['week'] = df['time'].dt.week df['minute'] = df['time'].dt.minute time_std = df.groupby(['sn', 'server_model']).agg( {'hour': 'std', 'minute': 'std'}).reset_index() time_std = time_std.rename( columns={ 'hour': 'hour_std', 'minute': 'minute_std'}) return time_std def get_key(all_data): all_data['msg_list'] = all_data['msg'].apply(lambda x: [i.strip() for i in x.split(' | ')]) class_fea_cnt_list = [] for label in [0,1,2,3]: class_df = all_data.query(f'label =={label}') counter = Counter() for i in class_df['msg_list']: counter.update(i) class_fea_cnt = pd.DataFrame({i[0]:i[1] for i in counter.most_common()},index = [f'fea_cnt_{label}']).T.reset_index().rename(columns = {'index':'fea'}) class_fea_cnt_list.append(class_fea_cnt) fea_cnt_df = class_fea_cnt_list[0] for tmp in class_fea_cnt_list[1:]: fea_cnt_df = fea_cnt_df.merge(tmp,on = 'fea') fea_cnt_df['fea_cnt_sum'] = fea_cnt_df.loc[:,['fea_cnt_0', 'fea_cnt_1', 'fea_cnt_2', 'fea_cnt_3']].sum(1) all_fea_cnt = fea_cnt_df['fea_cnt_sum'].sum() for i in ['fea_cnt_0', 'fea_cnt_1', 'fea_cnt_2', 'fea_cnt_3']: fea_cnt_df[f'{i}_ratio'] = fea_cnt_df[i]/fea_cnt_df['fea_cnt_sum'] fea_cnt_df[f'{i}_all_ratio'] = fea_cnt_df[i]/all_fea_cnt fea_cnt_df['fea_cnt_ratio_std'] = fea_cnt_df.loc[:,['fea_cnt_0_ratio','fea_cnt_1_ratio','fea_cnt_2_ratio','fea_cnt_3_ratio', ]].std(1) fea_cnt_df['fea_cnt_std'] = fea_cnt_df.loc[:,['fea_cnt_0', 'fea_cnt_1','fea_cnt_2','fea_cnt_3',]].std(1) fea_cnt_df['fea_cnt_all_ratio_std'] = fea_cnt_df.loc[:,['fea_cnt_0_all_ratio','fea_cnt_1_all_ratio', 'fea_cnt_2_all_ratio','fea_cnt_3_all_ratio',]].std(1) fea_cnt_df = fea_cnt_df[~fea_cnt_df['fea_cnt_ratio_std'].isnull()].sort_values('fea_cnt_ratio_std',ascending = False) fea_cnt_df['fea_max'] = np.argmax(fea_cnt_df.loc[:,['fea_cnt_0', 'fea_cnt_1', 'fea_cnt_2', 'fea_cnt_3',]].values,axis = 1) key_0 = fea_cnt_df.query('fea_max ==0 ')['fea'].to_list() key_1 = fea_cnt_df.query('fea_max ==1 ')['fea'].to_list() key_2 = fea_cnt_df.query('fea_max ==2 ')['fea'].to_list() key_3 = fea_cnt_df.query('fea_max ==3 ')['fea'].to_list() # key_1 = ['OEM record c2','Processor CPU_Core_Error','001c4c','System Event Sys_Event','Power Supply PS0_Status','Temperature CPU0_Margin_Temp','Reading 51 > Threshold 85 degrees C','Lower Non-critical going low','Temperature CPU1_Margin_Temp','System ACPI Power State #0x7d','Lower Critical going low'] # key_2 = ['OEM CPU0 MCERR','OEM CPU0 CATERR','Reading 0 < Threshold 2 degrees C','0203c0a80101','Unknown CPU0 MCERR','Unknown CPU0 CATERR','Microcontroller #0x3b','System Boot Initiated','Processor #0xfa','Power Unit Pwr Unit Status','Hard reset','Power off/down','System Event #0xff','Memory CPU1A1_DIMM_Stat','000000','Power cycle','OEM record c3','Memory CPU1C0_DIMM_Stat','Reading 0 < Threshold 1 degrees C','IERR'] # key_3 = ['Memory','Correctable ECC logging limit reached','Memory MEM_CHE0_Status','Memory Memory_Status','Memory #0x87','Memory CPU0F0_DIMM_Stat','Memory Device Disabled','Memory #0xe2','OS Stop/Shutdown OS Status','System Boot Initiated System Restart','OS Boot BIOS_Boot_Up','System Boot Initiated BIOS_Boot_UP','Memory DIMM101','OS graceful shutdown','OS Critical Stop OS Status','Memory #0xf9','Memory CPU0C0_DIMM_Stat','Memory DIMM111','Memory DIMM021',] # key_4 = ['Drive Fault','NMI/Diag Interrupt','Failure detected','Power Supply AC lost','Power Supply PSU0_Supply','AC out-of-range, but present','Predictive failure','Drive Present','Temperature Temp_DIMM_KLM','Temperature Temp_DIMM_DEF','Power Supply PS1_Status','Identify Status','Power Supply PS2_Status','Temperature DIMMG1_Temp','Upper Non-critical going high','Temperature DIMMG0_Temp','Upper Critical going high','Power Button pressed','System Boot Initiated #0xb8','Deasserted'] return key_0,key_1,key_2,key_3 def get_class_key_words_nunique(all_data): print('获取 class_key_words_nunique 特征') key_0,key_1,key_2,key_3 = get_key(all_data) df = all_data[['sn', 'fault_time', 'msg_list']] df_tmp = df.groupby(['sn' ]).agg({'msg_list':'sum'}).reset_index() df_tmp['class_0_key_words_nunique'] = df_tmp['msg_list'].apply(lambda x:len(set(x)&set(key_0))) df_tmp['class_1_key_words_nunique'] = df_tmp['msg_list'].apply(lambda x:len(set(x)&set(key_1))) df_tmp['class_2_key_words_nunique'] = df_tmp['msg_list'].apply(lambda x:len(set(x)&set(key_2))) df_tmp['class_3_key_words_nunique'] = df_tmp['msg_list'].apply(lambda x:len(set(x)&set(key_3))) del df_tmp['msg_list'] return df_tmp def get_key_for_top_fea(train,test): KEY_FOR_TOP_COLS = [] print('添加 key_for_top_fea 特征') for TIME in [3, 5, 10, 15, 30, 45, 60, 90, 120, 240, 360, 480, 540, 600,60000000]: for i in range(10): train[f'KEY_FOR_TOP_{i}_{TIME}'] = train[f'{KEY_1[i]}_{TIME}'].astype(str)+'_'+train[f'{KEY_2[i]}_{TIME}'].astype(str)+'_'+train[f'{KEY_3[i]}_{TIME}'].astype(str)+'_'+train[f'{KEY_4[i]}_{TIME}'].astype(str) test[f'KEY_FOR_TOP_{i}_{TIME}'] = test[f'{KEY_1[i]}_{TIME}'].astype(str)+'_'+test[f'{KEY_2[i]}_{TIME}'].astype(str)+'_'+test[f'{KEY_3[i]}_{TIME}'].astype(str)+'_'+test[f'{KEY_4[i]}_{TIME}'].astype(str) KEY_FOR_TOP_COLS.append(f'KEY_FOR_TOP_{i}_{TIME}') train = cat2num(train,KEY_FOR_TOP_COLS) test = cat2num(test,KEY_FOR_TOP_COLS) for KEY_FOR_TOP_COL in KEY_FOR_TOP_COLS: del train[KEY_FOR_TOP_COL] del test[KEY_FOR_TOP_COL] return train,test def get_key_word_cross_fea(train,test): print('获取关键词交叉特征......') KEY_WORDS_MAP = {'CPU0':KEY_1,'CPU1':KEY_2,'CPU2':KEY_3,'CPU3':KEY_4} KEY_WORDS_CROSS_COLS =[] for KEY_WORDS in KEY_WORDS_MAP: for i in [3, 5, 10, 15, 30, 45, 60, 90, 120, 240, 360, 480, 540, 600,60000000]: KEY_WORDS_COLS = [f'{col}_{i}' for col in KEY_WORDS_MAP[KEY_WORDS]] train[f'{KEY_WORDS}_WORDS_{i}'] = train[KEY_WORDS_COLS].astype(str).sum(1) test[f'{KEY_WORDS}_WORDS_{i}'] = test[KEY_WORDS_COLS].astype(str).sum(1) KEY_WORDS_CROSS_COLS.append(f'{KEY_WORDS}_WORDS_{i}') train = cat2num(train,KEY_WORDS_CROSS_COLS) test = cat2num(test,KEY_WORDS_CROSS_COLS) for COLS in KEY_WORDS_CROSS_COLS: del train[COLS] del test[COLS] print('获取关键词交叉特征完毕......') return train,test def get_time_quantile_fea(df): print(' 生成时间分位数特征') secs = [0.2, 0.4, 0.6, 0.8] time_fea_list = [] for sec in tqdm(secs): for time_fea_type in [ 'duration_minutes', 'log_duration_minutes', 'time_diff_1', 'log_time_diff_1']: temp = df.groupby(['sn', 'server_model', 'fault_time'])[time_fea_type].quantile(sec).reset_index( name=f'{time_fea_type}_' + str(sec * 100)) time_fea_list.append(temp) time_fea_df = time_fea_list[0] for time_fea in time_fea_list[1:]: time_fea_df = time_fea_df.merge( time_fea, how='left', on=[ 'sn', 'server_model', 'fault_time']) print(f' 生成时间分位数特征完毕，特征维度:{time_fea_df.shape}') return time_fea_df def get_server_model_fea(train, test): print('生成 server_model 特征') df = pd.concat([train, test], axis=0, ignore_index=True) df['server_model_count_sn'] = df.groupby( ['server_model'])['sn'].transform('count') df['server_model_nunique_sn'] = df.groupby( ['server_model'])['sn'].transform('nunique') # df['server_model_count'] = df.groupby('server_model')['server_model'].transform('count') # df['server_model_cnt_quantile'] = df['server_model'].map( # df['server_model'].value_counts().rank() / len(df['server_model'].unique())) # df['server_model_cnt_rank'] = df[f'server_model_cnt_quantile'].rank(method='min') df['sn_cnt'] = df['sn'].map(df['sn'].value_counts()) df['sn_freq'] = df['sn'].map(df['sn'].value_counts() / len(df)) df['server_model_cnt'] = df['server_model'].map( df['server_model'].value_counts()) df['server_model_freq'] = df['server_model'].map( df['server_model'].value_counts() / len(df)) select_cols = ['sn', 'server_model', 'server_model_count_sn', 'server_model_nunique_sn', 'sn_cnt', 'sn_freq', 'server_model_cnt', 'server_model_freq' # 'server_model_count','server_model_cnt_quantile', 'server_model_cnt_rank' ] server_model_fea = df[select_cols] cat_feats = [ 'server_model'] # ,'server_model_day_date','server_model_dayofmonth','server_model_dayofweek','server_model_hour'] # for name in cat_feats: # le = LabelEncoder() # server_model_fea[f'{name}_LabelEnc'] = le.fit_transform( # server_model_fea[name]) server_model_fea = cat2num(server_model_fea, cat_feats, Transfer2num=True) server_model_fea = server_model_fea.drop_duplicates().reset_index(drop=True) print(f'生成 server_model 特征完毕，特征维度:{server_model_fea.shape}') return server_model_fea def get_time_type_msg_unique_fea(df): df['msg_list'] = df['msg'].apply( lambda x: [i.strip() for i in x.split(' | ')]) df['msg_0'] = df['msg'].apply( lambda x: [ get_msg_location( x.split(' | '), 0)]) df['msg_1'] = df['msg'].apply( lambda x: [ get_msg_location( x.split(' | '), 1)]) df['msg_2'] = df['msg'].apply( lambda x: [ get_msg_location( x.split(' | '), 2)]) df = df.groupby(['sn', 'fault_time']).agg( {'msg_list': 'sum', 'msg_0': 'sum', 'msg_1': 'sum', 'msg_2': 'sum'}).reset_index() df['msg_set'] = df['msg_list'].apply(lambda x: '|'.join(list(set(x)))) df['msg_0_set'] = df['msg_0'].apply(lambda x: '|'.join(list(set(x)))) df['msg_1_set'] = df['msg_1'].apply(lambda x: '|'.join(list(set(x)))) df['msg_2_set'] = df['msg_2'].apply(lambda x: '|'.join(list(set(x)))) df = df[['sn', 'fault_time', 'msg_set', 'msg_0_set', 'msg_1_set', 'msg_2_set']] return df def get_msg_unique_fea(train, test, time_type='last'): print('生成msg_unique_ fea') common_cols = ['msg_set', 'msg_0_set', 'msg_1_set', 'msg_2_set'] df = pd.concat([train, test], axis=0, ignore_index=True) df['time_interval'] = ( pd.to_datetime( df['fault_time']) - df['time']).apply( lambda x: x.total_seconds()) last_fea = get_time_type_msg_unique_fea(df.query('time_interval >0')) last_fea = last_fea.rename(columns={i: f'last_{i}' for i in common_cols}) next_fea = get_time_type_msg_unique_fea(df.query('time_interval <0')) next_fea = next_fea.rename(columns={i: f'next_{i}' for i in common_cols}) all_fea = get_time_type_msg_unique_fea(df) all_fea = all_fea.rename(columns={i: f'all_{i}' for i in common_cols}) msg_unique_fea = all_fea.merge( last_fea, on=['sn', 'fault_time'], how='outer') msg_unique_fea = msg_unique_fea.merge( next_fea, on=['sn', 'fault_time'], how='outer') return msg_unique_fea def get_duration_minutes_fea(train, test): print('生成 duration_minutes 特征') df = pd.concat([train, test], axis=0, ignore_index=True) df['duration_minutes'] = (pd.to_datetime(df['fault_time']) - pd.to_datetime(df['time'])).apply( lambda x: x.total_seconds()) df['log_duration_minutes'] = np.log(df['duration_minutes']) df = df.sort_values(['sn', 'label', 'server_model', 'fault_time', 'time']).reset_index(drop=True) df['time_diff_1'] = (df.groupby(['sn', 'server_model', 'fault_time'])['time'].diff(1)).apply( lambda x: x.total_seconds()) df['time_diff_1'] = df['time_diff_1'].fillna(0) df['log_time_diff_1'] = np.log(df['time_diff_1']) # time_quantile_fea_df = get_time_quantile_fea(df) # time_stat_fea_df = get_4_time_stat_fea(df) # df_tmp = time_quantile_fea_df.merge(time_stat_fea_df, on= ['sn', 'server_model','fault_time'],how = 'left') time_stat_fea_df = get_4_time_stat_fea(df) df_tmp = time_stat_fea_df print(f'生成 duration_minutes 特征完毕，特征维度{df_tmp.shape}') return df_tmp def get_msg_text_fea_all(all_data): all_data['label'] = all_data['label'].fillna(-1) all_data['msg_list'] = all_data['msg'].apply(lambda x: [i.strip() for i in x.split(' | ')]) all_data['msg_0'] = all_data['msg'].apply(lambda x: [get_msg_location(x.split(' | '), 0)]) all_data['msg_1'] = all_data['msg'].apply(lambda x: [get_msg_location(x.split(' | '), 1)]) all_data['msg_2'] = all_data['msg'].apply(lambda x: [get_msg_location(x.split(' | '), 2)]) all_data = all_data.sort_values(['sn', 'fault_time', 'time']).reset_index(drop=True) del all_data['label'] last_data = all_data.query('time_interval >0') next_data = all_data.query('time_interval <=0') # id_cols = ['sn', 'fault_time', 'label'] # all_msg_text_fea = get_msg_text_fea(all_data, msg_type='all') last_msg_text_fea = get_msg_text_fea(last_data, msg_type='last') # next_msg_text_fea = get_msg_text_fea(next_data, msg_type='next') msg_text_fea = last_msg_text_fea return msg_text_fea def get_test_key_words(train,test): df = pd.concat([train[['sn', 'fault_time', 'label','msg']],test[['sn', 'fault_time', 'msg']]],ignore_index = True).drop_duplicates(['sn', 'fault_time', 'msg']) df['label'] = df['label'].fillna(5) df['msg_list'] = df['msg'].apply(lambda x:[i.strip() for i in x.split(' | ')]) words_cnt_df_list = [] for label in df['label'].unique(): label = int(label) df_tmp = df.query(f'label == {label}') counter = Counter() for words in df_tmp['msg_list']: words = [i.replace('_',' ') for i in words] # word_list = [] # for i in words: # word_list+=i.split(' ') # words = word_list counter.update(words) words_cnt_df = pd.DataFrame(counter,index = [0]).T.reset_index().rename(columns = {'index':'word',0:f'cnt_{label}'}) words_cnt_df_list.append(words_cnt_df) words_cnt_df = words_cnt_df_list[0] for i in words_cnt_df_list[1:]: words_cnt_df = words_cnt_df.merge(i,on = 'word',how = 'outer' ) words_cnt_df = words_cnt_df.fillna(-1) words_cnt_df1 = words_cnt_df.query('cnt_0 >10 and cnt_2 >10 and cnt_1 >10 and cnt_3>10 and cnt_5>10 ') cnt_class = ['cnt_0','cnt_1','cnt_2','cnt_3','cnt_5'] words_cnt_df1['word_cnt_sum'] = words_cnt_df1.loc[:,cnt_class].sum(1) for i in cnt_class: words_cnt_df1[f'{i}_ratio'] = words_cnt_df1[i]/words_cnt_df1['word_cnt_sum'] words_cnt_df1['word_cnt_ratio_std'] = words_cnt_df1.loc[:,['cnt_0_ratio','cnt_1_ratio', 'cnt_2_ratio', 'cnt_3_ratio']].std(1) words_cnt_df1['cnt_1_0_diff'] = (words_cnt_df1['cnt_1_ratio'] - words_cnt_df1['cnt_0_ratio']) test_key_words = words_cnt_df1.sort_values('cnt_5',ascending = False)['word'].to_list()[5:40] return test_key_words def get_w2v_mean(w2v_model,sentences): emb_matrix = list() vec = list() for w in sentences.split(): if w in w2v_model.wv: vec.append(w2v_model.wv[w]) if len(vec) > 0: emb_matrix.append(np.mean(vec, axis=0)) else: emb_matrix.append([0] * w2v_model.vector_size) return emb_matrix def get_tfidf_svd(tfv,svd,sentences, n_components=16): X_tfidf = tfv.transform(sentences) X_svd = svd.transform(X_tfidf) return np.mean(X_svd, axis=0) def get_w2v_tfidf_fea(all_data): print('w2v编码') df = all_data df['msg_list'] = df['msg'].apply(lambda x: [i.strip().lower().replace(' ','_') for i in x.split(" | ")]) df = df.groupby(['sn']).agg({'msg_list': 'sum'}).reset_index() df['text'] = df['msg_list'].apply(lambda x: ' '.join(x)) sentences_list = df['text'].values.tolist() sentences = [] for s in sentences_list: sentences.append([w for w in s.split()]) w2v_model = Word2Vec(sentences, vector_size=10, window=3, min_count=5, sg=0, hs=1, seed=2022) df['text_w2v'] = df['text'].apply(lambda x: get_w2v_mean(w2v_model, x)[0]) print('tfidf编码') X = df['text'].to_list() tfv = TfidfVectorizer(ngram_range=(1, 3), min_df=5, max_features=50000) tfv.fit(X) X_tfidf = tfv.transform(X) svd = TruncatedSVD(n_components=16) # 降维 svd.fit(X_tfidf) df['text_tfidf'] = df['text'].apply(lambda x: get_tfidf_svd(tfv, svd, x.split())) print("doc2vec编码") texts = df['text'].tolist() documents = [TaggedDocument(doc, [i]) for i, doc in enumerate(texts)] model = Doc2Vec(documents, window=5, min_count=3, workers=4) docvecs = model.docvecs df['doc2vec'] = [docvecs[i] for i in range(len(docvecs))] for i in range(32): df[f'msg_w2v_{i}'] = df['text_w2v'].apply(lambda x: x[i]) for i in range(16): df[f'msg_tfv_{i}'] = df['text_tfidf'].apply(lambda x: x[i]) for i in range(100): df[f'msg_doc2vec_{i}'] = df['doc2vec'].apply(lambda x: x[i]) save_cols = [i for i in df.columns if i not in ['msg_list', 'text', 'text_w2v', 'text_tfidf', 'doc2vec']] return df[save_cols] # w2v_tfidf_fea = get_w2v_tfidf_fea(all_data) class BetaEncoder(object): def __init__(self, group): self.group = group self.stats = None # get counts from df def fit(self, df, target_col): # 先验均值 self.prior_mean = np.mean(df[target_col]) stats = df[[target_col, self.group]].groupby(self.group) # count和sum stats = stats.agg(['sum', 'count'])[target_col] stats.rename(columns={'sum': 'n', 'count': 'N'}, inplace=True) stats.reset_index(level=0, inplace=True) self.stats = stats # extract posterior statistics def transform(self, df, stat_type, N_min=1): df_stats = pd.merge(df[[self.group]], self.stats, how='left') n = df_stats['n'].copy() N = df_stats['N'].copy() # fill in missing nan_indexs = np.isnan(n) n[nan_indexs] = self.prior_mean N[nan_indexs] = 1.0 # prior parameters N_prior = np.maximum(N_min - N, 0) alpha_prior = self.prior_mean * N_prior beta_prior = (1 - self.prior_mean) * N_prior # posterior parameters alpha = alpha_prior + n beta = beta_prior + N - n # calculate statistics if stat_type == 'mean': num = alpha dem = alpha + beta elif stat_type == 'mode': num = alpha - 1 dem = alpha + beta - 2 elif stat_type == 'median': num = alpha - 1 / 3 dem = alpha + beta - 2 / 3 elif stat_type == 'var': num = alpha * beta dem = (alpha + beta) ** 2 * (alpha + beta + 1) elif stat_type == 'skewness': num = 2 * (beta - alpha) * np.sqrt(alpha + beta + 1) dem = (alpha + beta + 2) * np.sqrt(alpha * beta) elif stat_type == 'kurtosis': num = 6 * (alpha - beta) ** 2 * (alpha + beta + 1) - \ alpha * beta * (alpha + beta + 2) dem = alpha * beta * (alpha + beta + 2) * (alpha + beta + 3) # replace missing value = num / dem value[np.isnan(value)] = np.nanmedian(value) return value def get_beta_target(train, test): N_min = 1000 feature_cols = [] # encode variables for c in ['server_model']: # fit encoder be = BetaEncoder(c) be.fit(train, 'label') # mean feature_name = f'{c}_mean' train[feature_name] = be.transform(train, 'mean', N_min) test[feature_name] = be.transform(test, 'mean', N_min) feature_cols.append(feature_name) # mode feature_name = f'{c}_mode' train[feature_name] = be.transform(train, 'mode', N_min) test[feature_name] = be.transform(test, 'mode', N_min) feature_cols.append(feature_name) # median feature_name = f'{c}_median' train[feature_name] = be.transform(train, 'median', N_min) test[feature_name] = be.transform(test, 'median', N_min) feature_cols.append(feature_name) # var feature_name = f'{c}_var' train[feature_name] = be.transform(train, 'var', N_min) test[feature_name] = be.transform(test, 'var', N_min) feature_cols.append(feature_name) # # skewness # feature_name = f'{c}_skewness' # train[feature_name] = be.transform(train, 'skewness', N_min) # test[feature_name] = be.transform(test, 'skewness', N_min) # feature_cols.append(feature_name) # kurtosis feature_name = f'{c}_kurtosis' train[feature_name] = be.transform(train, 'kurtosis', N_min) test[feature_name] = be.transform(test, 'kurtosis', N_min) feature_cols.append(feature_name) df = train.append(test).reset_index(drop=True) df = df[['sn', 'fault_time', 'server_model', 'server_model_mean', 'server_model_mode', 'server_model_median', 'server_model_var', 'server_model_kurtosis']].drop_duplicates().reset_index(drop=True) return df