import datetime import os import warnings import numpy as np import pandas as pd from generate_feature import get_beta_target, add_last_next_time4fault, get_feature, \ get_duration_minutes_fea, get_nearest_msg_fea, get_server_model_sn_fea_2, \ get_server_model_fea, get_msg_text_fea_all, get_key_word_cross_fea, get_server_model_time_interval_stat_fea, \ get_w2v_feats, get_key_for_top_fea,get_time_diff_feats_v2 from model import run_cbt from utils import RESULT_DIR, TRAIN_DIR, \ TEST_A_DIR, KEY_WORDS, get_word_counter, search_weight, macro_f1, TIME_INTERVAL,PSEUDO_FALG,GENERATION_DIR warnings.filterwarnings('ignore') def get_label(PSEUDO_FALG): preliminary_train_label_dataset = pd.read_csv(preliminary_train_label_dataset_path) preliminary_train_label_dataset_s = pd.read_csv(preliminary_train_label_dataset_s_path) if PSEUDO_FALG: print('获取伪标签LABEL') pseudo_labels = pd.read_csv(os.path.join(TRAIN_DIR, 'pseudo_labels.csv')) label = pd.concat([preliminary_train_label_dataset, pseudo_labels, preliminary_train_label_dataset_s], ignore_index=True, axis=0).sort_values( ['sn', 'fault_time']).reset_index(drop=True) else: print('不使用伪标签数据') label = pd.concat([preliminary_train_label_dataset, preliminary_train_label_dataset_s], ignore_index=True, axis=0).sort_values( ['sn', 'fault_time']).reset_index(drop=True) label['fault_time'] = label['fault_time'].apply(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d %H:%M:%S')) label['fault_time'] = label['fault_time'].apply(lambda x: str(x)) return label def get_log_dateset(PSEUDO_FALG): preliminary_sel_log_dataset = pd.read_csv(preliminary_sel_log_dataset_path) preliminary_sel_log_dataset_a = pd.read_csv(preliminary_sel_log_dataset_a_path) if PSEUDO_FALG: print('获取伪标签日志数据') pseudo_sel_log_dataset = pd.read_csv(os.path.join(TRAIN_DIR, 'pseudo_sel_log_dataset.csv')) log_dataset = pd.concat([preliminary_sel_log_dataset, pseudo_sel_log_dataset, preliminary_sel_log_dataset_a], ignore_index=True, axis=0).sort_values( ['sn', 'time', 'server_model']).reset_index(drop=True) else: print('不使用伪标签数据') log_dataset = pd.concat([preliminary_sel_log_dataset, preliminary_sel_log_dataset_a], ignore_index=True, axis=0).sort_values( ['sn', 'time', 'server_model']).reset_index(drop=True) log_dataset['time'] = log_dataset['time'].apply(lambda x: datetime.datetime.strptime(x, '%Y-%m-%d %H:%M:%S')) return log_dataset def get_fea_distribute(feature_df, feature_importances, dataset_type, top=30): print('根据特征重要性，获取数据集的分布情况，用于验证训练集和测试集是否分布一致') fea_distribute_list = [] for i in feature_importances[:top]['fea'].to_list(): fea_distribute_tmp = (feature_df[i].value_counts() / len(feature_df)).reset_index().rename( columns={'index': 'value'}) fea_distribute_list.append(fea_distribute_tmp) fea_distribute = fea_distribute_list[-1] for i in fea_distribute_list[:-1]: fea_distribute = fea_distribute.merge(i, on='value', how='left') fea_distribute['value'] = fea_distribute['value'].apply(lambda x: f'{dataset_type}_{int(x)}') return fea_distribute def get_train_test(label, preliminary_submit_dataset_a, log_dataset): print('获取训练集数据与测试集数据') train = label.merge(log_dataset, on='sn', how='left') test = preliminary_submit_dataset_a.merge(log_dataset, on='sn', how='left') # train['time_interval'] = (pd.to_datetime( train['fault_time'])-train['time'] ).apply(lambda x:x.total_seconds()) # test['time_interval'] = (pd.to_datetime( test['fault_time'])- test['time'] ).apply(lambda x:x.total_seconds()) # train = train.query('time_interval > 0') # test = test.query('time_interval > 0') print(f'训练集维度:{train.shape},测试集维度:{test.shape}') train = train.drop_duplicates().reset_index(drop=True) test = test.drop_duplicates().reset_index(drop=True) train['time'] = pd.to_datetime(train['time']) test['time'] = pd.to_datetime(test['time']) return train, test start_time = datetime.datetime.now() additional_sel_log_dataset_path = os.path.join(TRAIN_DIR, 'additional_sel_log_dataset.csv') preliminary_train_label_dataset_path = os.path.join(TRAIN_DIR, 'preliminary_train_label_dataset.csv') preliminary_train_label_dataset_s_path = os.path.join(TRAIN_DIR, 'preliminary_train_label_dataset_s.csv') preliminary_sel_log_dataset_path = os.path.join(TRAIN_DIR, 'preliminary_sel_log_dataset.csv') preliminary_submit_dataset_a_path = os.path.join(TEST_A_DIR, 'final_submit_dataset_b.csv') preliminary_sel_log_dataset_a_path = os.path.join(TEST_A_DIR, 'final_sel_log_dataset_b.csv') print(preliminary_submit_dataset_a_path, preliminary_sel_log_dataset_a_path) preliminary_submit_dataset_a = pd.read_csv(preliminary_submit_dataset_a_path) preliminary_submit_dataset_a.head() log_dataset = get_log_dateset(PSEUDO_FALG) label = get_label(PSEUDO_FALG) next_time_list = [i / TIME_INTERVAL for i in [3, 5, 10, 15, 30, 45, 60, 90, 120, 240, 360, 480, 540, 600]] + [1000000] label, preliminary_submit_dataset_a = add_last_next_time4fault(label, preliminary_submit_dataset_a, TIME_INTERVAL, next_time_list) train, test = get_train_test(label, preliminary_submit_dataset_a, log_dataset) train = train.drop_duplicates(['sn', 'fault_time', 'time', 'msg', 'server_model']).reset_index(drop=True) train['time_interval'] = (pd.to_datetime(train['fault_time']) - pd.to_datetime(train['time'])).apply( lambda x: x.total_seconds()) test['time_interval'] = (pd.to_datetime(test['fault_time']) - pd.to_datetime(test['time'])).apply( lambda x: x.total_seconds()) all_data = pd.concat([train, test], axis=0, ignore_index=True) all_data = all_data.sort_values(['sn','server_model', 'fault_time', 'time']) w2v_feats = get_w2v_feats(all_data, f1_list = ['sn'], f2_list = ['msg_list', 'msg_0', 'msg_1', 'msg_2']) # 获取 time_diff_feats_v2 time_diff_feats_v2 = get_time_diff_feats_v2(all_data) # 获取 server_model_time_interval_stat_fea server_model_time_interval_stat_fea = get_server_model_time_interval_stat_fea(all_data) msg_text_fea = get_msg_text_fea_all(all_data) # 获取时间差特征 duration_minutes_fea = get_duration_minutes_fea(train, test) # 获取时间server_model特征 server_model_fea = get_server_model_fea(train, test) counter = get_word_counter(train) # 获取时间 nearest_msg 特征 nearest_msg_fea = get_nearest_msg_fea(train, test) # 获取时间 server_model beta_target 特征 beta_target_fea = get_beta_target(train, test) key = ['sn', 'fault_time', 'label', 'server_model'] fea_num = len(KEY_WORDS) time_list = [i * TIME_INTERVAL for i in next_time_list] train = get_feature(train, time_list, KEY_WORDS, fea_num, key=['sn', 'fault_time', 'label', 'server_model']) test = get_feature(test, time_list, KEY_WORDS, fea_num, key=['sn', 'fault_time', 'server_model']) print('添加 时间差 特征') train = train.merge(duration_minutes_fea, on=['sn', 'fault_time', 'server_model']) test = test.merge(duration_minutes_fea, on=['sn', 'fault_time', 'server_model']) print('添加 server_model特征') train = train.merge(server_model_fea, on=['sn', 'server_model']) test = test.merge(server_model_fea, on=['sn', 'server_model']) print('添加 w2v_feats') train = train.merge(w2v_feats, on=['sn' ]) test = test.merge(w2v_feats, on=['sn', ]) print('添加 nearest_msg 特征') train = train.merge(nearest_msg_fea, on=['sn', 'server_model', 'fault_time']) test = test.merge(nearest_msg_fea, on=['sn', 'server_model', 'fault_time']) print('添加 beta_target 特征') train = train.merge(beta_target_fea, on=['sn', 'server_model', 'fault_time']) test = test.merge(beta_target_fea, on=['sn', 'server_model', 'fault_time']) server_model_sn_fea_2 = get_server_model_sn_fea_2(train, test) print('添加 server_model_sn_fea_2 特征') train = train.merge(server_model_sn_fea_2, on=['sn', 'server_model']) test = test.merge(server_model_sn_fea_2, on=['sn', 'server_model']) print('添加 time_diff_feats_v2 特征') train = train.merge(time_diff_feats_v2, on=['sn', 'server_model', 'fault_time']) test = test.merge(time_diff_feats_v2, on=['sn', 'server_model', 'fault_time']) # test.to_csv(os.path.join(GENERATION_DIR,'test.csv'),index =False) # train.to_csv(os.path.join(GENERATION_DIR,'train.csv'),index =False) # crashdump_venus_fea = pd.read_csv(os.path.join(GENERATION_DIR,'crashdump_venus_fea.csv') ) # print('添加 crashdump_venus_fea 特征') # print(train.shape,test.shape,crashdump_venus_fea.shape) # train = train.merge(crashdump_venus_fea, on=['sn' , 'fault_time'],how = 'left') # test = test.merge(crashdump_venus_fea, on=['sn', 'fault_time' ],how = 'left') # print(train.shape,test.shape ) # crashdump_venus_fea = pd.read_csv(os.path.join(GENERATION_DIR,'crashdump_venus_fea_v1.csv') ) # print('添加 crashdump_venus_fea 特征') # print(train.shape,test.shape,crashdump_venus_fea.shape) # train = train.merge(crashdump_venus_fea, on=['sn' , 'fault_time'],how = 'left') # test = test.merge(crashdump_venus_fea, on=['sn', 'fault_time' ],how = 'left') # print(train.shape,test.shape ) # test.to_csv(os.path.join(GENERATION_DIR,'test.csv'),index =False) # train.to_csv(os.path.join(GENERATION_DIR,'train.csv'),index =False) # print('添加 key_for_top_fea 特征') # train,test = get_key_for_top_fea(train,test) # print('添加 w2v_tfidf_doc2v_fea 特征') # w2v_tfidf_fea = pd.read_csv(os.path.join(GENERATION_DIR,'w2v_tfidf_fea.csv')) # drop_cols = [i for i in w2v_tfidf_fea if 'doc2vec' in i ]+[i for i in w2v_tfidf_fea if 'tfidf' in i ] # for col in drop_cols: # del w2v_tfidf_fea[col] # # train = train.merge(w2v_tfidf_fea, on=['sn' ], how='left') # test = test.merge(w2v_tfidf_fea, on=['sn' ], how='left') # print('添加 关键词交叉特征 ') # train,test = get_key_word_cross_fea(train,test) # print('添加 server_model_time_interval_stat_fea 特征') # train = train.merge(server_model_time_interval_stat_fea, on=['server_model' ],how ='left') # test = test.merge(server_model_time_interval_stat_fea, on=['server_model' ],how ='left') use_less_cols_1 = ['last_last_msg_cnt', 'last_first_msg_cnt','time_diff_1_min', 'last_msg_list_unique_LabelEnc', 'last_msg_0_unique_LabelEnc', 'last_msg_1_unique_LabelEnc', 'last_msg_2_unique_LabelEnc', 'last_msg_list_list_LabelEnc', 'last_msg_0_list_LabelEnc', 'last_msg_1_list_LabelEnc', 'last_msg_2_list_LabelEnc', 'last_msg_0_first_LabelEnc', 'last_msg_1_first_LabelEnc', 'last_msg_2_first_LabelEnc', 'last_msg_0_last_LabelEnc', 'last_msg_1_last_LabelEnc', 'last_msg_2_last_LabelEnc', 'last_msg_last_LabelEnc', 'last_msg_first_LabelEnc'] use_less_col = [i for i in train.columns if train[i].nunique() < 2] + use_less_cols_1 print(f'use_less_col:{len(use_less_col)}') use_cols = [i for i in train.columns if i not in ['sn', 'fault_time', 'label', 'server_model'] + use_less_col] cat_cols = ['server_model_LabelEnc', 'msg_LabelEnc', 'msg_0_LabelEnc', 'msg_1_LabelEnc', 'msg_2_LabelEnc',] use_cols = sorted(use_cols) cat_cols = [] for i in use_cols: if '_LabelEnc' in i: cat_cols.append(i) print('使用的特征维度:',len(use_cols),'类别特征维度:',len(cat_cols)) # fs = FeatureSelector(data=train[use_cols], labels=train['label']) # # # 选择出missing value 百分比大于60%的特征 # fs.identify_missing(missing_threshold=0.9) # # # # 查看选择出的特征 # # fs.ops['missing'] # # 不对feature进行one-hot encoding（默认为False）, 然后选择出相关性大于98%的feature, # fs.identify_collinear(correlation_threshold=0.99, one_hot=False) # # # # 查看选择的feature # # fs.ops['collinear'] # # # 选择出只有单个值的feature # fs.identify_single_unique() # # # # 查看选择出的feature # # fs.ops['single_unique'] # # train_removed = fs.remove(methods = ['missing', 'single_unique', 'collinear',], keep_one_hot=False) # use_cols = train_removed.columns # print('特征选择之后，使用的特征维度:',len(use_cols)) oof_prob = np.zeros((train.shape[0], 4)) test_prob = np.zeros((test.shape[0], 4)) # seeds = [42,4242,40424,1024,2048] seeds = [42 ] for seed in seeds: oof_prob, test_prob, fea_imp_df, model_list = run_cbt(train[use_cols] , train[['label']] , test[use_cols], k=5, seed=seed, cat_cols=cat_cols) oof_prob +=oof_prob/len(seeds) test_prob +=test_prob/len(seeds) weight = search_weight(train, train[['label']], oof_prob, init_weight=[1.0], class_num=4, step=0.001) oof_prob = oof_prob * np.array(weight) test_prob = test_prob * np.array(weight) target_df = train[['sn', 'fault_time', 'label']] submit_df = train[['sn', 'fault_time']] submit_df['label'] = oof_prob.argmax(axis=1) score = macro_f1(target_df=target_df, submit_df=submit_df) print(f'********************** BEST MACRO_F1 : {score} **********************') score = round(score, 5) y_pred = test_prob.argmax(axis=1) result = test[['sn', 'fault_time']] result['label'] = y_pred result = preliminary_submit_dataset_a.merge(result, on=['sn', 'fault_time'], how='left')[['sn', 'fault_time', 'label']] result['label'] = result['label'].fillna(0).astype(int) result.to_csv(os.path.join(RESULT_DIR,f'catboost_result.csv'), index=False) print(result['label'].value_counts()) fea_imp_df = fea_imp_df.reset_index(drop = True) fea_imp_df.to_csv(os.path.join(RESULT_DIR,f'./cat_fea_imp_{int(score*100000)}.csv'),index = False) train_result_prob = pd.DataFrame(oof_prob).add_prefix('cat_class_') test_result_prob = pd.DataFrame(test_prob).add_prefix('cat_class_') train_result_prob['label'] = train['label'] train_result_prob['sn'] = train['sn'] train_result_prob['fault_time'] = train['fault_time'] test_result_prob['sn'] = test['sn'] test_result_prob['fault_time'] = test['fault_time'] result_prob = pd.concat([train_result_prob,test_result_prob],ignore_index = True) result_prob.to_csv(os.path.join(RESULT_DIR,f'cat_prob_result.csv'),index = False) end_time = datetime.datetime.now() cost_time = end_time - start_time print('****************** CATBOOST COST TIME : ',str(cost_time),' ******************') ''' v7: 最优 线下 0.7303 v8: v7 添加 关键词交叉特征 并作为类别变量输入模型 0.73114 '''