msb-public-java
/
AiLearning-Theory-Applying
mirror of https://github.com/ben1234560/AiLearning-Theory-Applying
2
6
Code Issues Projects Releases Wiki Activity

Merge pull request #2 from AiIsBetter/master

Browse Source 
Add. code of LGB
master
ben.guo 4 years ago committed by GitHub
parent 91d738e8b8 e709f62cce
commit 55ba5258e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 1716 additions and 0 deletions
Show Stats Download Patch File Download Diff File

153
机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/LGB_13700/1_sdne_embedding_allnext.py
Unescape View File

@ -0,0 +1,153 @@
#coding=utf-8
"""
Author: Aigege
Code: https://github.com/AiIsBetter
"""
# date 2021.08.01
import numpy as np
import networkx as nx
import pandas as pd
from gem.embedding.node2vec import node2vec
import os
from utils import parallel_apply
from functools import partial
import gc
def link_id_find(gr):
gr_ = gr.copy()
tmp = list(gr_['link_id'])
link_id_tuple = []
for i in range(len(tmp)-1):
link_id_tuple.append([tmp[i],tmp[i+1]])
return link_id_tuple
if __name__ == '__main__':
root_path = '../data/giscup_2021/'
nrows = None
######################################nextlinks #######################################
nextlinks = pd.read_csv(root_path + 'nextlinks.txt', sep=' ', header=None)
nextlinks.columns = ['from_id', 'to_id']
nextlinks['to_id'] = nextlinks['to_id'].astype('str')
nextlinks['to_id'] = nextlinks['to_id'].apply(lambda x: x.split(","))
nextlinks = pd.DataFrame({'from_id': nextlinks.from_id.repeat(nextlinks.to_id.str.len()),
'to_id': np.concatenate(nextlinks.to_id.values)})
nextlinks['from_id'] = nextlinks['from_id'].astype(int)
nextlinks['to_id'] = nextlinks['to_id'].astype(int)
from_id = nextlinks['from_id'].unique()
# nextlinks.to_csv('../data/giscup_2021/nextlink_all.csv',index=False)
# nextlinks = pd.read_csv('../data/giscup_2021/nextlink_all.csv')
######################################nextlinks #######################################
if 'nextlinks_allday.csv' in os.listdir(root_path):
nextlinks = pd.read_csv(root_path + 'nextlinks_allday.csv')
else:
nextlinks_new = []
for name in os.listdir(root_path + 'train/'):
data_time = name.split('.')[0]
if data_time == '20200803':
continue
train = pd.read_csv(root_path + 'train/{}'.format(name),sep= ';;',header=None,nrows=nrows)
train_head = pd.DataFrame(train[0].str.split(' ').tolist(),
columns=['order_id', 'ata', 'distance', 'simple_eta', 'driver_id', 'slice_id'])
train_head['order_id'] = train_head['order_id'].astype(str)
train_head['ata'] = train_head['ata'].astype(float)
train_head['distance'] = train_head['distance'].astype(float)
train_head['simple_eta'] = train_head['simple_eta'].astype(float)
train_head['driver_id'] = train_head['driver_id'].astype(int)
train_head['slice_id'] = train_head['slice_id'].astype(int)
data_link = train[[1]]
print("flag:", 1)
data_link['index'] = train_head.index
data_link['order_id'] = train_head['order_id']
print("flag:", 2)
data_link['ata'] = train_head['ata']
data_link['distance'] = train_head['distance']
data_link['simple_eta'] = train_head['simple_eta']
print("flag:", 3)
data_link['slice_id'] = train_head['slice_id']
print("flag:", 4)
data_link_split = data_link[1].str.split(' ', expand=True).stack().to_frame()
print("flag:", 5)
data_link_split = data_link_split.reset_index(level=1, drop=True).rename(columns={0: 'link_info'})
print("flag:", 6)
data_link_split = data_link[['order_id', 'index', 'ata', 'distance', 'simple_eta', 'slice_id']].join(
data_link_split)
print("flag:", 7)
data_link_split = data_link_split.reset_index(drop=True)
data_link_split[['link_id',
'link_time',
'link_ratio',
'link_current_status',
'link_arrival_status']] = data_link_split['link_info'].str.split(':|,', 5, expand=True)
print("flag:", 8)
data_link_split = data_link_split[['order_id','link_id']]
data_link_split['link_id'] = data_link_split['link_id'].astype(int)
features = pd.DataFrame({'order_id': data_link_split['order_id'].unique()})
groupby = data_link_split.groupby(['order_id'])
func = partial(link_id_find)
g = parallel_apply(groupby, func, index_name='order_id', num_workers=5, chunk_size=10000)
g = pd.DataFrame(g,columns=['from_id','to_id'])
g = g.drop_duplicates()
nextlinks_new.append(g)
nextlinks_new = pd.concat(nextlinks_new, axis=0)
nextlinks_new = nextlinks_new.drop_duplicates()
nextlinks_new = nextlinks_new.sort_values(by='from_id').reset_index(drop=True)
nextlinks = pd.concat([nextlinks,nextlinks_new],axis=0)
nextlinks = nextlinks.drop_duplicates()
nextlinks = nextlinks.sort_values(by='from_id').reset_index(drop=True)
print('save all csv')
nextlinks.to_csv(root_path+'nextlinks_allday.csv',index=False)
print('calcute weight')
nextlinks = nextlinks.sort_values(by='from_id').reset_index(drop=True)
nextlinks = nextlinks.drop_duplicates()
from_id_weight = nextlinks['from_id'].value_counts()
from_id_weight = from_id_weight.to_frame()
from_id_weight['index'] = from_id_weight.index
from_id_weight.columns = ['weight', 'from_id']
nextlinks = pd.merge(nextlinks, from_id_weight, 'left', on=['from_id'])
print('calcute weight finish!')
nextlinks['to_id'] = nextlinks['to_id'].astype(int)
nextlinks['from_id'] = nextlinks['from_id'].astype(int)
id_key = list(set(nextlinks['from_id'].unique().tolist() + nextlinks['to_id'].unique().tolist()))
id_key_to_connected = dict(zip(id_key, range(len(id_key))))
nextlinks['from_id'] = nextlinks['from_id'].map(id_key_to_connected)
nextlinks['to_id'] = nextlinks['to_id'].map(id_key_to_connected)
np.save(root_path + 'id_key_to_connected_allday.npy', id_key_to_connected)
print('id key save finish!')
print('start creating graph')
G = nx.DiGraph()
from_id = nextlinks['from_id'].to_list()
to_id = nextlinks['to_id'].to_list()
weight = nextlinks['weight'].to_list()
edge_tuple = list(zip(from_id, to_id,weight))
# edge_tuple = tuple(from_id,to_id,weight)
print('adding')
G.add_weighted_edges_from(edge_tuple)
G = G.to_directed()
print('finish create graph!')
print('start train n2v')
look_back = list(G.nodes())
embeddings = {}
models = []
models.append(node2vec(d=128, max_iter=10, walk_len=80, num_walks=10, con_size=10, ret_p=1, inout_p=1))
for embedding in models:
Y, t = embedding.learn_embedding(graph=G, edge_f=None,
is_weighted=True, no_python=True)
for i, embedding in enumerate(embedding.get_embedding()):
embeddings[look_back[i]] = embedding
np.save(root_path+'graph_embeddings_retp1.npy', embeddings)
print('nextlink graph embedding retp 1 finish!') # displays "world"
del models
gc.collect()
look_back = list(G.nodes())
embeddings = {}
models = []
models.append(node2vec(d=128, max_iter=10, walk_len=80, num_walks=10, con_size=10, ret_p=0.5, inout_p=1))
for embedding in models:
Y, t = embedding.learn_embedding(graph=G, edge_f=None,
is_weighted=True, no_python=True)
for i, embedding in enumerate(embedding.get_embedding()):
embeddings[look_back[i]] = embedding
np.save(root_path + 'graph_embeddings_retp05.npy', embeddings)
print('nextlink graph embedding retp 0.5 finish!')

400
机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/LGB_13700/2_cross_fea_order_id_level.py
Unescape View File

@ -0,0 +1,400 @@
#coding=utf-8
"""
Author: Aigege
Code: https://github.com/AiIsBetter
"""
# date 2021.08.01
import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.feature_extraction.text import CountVectorizer
import networkx as nx
import os
import gc
import warnings
from utils import parallel_apply_fea,add_features_in_group
from functools import partial
warnings.filterwarnings("ignore")
def last_k_cross_time_interval(gr, periods):
gr_ = gr.copy()
gr_ = gr_.iloc[::-1]
gr_['t_i_v'] = gr_['cross_time'].diff()
gr_['t_i_v'] = gr_['t_i_v']
gr_['t_i_v'] = gr_['t_i_v'].fillna(0)
gr_ = gr_.drop_duplicates().reset_index(drop = True)
# cross time变化
features = {}
for period in periods:
if period > 10e5:
period_name = 'zsl_cross_time_interval_all'
gr_period = gr_.copy()
else:
period_name = 'zsl_cross_time_interval_last_{}_'.format(period)
gr_period = gr_.iloc[:period]
features = add_features_in_group(features, gr_period, 't_i_v',
['mean','max', 'min', 'std','sum'],
period_name)
return features
# last k cross id time trend
def last_cross_time_features(gr,periods):
gr_ = gr.copy()
gr_ = gr_.iloc[::-1]
features = {}
for period in periods:
if period > 10e5:
period_name = 'zsl_all_'
gr_period = gr_.copy()
else:
period_name = 'zsl_last_{}_'.format(period)
gr_period = gr_.iloc[:period]
features = add_features_in_group(features, gr_period, 'cross_time',
['max', 'sum', 'mean','min','std'],
period_name)
return features
# last k cross id time trend
def trend_in_last_k_cross_id_time(gr, periods):
gr_ = gr.copy()
gr_ = gr_.iloc[::-1]
features = {}
for period in periods:
gr_period = gr_.iloc[:period]
features = add_trend_feature(features, gr_period,
'cross_time', 'zsl_{}_period_trend_'.format(period)
)
return features
# trend feature
def add_trend_feature(features, gr, feature_name, prefix):
y = gr[feature_name].values
try:
x = np.arange(0, len(y)).reshape(-1, 1)
lr = LinearRegression()
lr.fit(x, y)
trend = lr.coef_[0]
except:
trend = np.nan
features['{}{}'.format(prefix, feature_name)] = trend
return features
def slice_id_change(x):
hour = x * 5 / 60
hour = np.floor(hour)
hour += 8
if hour >= 24:
hour = hour - 24
return hour
if __name__ == '__main__':
nrows = None
root_path = '../data/giscup_2021/'
read_idkey = np.load(root_path + 'id_key_to_connected_allday.npy', allow_pickle=True).item()
read_grapheb = np.load(root_path + 'graph_embeddings_retp1_directed.npy', allow_pickle=True).item()
read_grapheb_retp = np.load(root_path + 'graph_embeddings_retp05_directed.npy', allow_pickle=True).item()
for i in read_grapheb:
read_grapheb[i] = list(read_grapheb[i]) + list(read_grapheb_retp[i])
del read_grapheb_retp
head_columns = ['order_id', 'ata', 'distance', 'simple_eta', 'driver_id','slice_id']
embedding_k = 256
fill_list = [0] * embedding_k
df = []
#######################################nextlinks #######################################
nextlinks = pd.read_csv(root_path+'nextlinks.txt', sep=' ', header=None)
nextlinks.columns=['from_id', 'to_id']
nextlinks['to_id'] = nextlinks['to_id'].astype('str')
nextlinks['to_id'] = nextlinks['to_id'].apply(lambda x: x.split(","))
nextlinks = pd.DataFrame({'from_id':nextlinks.from_id.repeat(nextlinks.to_id.str.len()),
'to_id':np.concatenate(nextlinks.to_id.values)})
from_id_weight = nextlinks['from_id'].value_counts()
from_id_weight = from_id_weight.to_frame()
from_id_weight['index'] = from_id_weight.index
from_id_weight.columns=['weight', 'from_id']
nextlinks = pd.merge(nextlinks,from_id_weight, 'left', on=['from_id'])
nextlinks = nextlinks.sort_values(by='weight',ascending=False)
G = nx.DiGraph()
from_id = nextlinks['from_id'].astype(str).to_list()
to_id = nextlinks['to_id'].to_list()
weight = nextlinks['weight'].to_list()
edge_tuple = list(zip(from_id, to_id,weight))
print('adding')
G.add_weighted_edges_from(edge_tuple)
dc = nx.algorithms.centrality.degree_centrality(G)
dc = sorted(dc.items(), key=lambda d: d[1],reverse=True)
dc = dc[:50000]
dc = [str(i[0]) for i in dc ]
#######################################cross #######################################
for name in os.listdir(root_path+'train/'):
data_time = name.split('.')[0]
if data_time=='20200803':
continue
train = pd.read_csv(root_path+'train/{}'.format(name),sep= ';;',header=None,nrows=nrows)
print("开始处理", data_time)
train_head = pd.DataFrame(train[0].str.split(' ').tolist(),columns = ['order_id', 'ata', 'distance','simple_eta', 'driver_id', 'slice_id'])
train_head['order_id'] = train_head['order_id'].astype(str)
train_head['ata'] = train_head['ata'].astype(float)
train_head['distance'] = train_head['distance'].astype(float)
train_head['simple_eta'] = train_head['simple_eta'].astype(float)
train_head['driver_id'] = train_head['driver_id'].astype(int)
train_head['slice_id'] = train_head['slice_id'].astype(int)
# 处理corss数据
data_cross = train[[2]]
data_cross['index'] = train_head.index
data_cross['order_id'] = train_head['order_id']
data_cross_split = data_cross[2].str.split(' ', expand=True).stack().to_frame()
data_cross_split = data_cross_split.reset_index(level=1, drop=True).rename(columns={0: 'cross_info'})
data_cross_split = data_cross[['index', 'order_id']].join(data_cross_split)
data_cross_split[['cross_id', 'cross_time']] = data_cross_split['cross_info'].str.split(':', 2, expand=True)
data_cross_split['cross_time'] = data_cross_split['cross_time'].astype(float)
tmp_cross_id = data_cross_split['cross_id'].str.split('_', expand=True)
tmp_cross_id.columns=['cross_id_in','cross_id_out']
data_cross_split = pd.concat([data_cross_split,tmp_cross_id],axis=1).drop(['cross_id','cross_info'],axis=1)
data_cross_split['date_time'] = data_time
data_cross_split = data_cross_split.drop('index',axis=1).reset_index(drop=True)
print('preprocess finish!')
print('start feature engineering')
feature = train_head[['order_id', 'distance']]
###################static fea#############################################
data_cross_split['zsl_cross_id_isnull'] =0
data_cross_split.loc[data_cross_split['cross_id_in'].isnull(),'zsl_cross_id_isnull'] = 1
data_cross_split.loc[data_cross_split['cross_id_in'].isnull(),'cross_id_in'] = '-1'
data_cross_split.loc[data_cross_split['cross_id_out'].isnull(),'cross_id_out'] = '-1'
#######################order cross_id count###############################
df = data_cross_split.groupby('order_id', as_index=False)
tmp_crossid_agg = df['cross_id_in'].agg({'zsl_order_cross_id_in_count': 'count'})
tmp_crossid_agg['zsl_order_cross_id_in_count_bins'] = 0
tmp_crossid_agg.loc[(tmp_crossid_agg['zsl_order_cross_id_in_count']>=5)&(tmp_crossid_agg['zsl_order_cross_id_in_count']<10),'zsl_order_cross_id_in_count_bins']=1
tmp_crossid_agg.loc[(tmp_crossid_agg['zsl_order_cross_id_in_count']>=10)&(tmp_crossid_agg['zsl_order_cross_id_in_count']<20),'zsl_order_cross_id_in_count_bins']=2
tmp_crossid_agg.loc[(tmp_crossid_agg['zsl_order_cross_id_in_count']>=20),'zsl_order_cross_id_in_count_bins']=3
feature = feature.merge(tmp_crossid_agg,on='order_id',how='left')
print('order cross_id count finish!')
#######################order cross id & distance###############################
feature['zsl_order_cross_is_highspeed'] = 0
feature.loc[(feature['distance']>90000)&(feature['zsl_order_cross_id_in_count']<30),'zsl_order_cross_is_highspeed'] = 1
print('order cross id & distance finish!')
#######################order cross id & nextlinks centry###############################
tmp = data_cross_split[data_cross_split['cross_id_in'].isin(dc)]
tmp = tmp.groupby('order_id', as_index=False)
tmp_linkid_centry_count = tmp['cross_id_in'].agg({'zsl_order_cross_id_in_centry_count': 'count'})
feature = feature.merge(tmp_linkid_centry_count,on='order_id',how='left')
feature['zsl_order_cross_id_in_centry_count'] = feature['zsl_order_cross_id_in_centry_count'].fillna(0)
tmp = data_cross_split[data_cross_split['cross_id_out'].isin(dc)]
tmp = tmp.groupby('order_id', as_index=False)
tmp_linkid_centry_count = tmp['cross_id_out'].agg({'zsl_order_cross_id_out_centry_count': 'count'})
feature = feature.merge(tmp_linkid_centry_count, on='order_id', how='left')
feature['zsl_order_cross_id_out_centry_count'] = feature['zsl_order_cross_id_out_centry_count'].fillna(0)
print('order cross_id & nextlinks centry finish!')
#######################order cross_time sum mean max min var std###############################
tmp_linktime_agg = df['cross_time'].agg({'zsl_order_cross_time_sum': 'sum','zsl_order_cross_time_mean': 'mean',
'zsl_order_cross_time_max': 'max','zsl_order_cross_time_min': 'min',
'zsl_order_cross_time_var': 'var'})
feature = feature.merge(tmp_linktime_agg,on='order_id',how='left')
print('order cross_time sum mean max min var std finish!')
#######################order distance/link_id_count###############################
feature['zsl_distance_div_cross_id_count'] = feature['distance']*10/feature['zsl_order_cross_id_in_count']
feature = feature.drop('distance', axis=1)
print('order distance div link_id_count finish!')
###################trend fea#############################################
###################trend cross time#####################################
groupby = data_cross_split.groupby(['order_id'])
func = partial(trend_in_last_k_cross_id_time, periods=[2, 5, 10, 20,100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=5, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
func = partial(last_cross_time_features, periods=[2, 5, 10, 20,100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=5, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
func = partial(last_k_cross_time_interval, periods=[2, 5, 10, 20, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=5, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
print('trend cross time finish!')
####################nextlinks graph embedding#######################
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].astype(int)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].map(read_idkey)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].fillna(0)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].astype(int)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].map(read_grapheb)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].fillna('0')
def replace_list(x):
if isinstance(x, str):
x = fill_list
return x
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].apply(replace_list)
cross_id_in_col = ['zsl_cross_id_in_eb{}'.format(i) for i in range(embedding_k)]
agg_col = dict(zip(cross_id_in_col, ['mean'] * len(cross_id_in_col)))
cross_id_in_array = np.array(data_cross_split.pop('cross_id_in').to_list())
cross_id_in_array = pd.DataFrame(cross_id_in_array, columns=agg_col, dtype=np.float16)
data_cross_split = pd.concat([data_cross_split, cross_id_in_array], axis=1)
tmp = data_cross_split.groupby('order_id', as_index=False)
tmp_crossidin_agg = tmp.agg(agg_col)
feature = feature.merge(tmp_crossidin_agg, on='order_id', how='left')
print('trend cross_id_in eb finish!')
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].astype(int)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].map(read_idkey)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].fillna(0)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].astype(int)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].map(read_grapheb)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].fillna('0')
def replace_list(x):
if isinstance(x, str):
x = fill_list
return x
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].apply(replace_list)
cross_id_out_col = ['zsl_cross_id_out_eb{}'.format(i) for i in range(embedding_k)]
agg_col = dict(zip(cross_id_out_col, ['mean'] * len(cross_id_out_col)))
cross_id_out_array = np.array(data_cross_split.pop('cross_id_out').to_list())
cross_id_out_array = pd.DataFrame(cross_id_out_array, columns=agg_col, dtype=np.float16)
data_cross_split = pd.concat([data_cross_split, cross_id_out_array], axis=1)
tmp = data_cross_split.groupby('order_id', as_index=False)
tmp_crossidout_agg = tmp.agg(agg_col)
feature = feature.merge(tmp_crossidout_agg, on='order_id', how='left')
print('trend cross_id_out eb finish!')
multipy_df = []
multipy_col = []
for col1, col2 in zip(cross_id_in_col, cross_id_out_col):
tmp = feature[col1] * feature[col2]
multipy_df.append(tmp)
multipy_col.append(col1 + '_mul_' + col2)
multipy_df = pd.concat(multipy_df, axis=1)
multipy_df.columns = multipy_col
feature = pd.concat([feature, multipy_df], axis=1)
print('trend cross_id_out eb multipy finish!')
feature.to_csv(root_path + 'feature/train/cross_fea_order_id_level_{}.csv'.format(data_time), index=False)
del train
gc.collect()
test = pd.read_csv(root_path+'20200901_test.txt',sep= ';;',header=None,nrows=nrows)
test_head = pd.DataFrame(test[0].str.split(' ').tolist(),columns = ['order_id', 'ata', 'distance','simple_eta', 'driver_id', 'slice_id'])
test_head['order_id'] = test_head['order_id'].astype(str)
test_head['ata'] = test_head['ata'].astype(float)
test_head['distance'] = test_head['distance'].astype(float)
test_head['simple_eta'] = test_head['simple_eta'].astype(float)
test_head['driver_id'] = test_head['driver_id'].astype(int)
test_head['slice_id'] = test_head['slice_id'].astype(int)
# 处理corss数据
data_cross = test[[2]]
data_cross['index'] = test_head.index
data_cross['order_id'] = test_head['order_id']
data_cross_split = data_cross[2].str.split(' ', expand=True).stack().to_frame()
data_cross_split = data_cross_split.reset_index(level=1, drop=True).rename(columns={0: 'cross_info'})
data_cross_split = data_cross[['index', 'order_id']].join(data_cross_split)
data_cross_split[['cross_id', 'cross_time']] = data_cross_split['cross_info'].str.split(':', 2, expand=True)
data_cross_split['cross_time'] = data_cross_split['cross_time'].astype(float)
tmp_cross_id = data_cross_split['cross_id'].str.split('_', expand=True)
tmp_cross_id.columns = ['cross_id_in', 'cross_id_out']
data_cross_split = pd.concat([data_cross_split, tmp_cross_id], axis=1).drop(['cross_id', 'cross_info'], axis=1)
data_cross_split['date_time'] = '20200901'
data_cross_split = data_cross_split.drop('index', axis=1).reset_index(drop=True)
print('preprocess finish!')
print('start feature engineering')
feature = test_head[['order_id', 'distance']]
###################static fea#############################################
data_cross_split['zsl_cross_id_isnull'] = 0
data_cross_split.loc[data_cross_split['cross_id_in'].isnull(), 'zsl_cross_id_isnull'] = 1
data_cross_split.loc[data_cross_split['cross_id_in'].isnull(), 'cross_id_in'] = '-1'
data_cross_split.loc[data_cross_split['cross_id_out'].isnull(), 'cross_id_out'] = '-1'
#######################order cross_id count###############################
df = data_cross_split.groupby('order_id', as_index=False)
tmp_crossid_agg = df['cross_id_in'].agg({'zsl_order_cross_id_in_count': 'count'})
tmp_crossid_agg['zsl_order_cross_id_in_count_bins'] = 0
tmp_crossid_agg.loc[(tmp_crossid_agg['zsl_order_cross_id_in_count'] >= 5) & (
tmp_crossid_agg['zsl_order_cross_id_in_count'] < 10), 'zsl_order_cross_id_in_count_bins'] = 1
tmp_crossid_agg.loc[(tmp_crossid_agg['zsl_order_cross_id_in_count'] >= 10) & (
tmp_crossid_agg['zsl_order_cross_id_in_count'] < 20), 'zsl_order_cross_id_in_count_bins'] = 2
tmp_crossid_agg.loc[(tmp_crossid_agg['zsl_order_cross_id_in_count'] >= 20), 'zsl_order_cross_id_in_count_bins'] = 3
feature = feature.merge(tmp_crossid_agg, on='order_id', how='left')
print('order cross_id count finish!')
#######################order cross id & distance###############################
feature['zsl_order_cross_is_highspeed'] = 0
feature.loc[(feature['distance'] > 90000) & (
feature['zsl_order_cross_id_in_count'] < 30), 'zsl_order_cross_is_highspeed'] = 1
print('order cross id & distance finish!')
#######################order cross id & nextlinks centry###############################
tmp = data_cross_split[data_cross_split['cross_id_in'].isin(dc)]
tmp = tmp.groupby('order_id', as_index=False)
tmp_linkid_centry_count = tmp['cross_id_in'].agg({'zsl_order_cross_id_in_centry_count': 'count'})
feature = feature.merge(tmp_linkid_centry_count, on='order_id', how='left')
feature['zsl_order_cross_id_in_centry_count'] = feature['zsl_order_cross_id_in_centry_count'].fillna(0)
tmp = data_cross_split[data_cross_split['cross_id_out'].isin(dc)]
tmp = tmp.groupby('order_id', as_index=False)
tmp_linkid_centry_count = tmp['cross_id_out'].agg({'zsl_order_cross_id_out_centry_count': 'count'})
feature = feature.merge(tmp_linkid_centry_count, on='order_id', how='left')
feature['zsl_order_cross_id_out_centry_count'] = feature['zsl_order_cross_id_out_centry_count'].fillna(0)
print('order cross_id & nextlinks centry finish!')
#######################order cross_time sum mean max min var std###############################
tmp_linktime_agg = df['cross_time'].agg({'zsl_order_cross_time_sum': 'sum', 'zsl_order_cross_time_mean': 'mean',
'zsl_order_cross_time_max': 'max', 'zsl_order_cross_time_min': 'min',
'zsl_order_cross_time_var': 'var'})
feature = feature.merge(tmp_linktime_agg, on='order_id', how='left')
print('order cross_time sum mean max min var std finish!')
#######################order distance/link_id_count###############################
feature['zsl_distance_div_cross_id_count'] = feature['distance'] * 10 / feature['zsl_order_cross_id_in_count']
feature = feature.drop('distance', axis=1)
print('order distance div link_id_count finish!')
###################trend fea#############################################
###################trend cross time#####################################
groupby = data_cross_split.groupby(['order_id'])
func = partial(trend_in_last_k_cross_id_time, periods=[2, 5, 10, 20, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=5, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
func = partial(last_cross_time_features, periods=[2, 5, 10, 20, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=5, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
func = partial(last_k_cross_time_interval, periods=[2, 5, 10, 20, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=5, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
print('trend cross time finish!')
####################nextlinks graph embedding#######################
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].astype(int)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].map(read_idkey)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].fillna(0)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].astype(int)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].map(read_grapheb)
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].fillna('0')
def replace_list(x):
if isinstance(x, str):
x = fill_list
return x
data_cross_split['cross_id_in'] = data_cross_split['cross_id_in'].apply(replace_list)
cross_id_in_col = ['zsl_cross_id_in_eb{}'.format(i) for i in range(embedding_k)]
agg_col = dict(zip(cross_id_in_col, ['mean'] * len(cross_id_in_col)))
cross_id_in_array = np.array(data_cross_split.pop('cross_id_in').to_list())
cross_id_in_array = pd.DataFrame(cross_id_in_array, columns=agg_col, dtype=np.float16)
data_cross_split = pd.concat([data_cross_split, cross_id_in_array], axis=1)
tmp = data_cross_split.groupby('order_id', as_index=False)
tmp_crossidin_agg = tmp.agg(agg_col)
feature = feature.merge(tmp_crossidin_agg, on='order_id', how='left')
print('trend cross_id_in eb finish!')
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].astype(int)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].map(read_idkey)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].fillna(0)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].astype(int)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].map(read_grapheb)
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].fillna('0')
def replace_list(x):
if isinstance(x, str):
x = fill_list
return x
data_cross_split['cross_id_out'] = data_cross_split['cross_id_out'].apply(replace_list)
cross_id_out_col = ['zsl_cross_id_out_eb{}'.format(i) for i in range(embedding_k)]
agg_col = dict(zip(cross_id_out_col, ['mean'] * len(cross_id_out_col)))
cross_id_out_array = np.array(data_cross_split.pop('cross_id_out').to_list())
cross_id_out_array = pd.DataFrame(cross_id_out_array, columns=agg_col, dtype=np.float16)
data_cross_split = pd.concat([data_cross_split, cross_id_out_array], axis=1)
tmp = data_cross_split.groupby('order_id', as_index=False)
tmp_crossidout_agg = tmp.agg(agg_col)
feature = feature.merge(tmp_crossidout_agg, on='order_id', how='left')
print('trend cross_id_out eb finish!')
multipy_df = []
multipy_col = []
for col1, col2 in zip(cross_id_in_col, cross_id_out_col):
tmp = feature[col1] * feature[col2]
multipy_df.append(tmp)
multipy_col.append(col1 + '_mul_' + col2)
multipy_df = pd.concat(multipy_df, axis=1)
multipy_df.columns = multipy_col
feature = pd.concat([feature, multipy_df], axis=1)
print('trend cross_id_out eb multipy finish!')
feature.to_csv(root_path + 'feature/test/cross_fea_order_id_level_20200901.csv', index=False)

438
机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/LGB_13700/3_link_fea_order_id_level.py
Unescape View File

@ -0,0 +1,438 @@
#coding=utf-8
"""
Author: Aigege
Code: https://github.com/AiIsBetter
"""
# date 2021.08.01
import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.feature_extraction.text import CountVectorizer
import networkx as nx
import os
import gc
import warnings
from utils import parallel_apply_fea,add_features_in_group
from functools import partial
warnings.filterwarnings("ignore")
def last_k_link_time_interval(gr, periods):
gr_ = gr.copy()
gr_ = gr_.iloc[::-1]
gr_['t_i_v'] = gr_['link_time'].diff()
gr_['t_i_v'] = gr_['t_i_v']
gr_['t_i_v'] = gr_['t_i_v'].fillna(0)
gr_['c_s_v'] = gr_['link_current_status'].diff()
gr_['c_s_v'] = gr_['c_s_v']
gr_['c_s_v'] = gr_['c_s_v'].fillna(0)
gr_ = gr_.drop_duplicates().reset_index(drop = True)
# link time变化
features = {}
for period in periods:
if period > 10e5:
period_name = 'zsl_link_time_interval_all'
gr_period = gr_.copy()
else:
period_name = 'zsl_link_time_interval_last_{}_'.format(period)
gr_period = gr_.iloc[:period]
features = add_features_in_group(features, gr_period, 't_i_v',
['mean','max', 'min', 'std','skew','sum'],
# ['diff'],
period_name)
# current status变化
for period in periods:
if period > 10e5:
period_name = 'zsl_link_current_status_interval_all'
gr_period = gr_.copy()
else:
period_name = 'zsl_link_current_status_interval_last_{}_'.format(period)
gr_period = gr_.iloc[:period]
features = add_features_in_group(features, gr_period, 'c_s_v',
['mean', 'std', 'skew'],
# ['diff'],
period_name)
return features
# last k link id time trend
def last_link_time_features(gr,periods):
gr_ = gr.copy()
gr_ = gr_.iloc[::-1]
features = {}
for period in periods:
if period > 10e5:
period_name = 'zsl_all_'
gr_period = gr_.copy()
else:
period_name = 'zsl_last_{}_'.format(period)
gr_period = gr_.iloc[:period]
features = add_features_in_group(features, gr_period, 'link_time',
['max', 'sum', 'mean','min','skew','std'],
period_name)
features = add_features_in_group(features, gr_period, 'link_current_status',
['mean', 'nunique'],
period_name)
return features
# last k link id time trend
def trend_in_last_k_link_id_time(gr, periods):
gr_ = gr.copy()
gr_ = gr_.iloc[::-1]
features = {}
for period in periods:
gr_period = gr_.iloc[:period]
features = add_trend_feature(features, gr_period,
'link_time', 'zsl_{}_period_trend_'.format(period)
)
return features
# trend feature
def add_trend_feature(features, gr, feature_name, prefix):
y = gr[feature_name].values
try:
x = np.arange(0, len(y)).reshape(-1, 1)
lr = LinearRegression()
lr.fit(x, y)
trend = lr.coef_[0]
except:
trend = np.nan
features['{}{}'.format(prefix, feature_name)] = trend
return features
def slice_id_change(x):
hour = x * 5 / 60
hour = np.floor(hour)
hour += 8
if hour >= 24:
hour = hour - 24
return hour
if __name__ == '__main__':
nrows = None
root_path = '../data/giscup_2021/'
read_idkey = np.load(root_path + 'id_key_to_connected_allday.npy', allow_pickle=True).item()
read_grapheb = np.load(root_path + 'graph_embeddings_retp1_directed.npy', allow_pickle=True).item()
read_grapheb_retp = np.load(root_path + 'graph_embeddings_retp05_directed.npy', allow_pickle=True).item()
for i in read_grapheb:
read_grapheb[i] = list(read_grapheb[i]) + list(read_grapheb_retp[i])
del read_grapheb_retp
head_columns = ['order_id', 'ata', 'distance', 'simple_eta', 'driver_id','slice_id']
embedding_k = 256
fill_list = [0] * embedding_k
#######################################nextlinks #######################################
nextlinks = pd.read_csv(root_path+'nextlinks.txt', sep=' ', header=None)
nextlinks.columns=['from_id', 'to_id']
nextlinks['to_id'] = nextlinks['to_id'].astype('str')
nextlinks['to_id'] = nextlinks['to_id'].apply(lambda x: x.split(","))
nextlinks = pd.DataFrame({'from_id':nextlinks.from_id.repeat(nextlinks.to_id.str.len()),
'to_id':np.concatenate(nextlinks.to_id.values)})
from_id_weight = nextlinks['from_id'].value_counts()
from_id_weight = from_id_weight.to_frame()
from_id_weight['index'] = from_id_weight.index
from_id_weight.columns=['weight', 'from_id']
nextlinks = pd.merge(nextlinks,from_id_weight, 'left', on=['from_id'])
nextlinks = nextlinks.sort_values(by='weight',ascending=False)
G = nx.DiGraph()
from_id = nextlinks['from_id'].astype(str).to_list()
to_id = nextlinks['to_id'].to_list()
weight = nextlinks['weight'].to_list()
edge_tuple = list(zip(from_id, to_id,weight))
print('adding')
G.add_weighted_edges_from(edge_tuple)
dc = nx.algorithms.centrality.degree_centrality(G)
dc = sorted(dc.items(), key=lambda d: d[1],reverse=True)
dc = dc[:50000]
dc = [str(i[0]) for i in dc ]
#######################################link #######################################
for name in os.listdir(root_path+'train/'):
data_time = name.split('.')[0]
if data_time=='20200803':
continue
train = pd.read_csv(root_path+'train/{}'.format(name),sep= ';;',header=None,nrows=nrows)
print("开始处理", data_time)
train_head = pd.DataFrame(train[0].str.split(' ').tolist(),columns = ['order_id', 'ata', 'distance','simple_eta', 'driver_id', 'slice_id'])
train_head['order_id'] = train_head['order_id'].astype(str)
train_head['ata'] = train_head['ata'].astype(float)
train_head['distance'] = train_head['distance'].astype(float)
train_head['simple_eta'] = train_head['simple_eta'].astype(float)
train_head['driver_id'] = train_head['driver_id'].astype(int)
train_head['slice_id'] = train_head['slice_id'].astype(int)
#link preprocess
data_link = train[[1]]
data_link['index'] = train_head.index
data_link['order_id'] = train_head['order_id']
data_link['ata'] = train_head['ata']
data_link['distance'] = train_head['distance']
data_link['simple_eta'] = train_head['simple_eta']
data_link['slice_id'] = train_head['slice_id']
# data_link['slice_id'] = data_link['slice_id'].apply(slice_id_change)
gc.collect()
data_link_split = data_link[1].str.split(' ', expand=True).stack().to_frame()
data_link_split = data_link_split.reset_index(level=1, drop=True).rename(columns={0: 'link_info'})
# data_link_split = data_link_split.reset_index(drop=True)
data_link_split = data_link[['order_id', 'index', 'ata', 'distance', 'simple_eta', 'slice_id']].join(
data_link_split)
data_link_split = data_link_split.reset_index(drop=True)
data_link_split[['link_id',
'link_time',
'link_ratio',
'link_current_status',
'link_arrival_status']] = data_link_split['link_info'].str.split(':|,', 5, expand=True)
data_link_split = data_link_split.drop(['link_info'], axis=1)
data_link_split['link_ratio'] = data_link_split['link_ratio'].astype(float)
data_link_split['link_time'] = data_link_split['link_time'].astype(float)
data_link_split['link_current_status'] = data_link_split['link_current_status'].astype(int)
print('preprocess finish!')
print('start feature engineering')
feature = train_head[['order_id', 'distance']]
###################static fea#############################################
#######################order link id count###############################
df = data_link_split.groupby('order_id', as_index=False)
tmp_linkid_agg = df['link_id'].agg({'zsl_order_link_id_count': 'count'})
tmp_linkid_agg['zsl_order_link_id_count_bins'] = 0
tmp_linkid_agg.loc[(tmp_linkid_agg['zsl_order_link_id_count']>=75)&(tmp_linkid_agg['zsl_order_link_id_count']<100),'zsl_order_link_id_count_bins']=1
tmp_linkid_agg.loc[(tmp_linkid_agg['zsl_order_link_id_count']>=100)&(tmp_linkid_agg['zsl_order_link_id_count']<120),'zsl_order_link_id_count_bins']=2
tmp_linkid_agg.loc[(tmp_linkid_agg['zsl_order_link_id_count']>=120),'zsl_order_link_id_count_bins']=3
feature = feature.merge(tmp_linkid_agg,on='order_id',how='left')
print('order link id count finish!')
#######################order link id & distance###############################
feature['zsl_order_is_highspeed'] = 0
feature.loc[(feature['distance']>90000)&(feature['zsl_order_link_id_count']<300),'zsl_order_is_highspeed'] = 1
print('order link id & distance finish!')
#######################order link id & nextlinks centry###############################
tmp = data_link_split[data_link_split['link_id'].isin(dc)]
tmp = tmp.groupby('order_id', as_index=False)
tmp_linkid_centry_count = tmp['link_id'].agg({'zsl_order_link_id_centry_count': 'count'})
feature = feature.merge(tmp_linkid_centry_count,on='order_id',how='left')
feature['zsl_order_link_id_centry_count'] = feature['zsl_order_link_id_centry_count'].fillna(0)
print('order link id & nextlinks centry finish!')
#######################order link time sum mean max min var std###############################
tmp_linktime_agg = df['link_time'].agg({'zsl_order_link_time_sum': 'sum','zsl_order_link_time_mean': 'mean',
'zsl_order_link_time_max': 'max','zsl_order_link_time_min': 'min',
'zsl_order_link_time_var': 'var','zsl_order_link_time_skew': 'skew'})
feature = feature.merge(tmp_linktime_agg,on='order_id',how='left')
print('order link time sum mean max min var std finish!')
#######################order link current status mean nunique###############################
tmp_linktime_agg = df['link_current_status'].agg({'zsl_link_current_status_mean': 'mean', 'zsl_link_current_status_nunique': 'nunique'})
feature = feature.merge(tmp_linktime_agg, on='order_id', how='left')
print('order link current status mean nunique finish!')
#######################order link current status count vector###############################
data_link_split['link_current_status'] = data_link_split['link_current_status'].astype(str)
data_link_split.loc[data_link_split['link_current_status'].astype(int)<0,'link_current_status'] = '0'
data_link_split.loc[data_link_split['link_current_status'].astype(int)>3,'link_current_status'] = '3'
data = data_link_split.groupby('order_id')['link_current_status'].apply(lambda x: x.str.cat(sep=',')).reset_index()
cv_encode = CountVectorizer(token_pattern=u'(?u)\\b\\w+\\b')
train_x = cv_encode.fit_transform(data['link_current_status'])
train_x = train_x.toarray()
link_current_status = pd.DataFrame(train_x, columns=['zsl_link_current_status0', 'zsl_link_current_status1', 'zsl_link_current_status2',
'zsl_link_current_status3'])
data = pd.concat([data[['order_id']],link_current_status],axis=1)
feature = feature.merge(data, on='order_id', how='left')
print('order link current status count vector finish!')
#######################order distance/link_id_count###############################
feature['zsl_distance_div_link_id_count'] = feature['distance']*10/feature['zsl_order_link_id_count']
feature = feature.drop('distance', axis=1)
print('order distance div link_id_count finish!')
#######################order link ratio sum mean max min var std###############################
tmp_linkratio_agg = df['link_ratio'].agg({'zsl_order_link_ratio_sum': 'sum', 'zsl_order_link_ratio_mean': 'mean',
'zsl_order_link_ratio_min': 'min',
'zsl_order_link_ratio_var': 'var', 'zsl_order_link_ratio_skew': 'skew'})
feature = feature.merge(tmp_linkratio_agg, on='order_id', how='left')
print('order link ratio sum mean max min var std finish!')
#######################weather###################################################################
weather = pd.read_csv(root_path+'weather.csv')
weather_dict={'rainstorm':0,'heavy rain':1,'moderate rain':2,'cloudy':3,
'showers':4}
weather['weather'] = weather['weather'].map(weather_dict)
weather['date'] = weather['date'].astype(str)
weather=weather[weather['date']==data_time]
feature['weather'] = weather['weather'].values[0]
feature['hightemp'] = weather['hightemp'].values[0]
feature['lowtemp'] = weather['lowtemp'].values[0]
print('weather finish!')
###################trend fea#############################################
###################trend link time#####################################
data_link_split['link_current_status'] = data_link_split['link_current_status'].astype(int)
groupby = data_link_split.groupby(['order_id'])
func = partial(trend_in_last_k_link_id_time, periods=[2, 5, 7, 10, 15, 20, 30, 50, 80, 100, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=20, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
func = partial(last_link_time_features, periods=[2, 5, 7, 10, 15, 20, 30, 50, 80, 100, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=20, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
func = partial(last_k_link_time_interval, periods=[2, 5, 7, 10, 15, 20, 30, 50, 80, 100, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=20, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
print('trend link time finish!')
####################nextlinks graph embedding#######################
data_link_split['link_id'] = data_link_split['link_id'].astype(int)
data_link_split['link_id'] = data_link_split['link_id'].map(read_idkey)
data_link_split['link_id'] = data_link_split['link_id'].fillna(0)
data_link_split['link_id'] = data_link_split['link_id'].astype(int)
data_link_split['link_id'] = data_link_split['link_id'].map(read_grapheb)
data_link_split['link_id'] = data_link_split['link_id'].fillna('0')
def replace_list(x):
if isinstance(x, str):
x = fill_list
return x
data_link_split['link_id'] = data_link_split['link_id'].apply(replace_list)
link_id_col = ['zsl_link_id_eb{}'.format(i) for i in range(embedding_k)]
agg_col = dict(zip(link_id_col, ['mean'] * len(link_id_col)))
link_id_array = np.array(data_link_split.pop('link_id').to_list())
link_id_array = pd.DataFrame(link_id_array, columns=agg_col, dtype=np.float16)
data_link_split = pd.concat([data_link_split, link_id_array], axis=1)
tmp = data_link_split.groupby('order_id', as_index=False)
tmp_linkid_agg = tmp.agg(agg_col)
feature = feature.merge(tmp_linkid_agg, on='order_id', how='left')
feature.to_csv(root_path + 'feature/train/link_fea_order_id_level_{}.csv'.format(data_time), index=False)
del train
gc.collect()
test = pd.read_csv(root_path+'20200901_test.txt',sep= ';;',header=None,nrows=nrows)
test_head = pd.DataFrame(test[0].str.split(' ').tolist(),columns = ['order_id', 'ata', 'distance','simple_eta', 'driver_id', 'slice_id'])
test_head['order_id'] = test_head['order_id'].astype(str)
test_head['ata'] = test_head['ata'].astype(float)
test_head['distance'] = test_head['distance'].astype(float)
test_head['simple_eta'] = test_head['simple_eta'].astype(float)
test_head['driver_id'] = test_head['driver_id'].astype(int)
test_head['slice_id'] = test_head['slice_id'].astype(int)
# link preprocess
data_link = test[[1]]
data_link['index'] = test_head.index
data_link['order_id'] = test_head['order_id']
data_link['ata'] = test_head['ata']
data_link['distance'] = test_head['distance']
data_link['simple_eta'] = test_head['simple_eta']
data_link['slice_id'] = test_head['slice_id']
# data_link['slice_id'] = data_link['slice_id'].apply(slice_id_change)
gc.collect()
data_link_split = data_link[1].str.split(' ', expand=True).stack().to_frame()
data_link_split = data_link_split.reset_index(level=1, drop=True).rename(columns={0: 'link_info'})
# data_link_split = data_link_split.reset_index(drop=True)
data_link_split = data_link[['order_id', 'index', 'ata', 'distance', 'simple_eta', 'slice_id']].join(
data_link_split)
data_link_split = data_link_split.reset_index(drop=True)
data_link_split[['link_id',
'link_time',
'link_ratio',
'link_current_status',
'link_arrival_status']] = data_link_split['link_info'].str.split(':|,', 5, expand=True)
data_link_split = data_link_split.drop(['link_info'], axis=1)
data_link_split['link_ratio'] = data_link_split['link_ratio'].astype(float)
data_link_split['link_time'] = data_link_split['link_time'].astype(float)
data_link_split['link_current_status'] = data_link_split['link_current_status'].astype(int)
print('preprocess finish!')
print('start feature engineering')
feature = test_head[['order_id', 'distance']]
###################static fea#############################################
#######################order link id count###############################
df = data_link_split.groupby('order_id', as_index=False)
tmp_linkid_agg = df['link_id'].agg({'zsl_order_link_id_count': 'count'})
tmp_linkid_agg['zsl_order_link_id_count_bins'] = 0
tmp_linkid_agg.loc[(tmp_linkid_agg['zsl_order_link_id_count'] >= 75) & (
tmp_linkid_agg['zsl_order_link_id_count'] < 100), 'zsl_order_link_id_count_bins'] = 1
tmp_linkid_agg.loc[(tmp_linkid_agg['zsl_order_link_id_count'] >= 100) & (
tmp_linkid_agg['zsl_order_link_id_count'] < 120), 'zsl_order_link_id_count_bins'] = 2
tmp_linkid_agg.loc[(tmp_linkid_agg['zsl_order_link_id_count'] >= 120), 'zsl_order_link_id_count_bins'] = 3
feature = feature.merge(tmp_linkid_agg, on='order_id', how='left')
print('order link id count finish!')
#######################order link id & distance###############################
feature['zsl_order_is_highspeed'] = 0
feature.loc[
(feature['distance'] > 90000) & (feature['zsl_order_link_id_count'] < 300), 'zsl_order_is_highspeed'] = 1
print('order link id & distance finish!')
#######################order link id & nextlinks centry###############################
tmp = data_link_split[data_link_split['link_id'].isin(dc)]
tmp = tmp.groupby('order_id', as_index=False)
tmp_linkid_centry_count = tmp['link_id'].agg({'zsl_order_link_id_centry_count': 'count'})
feature = feature.merge(tmp_linkid_centry_count, on='order_id', how='left')
feature['zsl_order_link_id_centry_count'] = feature['zsl_order_link_id_centry_count'].fillna(0)
print('order link id & nextlinks centry finish!')
#######################order link time sum mean max min var std###############################
tmp_linktime_agg = df['link_time'].agg({'zsl_order_link_time_sum': 'sum', 'zsl_order_link_time_mean': 'mean',
'zsl_order_link_time_max': 'max', 'zsl_order_link_time_min': 'min',
'zsl_order_link_time_var': 'var', 'zsl_order_link_time_skew': 'skew'})
feature = feature.merge(tmp_linktime_agg, on='order_id', how='left')
print('order link time sum mean max min var std finish!')
#######################order link current status mean nunique###############################
tmp_linktime_agg = df['link_current_status'].agg(
{'zsl_link_current_status_mean': 'mean', 'zsl_link_current_status_nunique': 'nunique'})
feature = feature.merge(tmp_linktime_agg, on='order_id', how='left')
print('order link current status mean nunique finish!')
#######################order link current status count vector###############################
data_link_split['link_current_status'] = data_link_split['link_current_status'].astype(str)
data_link_split.loc[data_link_split['link_current_status'].astype(int) < 0, 'link_current_status'] = '0'
data_link_split.loc[data_link_split['link_current_status'].astype(int) > 3, 'link_current_status'] = '3'
data = data_link_split.groupby('order_id')['link_current_status'].apply(lambda x: x.str.cat(sep=',')).reset_index()
cv_encode = CountVectorizer(token_pattern=u'(?u)\\b\\w+\\b')
test_x = cv_encode.fit_transform(data['link_current_status'])
test_x = test_x.toarray()
link_current_status = pd.DataFrame(test_x, columns=['zsl_link_current_status0', 'zsl_link_current_status1',
'zsl_link_current_status2',
'zsl_link_current_status3'])
data = pd.concat([data[['order_id']], link_current_status], axis=1)
feature = feature.merge(data, on='order_id', how='left')
print('order link current status count vector finish!')
#######################order distance/link_id_count###############################
feature['zsl_distance_div_link_id_count'] = feature['distance'] * 10 / feature['zsl_order_link_id_count']
feature = feature.drop('distance', axis=1)
print('order distance div link_id_count finish!')
#######################order link ratio sum mean max min var std###############################
tmp_linkratio_agg = df['link_ratio'].agg({'zsl_order_link_ratio_sum': 'sum', 'zsl_order_link_ratio_mean': 'mean',
'zsl_order_link_ratio_min': 'min',
'zsl_order_link_ratio_var': 'var', 'zsl_order_link_ratio_skew': 'skew'})
feature = feature.merge(tmp_linkratio_agg, on='order_id', how='left')
print('order link ratio sum mean max min var std finish!')
#######################weather###################################################################
weather = pd.read_csv(root_path + 'weather.csv')
weather_dict = {'rainstorm': 0, 'heavy rain': 1, 'moderate rain': 2, 'cloudy': 3,
'showers': 4}
weather['weather'] = weather['weather'].map(weather_dict)
weather['date'] = weather['date'].astype(str)
weather = weather[weather['date'] == data_time]
feature['weather'] = weather['weather'].values[0]
feature['hightemp'] = weather['hightemp'].values[0]
feature['lowtemp'] = weather['lowtemp'].values[0]
print('weather finish!')
###################trend fea#############################################
###################trend link time#####################################
data_link_split['link_current_status'] = data_link_split['link_current_status'].astype(int)
groupby = data_link_split.groupby(['order_id'])
func = partial(trend_in_last_k_link_id_time, periods=[2, 5, 7, 10, 15, 20, 30, 50, 80, 100, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=20, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
func = partial(last_link_time_features, periods=[2, 5, 7, 10, 15, 20, 30, 50, 80, 100, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=20, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
func = partial(last_k_link_time_interval, periods=[2, 5, 7, 10, 15, 20, 30, 50, 80, 100, 100000000])
g = parallel_apply_fea(groupby, func, index_name='order_id', num_workers=20, chunk_size=10000)
feature = feature.merge(g, on='order_id', how='left')
print('trend link time finish!')
####################nextlinks graph embedding#######################
data_link_split['link_id'] = data_link_split['link_id'].astype(int)
data_link_split['link_id'] = data_link_split['link_id'].map(read_idkey)
data_link_split['link_id'] = data_link_split['link_id'].fillna(0)
data_link_split['link_id'] = data_link_split['link_id'].astype(int)
data_link_split['link_id'] = data_link_split['link_id'].map(read_grapheb)
data_link_split['link_id'] = data_link_split['link_id'].fillna('0')
def replace_list(x):
if isinstance(x, str):
x = fill_list
return x
data_link_split['link_id'] = data_link_split['link_id'].apply(replace_list)
link_id_col = ['zsl_link_id_eb{}'.format(i) for i in range(embedding_k)]
agg_col = dict(zip(link_id_col, ['mean'] * len(link_id_col)))
link_id_array = np.array(data_link_split.pop('link_id').to_list())
link_id_array = pd.DataFrame(link_id_array, columns=agg_col, dtype=np.float16)
data_link_split = pd.concat([data_link_split, link_id_array], axis=1)
tmp = data_link_split.groupby('order_id', as_index=False)
tmp_linkid_agg = tmp.agg(agg_col)
feature = feature.merge(tmp_linkid_agg, on='order_id', how='left')
feature.to_csv(root_path+'feature/test/link_fea_order_id_level_20200901.csv',index=False)

207
机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/LGB_13700/4_single_model.py
Unescape View File

@ -0,0 +1,207 @@
#coding=utf-8
"""
Author: Aigege
Code: https://github.com/AiIsBetter
"""
# date 2021.08.01
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
import lightgbm as lgb
from utils import reduce_mem_usage,reduce_mem_usage_parallel
import os
import gc
import warnings
import time
warnings.filterwarnings("ignore")
def slice_id_change(x):
hour = x * 5 / 60
hour = np.floor(hour)
hour += 8
if hour >= 24:
hour = hour - 24
return hour
# 评估指标
def MAPE(true, pred):
diff = np.abs(np.array(pred) - np.array(true))
return np.mean(diff / true)
# 自定义lgb评估指标
def lgb_score_mape(train_data,preds):
labels = train_data
diff = np.abs(np.array(preds) - np.array(labels))
result = np.mean(diff / labels)
return 'mape',result, False
head_columns = ['order_id', 'ata', 'distance', 'simple_eta', 'driver_id','slice_id']
result = []
result_time_weight = []
result_dis_weight = []
count = 0
df = []
nrows=None
root_path = '../data/giscup_2021/'
data_list = ['20200818', '20200819', '20200820', '20200821', '20200822', '20200823', '20200824',
'20200825', '20200826', '20200827', '20200828', '20200829', '20200830', '20200831']
#######################################本地验证#######################################
for name in os.listdir(root_path+'train/'):
data_time = name.split('.')[0]
if data_time not in data_list:
continue
train = pd.read_csv(root_path+'train/{}'.format(name),sep= ';;',header=None,nrows=nrows)
feature_cross = pd.read_csv(root_path+'feature/train/cross_fea_order_id_level_{}.csv'.format(data_time),nrows=nrows)
feature_link = pd.read_csv(root_path+'feature/train/link_fea_order_id_level_{}.csv'.format(data_time),nrows=nrows)
feature_head = pd.read_csv(root_path+'feature/train/head_link_{}.csv'.format(data_time),nrows=nrows)
feature_sqe = pd.read_csv(root_path + 'feature/train/{}.csv'.format(data_time),nrows=nrows)
feature_cross['order_id'] = feature_cross['order_id'].astype(str)
feature_link['order_id'] = feature_link['order_id'].astype(str)
feature_head['order_id'] = feature_head['order_id'].astype(str)
feature_sqe['order_id'] = feature_sqe['order_id'].astype(str)
print("开始处理", data_time)
# train.columns = ['head','link','cross']
# train['head'] = train['head'].apply(lambda x:x.split(' '))
train_head = pd.DataFrame(train[0].str.split(' ').tolist(),columns = ['order_id', 'ata', 'distance','simple_eta', 'driver_id', 'slice_id'])
train_head['order_id'] = train_head['order_id'].astype(str)
train_head['ata'] = train_head['ata'].astype(float)
train_head['distance'] = train_head['distance'].astype(float)
train_head['simple_eta'] = train_head['simple_eta'].astype(float)
train_head['driver_id'] = train_head['driver_id'].astype(int)
train_head['slice_id'] = train_head['slice_id'].astype(int)
train_head['date_time'] = int(data_time)
train_head = train_head.merge(feature_cross,on='order_id',how='left')
train_head = train_head.merge(feature_link,on='order_id',how='left')
feature_head = feature_head.drop(['ata', 'distance', 'simple_eta', 'driver_id', 'slice_id', 'index',
'date_time', 'link_count', 'link_time_sum', 'link_ratio_sum',
'date_time_dt', 'weekday', 'hour', 'weather', 'hightemp', 'lowtemp',
'len_tmp',
'link_time_mean', 'link_time_std'],
axis=1)
feature_sqe = feature_sqe.drop(['pre_arrival_status', 'arrive_slice_id', 'slice_id'], axis=1)
train_head = train_head.merge(feature_sqe, on='order_id', how='left')
train_head = train_head.merge(feature_head, on='order_id', how='left')
print('merge finish!')
train_head = reduce_mem_usage_parallel(train_head,28)
df.append(train_head.drop('order_id',axis=1))
del train
gc.collect()
count +=1
df = pd.concat(df,axis=0)
test = pd.read_csv(root_path+'20200901_test.txt',sep= ';;',header=None,nrows=nrows)
test_head = pd.DataFrame(test[0].str.split(' ').tolist(),columns = ['order_id', 'ata', 'distance','simple_eta', 'driver_id', 'slice_id'])
test_head['order_id'] = test_head['order_id'].astype(str)
test_head['ata'] = test_head['ata'].astype(float)
test_head['distance'] = test_head['distance'].astype(float)
test_head['simple_eta'] = test_head['simple_eta'].astype(float)
test_head['driver_id'] = test_head['driver_id'].astype(int)
test_head['slice_id'] = test_head['slice_id'].astype(int)
feature_cross = pd.read_csv(root_path + 'feature/test/cross_fea_order_id_level_{}.csv'.format('20200901'),nrows=nrows)
feature_link = pd.read_csv(root_path + 'feature/test/link_fea_order_id_level_{}.csv'.format('20200901'), nrows=nrows)
feature_head = pd.read_csv(root_path + 'feature/test/head_link_{}.csv'.format('20200901'),nrows=nrows)
feature_sqe = pd.read_csv(root_path + 'feature/test/{}.csv'.format('20200901'),nrows=nrows)
test_head['date_time'] = 20200901
feature_cross['order_id'] = feature_cross['order_id'].astype(str)
feature_link['order_id'] = feature_link['order_id'].astype(str)
feature_head['order_id'] = feature_head['order_id'].astype(str)
feature_sqe['order_id'] = feature_sqe['order_id'].astype(str)
test_head = test_head.merge(feature_cross, on='order_id', how='left')
test_head = test_head.merge(feature_link,on='order_id',how='left')
feature_head = feature_head.drop(['ata', 'distance', 'simple_eta', 'driver_id', 'slice_id', 'index',
'date_time', 'link_count', 'link_time_sum', 'link_ratio_sum',
'date_time_dt', 'weekday', 'hour', 'weather', 'hightemp', 'lowtemp',
'len_tmp',
'link_time_mean', 'link_time_std'],
axis=1)
feature_sqe = feature_sqe.drop(['pre_arrival_status', 'arrive_slice_id', 'slice_id'], axis=1)
test_head = test_head.merge(feature_sqe, on='order_id', how='left')
test_head = test_head.merge(feature_head, on='order_id', how='left')
test_head = reduce_mem_usage_parallel(test_head,28)
del feature_cross,feature_link
gc.collect()
X_train = df.drop('ata',axis=1)
y_train = df['ata']
X_test = test_head.drop(['order_id','ata'],axis=1)
folds = 5
skf = KFold(n_splits=folds, shuffle=True, random_state=2021)
train_mean = np.zeros(shape=[1,folds])
test_predict = np.zeros(shape=[X_test.shape[0], folds],dtype=float)
k_fold_mape = []
feature_importance_df = pd.DataFrame()
# Display/plot feature importance
def display_importances(feature_importance_df_):
feature_importance_df_.to_csv('feature_importances.csv',index=False)
cols = feature_importance_df_[["feature", "importance"]].groupby("feature").mean().sort_values(by="importance", ascending=False)[:100].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
best_features = best_features.groupby('feature',as_index = False)['importance'].mean()
best_features = best_features.sort_values(by = 'importance',ascending=False)
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features (avg over folds)')
plt.tight_layout()
plt.savefig('feature_importances.jpg')
# plt.show()
scores = 0
threshold = 0
print('start training......')
print('训练集维度：',X_train.shape)
print('测试集维度：',X_test.shape)
for i, (trn_idx, val_idx) in enumerate(skf.split(X_train, y_train)):
clf = lgb.LGBMRegressor(
boosting_type='gbdt',
objective='regression',
n_estimators=10000,
learning_rate=0.1,
num_leaves=170,
max_bin=63,
max_depth=-1,
random_state = 2021,
subsample_for_bin=200000,
feature_fraction=0.84,
bagging_fraction=0.86,
bagging_freq=7,
min_child_samples=89,
lambda_l1=0.006237830242067111,
lambda_l2=2.016472023736186e-05,
metric=None,
n_jobs = 30,
# device='gpu'
)
clf.fit(X_train.iloc[trn_idx], y_train.iloc[trn_idx], eval_set=[(X_train.iloc[trn_idx], y_train.iloc[trn_idx])
, (X_train.iloc[val_idx], y_train.iloc[val_idx])],
eval_metric=lambda y_true, y_pred:[lgb_score_mape(y_true, y_pred)],
verbose=100, early_stopping_rounds=100)
fold_importance_df = pd.DataFrame()
fold_importance_df["feature"] = X_train.columns
fold_importance_df["importance"] = clf.feature_importances_
fold_importance_df["fold"] = i + 1
feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
print('predicting')
val_predict = clf.predict(X_train.iloc[val_idx], num_iteration=clf.best_iteration_)
test_predict[:,i] = clf.predict(X_test, num_iteration=clf.best_iteration_)
k_fold_mape.append(MAPE(y_train.iloc[val_idx],val_predict))
print("kfold_{}_mape_score:{} ".format(i, k_fold_mape[i]))
print('Train set kfold {} mean mape:'.format(i), np.mean(k_fold_mape))
display_importances(feature_importance_df)
test_head['result'] = np.mean(test_predict,axis=1)
test_head['id'] = test_head['order_id']
test_head[['id','result']].to_csv('submission.csv',index=False)

301
机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/LGB_13700/5_model_final.py
Unescape View File

@ -0,0 +1,301 @@
#coding=utf-8
"""
Author: Aigege
Code: https://github.com/AiIsBetter
"""
# date 2021.08.01
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.model_selection import KFold
import lightgbm as lgb
from utils import reduce_mem_usage,reduce_mem_usage_parallel,lgb_score_mape,MAPE
import gc
import warnings
import os,random,pickle
import optuna
warnings.filterwarnings("ignore")
def slice_id_change(x):
hour = x * 5 / 60
hour = np.floor(hour)
hour += 8
if hour >= 24:
hour = hour - 24
return hour
def optuna_print(tr_x, tr_y, te_x,te_y):
def objective(trial,tr_x, tr_y, te_x,te_y):
dtrain = lgb.Dataset(tr_x, label=tr_y)
dvalid = lgb.Dataset(te_x, label=te_y)
param = {
"objective": "regression",
"metric": "mape",
"verbosity": -1,
"boosting_type": "gbdt",
'min_split_gain': 0,
'random_state':2021,
'max_bin':trial.suggest_int('max_bin',63,250),
'subsample_for_bin': trial.suggest_int('subsample_for_bin', 40000, 300000),
"lambda_l1": trial.suggest_loguniform("lambda_l1", 1e-8, 10.0),
"lambda_l2": trial.suggest_loguniform("lambda_l2", 1e-8, 10.0),
"num_leaves": trial.suggest_int("num_leaves", 2, 256),
"feature_fraction": trial.suggest_uniform("feature_fraction", 0.4, 1.0),
"bagging_fraction": trial.suggest_uniform("bagging_fraction", 0.4, 1.0),
"bagging_freq": trial.suggest_int("bagging_freq", 1, 7),
"min_child_samples": trial.suggest_int("min_child_samples", 5, 100),
}
# Add a callback for pruning.
pruning_callback = optuna.integration.LightGBMPruningCallback(trial, "mape")
gbm = lgb.train(
param, dtrain, valid_sets=[dvalid], verbose_eval=False, callbacks=[pruning_callback]
)
preds = gbm.predict(te_x)
pred_labels = np.rint(preds)
mape = MAPE(te_y, pred_labels)
return mape
study = optuna.create_study(
pruner=optuna.pruners.MedianPruner(n_warmup_steps=10), direction="minimize"
)
study.optimize(lambda trial: objective(trial, tr_x, tr_y, te_x, te_y),
n_trials=100)
print("Number of finished trials: {}".format(len(study.trials)))
print("Best trial:")
trial = study.best_trial
print(" Value: {}".format(trial.value))
print(" Params: ")
for key, value in trial.params.items():
print(" {}: {}".format(key, value))
head_columns = ['order_id', 'ata', 'distance', 'simple_eta', 'driver_id','slice_id']
result = []
result_time_weight = []
result_dis_weight = []
count = 0
df = []
nrows=None
root_path = '../data/giscup_2021/'
data_list = ['20200818', '20200819', '20200820', '20200821', '20200822', '20200823', '20200824',
'20200825', '20200826', '20200827', '20200828', '20200829', '20200830', '20200831']
for name in os.listdir(root_path+'train/'):
data_time = name.split('.')[0]
if data_time not in data_list:
continue
train = pd.read_csv(root_path+'train/{}'.format(name),sep= ';;',header=None,nrows=nrows)
feature_cross = pd.read_csv(root_path+'feature/train/cross_fea_order_id_level_{}.csv'.format(data_time),nrows=nrows)
feature_link = pd.read_csv(root_path+'feature/train/link_fea_order_id_level_{}.csv'.format(data_time),nrows=nrows)
feature_head = pd.read_csv(root_path+'feature/train/head_link_{}.csv'.format(data_time),nrows=nrows)
feature_sqe = pd.read_csv(root_path + 'feature/train/{}.csv'.format(data_time),nrows=nrows)
feature_cross['order_id'] = feature_cross['order_id'].astype(str)
feature_link['order_id'] = feature_link['order_id'].astype(str)
feature_head['order_id'] = feature_head['order_id'].astype(str)
feature_sqe['order_id'] = feature_sqe['order_id'].astype(str)
print("开始处理", data_time)
# train.columns = ['head','link','cross']
# train['head'] = train['head'].apply(lambda x:x.split(' '))
train_head = pd.DataFrame(train[0].str.split(' ').tolist(),columns = ['order_id', 'ata', 'distance','simple_eta', 'driver_id', 'slice_id'])
train_head['order_id'] = train_head['order_id'].astype(str)
train_head['ata'] = train_head['ata'].astype(float)
train_head['distance'] = train_head['distance'].astype(float)
train_head['simple_eta'] = train_head['simple_eta'].astype(float)
train_head['driver_id'] = train_head['driver_id'].astype(int)
train_head['slice_id'] = train_head['slice_id'].astype(int)
train_head['date_time'] = int(data_time)
train_head = train_head.merge(feature_cross,on='order_id',how='left')
train_head = train_head.merge(feature_link,on='order_id',how='left')
feature_head = feature_head.drop(['ata', 'distance', 'simple_eta', 'driver_id', 'slice_id', 'index',
'date_time', 'link_count', 'link_time_sum', 'link_ratio_sum',
'date_time_dt', 'weekday', 'hour', 'weather', 'hightemp', 'lowtemp',
'len_tmp',
'link_time_mean', 'link_time_std'],
axis=1)
feature_sqe = feature_sqe.drop(['pre_arrival_status', 'arrive_slice_id', 'slice_id'], axis=1)
train_head = train_head.merge(feature_sqe, on='order_id', how='left')
train_head = train_head.merge(feature_head, on='order_id', how='left')
print('merge finish!')
train_head = reduce_mem_usage_parallel(train_head,28)
df.append(train_head.drop('order_id',axis=1))
del train
gc.collect()
count +=1
df = pd.concat(df,axis=0)
test = pd.read_csv(root_path+'20200901_test.txt',sep= ';;',header=None,nrows=nrows)
test_head = pd.DataFrame(test[0].str.split(' ').tolist(),columns = ['order_id', 'ata', 'distance','simple_eta', 'driver_id', 'slice_id'])
test_head['order_id'] = test_head['order_id'].astype(str)
test_head['ata'] = test_head['ata'].astype(float)
test_head['distance'] = test_head['distance'].astype(float)
test_head['simple_eta'] = test_head['simple_eta'].astype(float)
test_head['driver_id'] = test_head['driver_id'].astype(int)
test_head['slice_id'] = test_head['slice_id'].astype(int)
feature_cross = pd.read_csv(root_path + 'feature/test/cross_fea_order_id_level_{}.csv'.format('20200901'),nrows=nrows)
feature_link = pd.read_csv(root_path + 'feature/test/link_fea_order_id_level_{}.csv'.format('20200901'), nrows=nrows)
feature_head = pd.read_csv(root_path + 'feature/test/head_link_{}.csv'.format('20200901'),nrows=nrows)
feature_sqe = pd.read_csv(root_path + 'feature/test/{}.csv'.format('20200901'),nrows=nrows)
test_head['date_time'] = 20200901
feature_cross['order_id'] = feature_cross['order_id'].astype(str)
feature_link['order_id'] = feature_link['order_id'].astype(str)
feature_head['order_id'] = feature_head['order_id'].astype(str)
feature_sqe['order_id'] = feature_sqe['order_id'].astype(str)
test_head = test_head.merge(feature_cross, on='order_id', how='left')
test_head = test_head.merge(feature_link,on='order_id',how='left')
feature_head = feature_head.drop(['ata', 'distance', 'simple_eta', 'driver_id', 'slice_id', 'index',
'date_time', 'link_count', 'link_time_sum', 'link_ratio_sum',
'date_time_dt', 'weekday', 'hour', 'weather', 'hightemp', 'lowtemp',
'len_tmp',
'link_time_mean', 'link_time_std'],
axis=1)
feature_sqe = feature_sqe.drop(['pre_arrival_status', 'arrive_slice_id', 'slice_id'], axis=1)
test_head = test_head.merge(feature_sqe, on='order_id', how='left')
test_head = test_head.merge(feature_head, on='order_id', how='left')
test_head = reduce_mem_usage_parallel(test_head,28)
del feature_cross,feature_link
gc.collect()
X_train = df.drop('ata',axis=1)
y_train = df['ata']
X_test = test_head.drop(['order_id','ata'],axis=1)
#调参
#tr_x, te_x,tr_y,te_y = train_test_split(X_train,y_train,test_size=0.2,random_state=2021)
#optuna_print(tr_x, tr_y, te_x,te_y)
#del tr_x, te_x,tr_y,te_y
#gc.collect()
folds = 5
skf = KFold(n_splits=folds, shuffle=True, random_state=2021)
train_mean = np.zeros(shape=[1,folds])
test_predict = np.zeros(shape=[X_test.shape[0], folds],dtype=float)
k_fold_mape = []
feature_importance_df = pd.DataFrame()
# Display/plot feature importance
def display_importances(feature_importance_df_):
feature_importance_df_.to_csv('feature_importances.csv',index=False)
cols = feature_importance_df_[["feature", "importance"]].groupby("feature").mean().sort_values(by="importance", ascending=False)[:100].index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
best_features = best_features.groupby('feature',as_index = False)['importance'].mean()
best_features = best_features.sort_values(by = 'importance',ascending=False)
plt.figure(figsize=(8, 10))
sns.barplot(x="importance", y="feature", data=best_features.sort_values(by="importance", ascending=False))
plt.title('LightGBM Features (avg over folds)')
plt.tight_layout()
plt.savefig('feature_importances.jpg')
# plt.show()
#use single model feature importance as best_feature_importances
feature_importance_df_ = pd.read_csv('best_feature_importances.csv')
cols = feature_importance_df_[["feature", "importance"]].groupby("feature").mean().sort_values(by="importance", ascending=False).index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
best_features = best_features.groupby('feature',as_index = False)['importance'].mean()
best_features = best_features.sort_values(by = 'importance',ascending=False)
data=best_features.sort_values(by="importance", ascending=False)
feature_select = list(data['feature'].values)
feature_cols = feature_select
random_seed = list(range(2021))
max_depth = [4,4,4,4,5,5,5,5,6,6,6,6,7,7,7]
lambd1 = np.arange(0, 1, 0.0001)
lambd2 = np.arange(0, 1, 0.0001)
bagging_fraction = [i / 1000.0 for i in range(700, 800)]
feature_fraction = [i / 1000.0 for i in range(700, 800)]
min_child_weight = [i / 100.0 for i in range(150, 250)]
n_feature = [i / 100.0 for i in range(1, 32,2)]
max_bin = list(range(130, 240))
subsample_for_bin = list(range(50000, 220000,10000))
bagging_freq = [1,2,3,4,5,6,7,8,9,10,1,2,3,4,5]
num_leaves = list(range(130, 250))
random.shuffle(random_seed)
random.shuffle(max_depth)
random.shuffle(lambd1)
random.shuffle(lambd2)
random.shuffle(bagging_fraction)
random.shuffle(feature_fraction)
random.shuffle(min_child_weight)
random.shuffle(max_bin)
random.shuffle(subsample_for_bin)
random.shuffle(bagging_freq)
random.shuffle(num_leaves)
random.shuffle(n_feature)
with open('params.pkl', 'wb') as f:
pickle.dump((random_seed, max_depth, lambd1,lambd2, bagging_fraction, feature_fraction, min_child_weight, max_bin,subsample_for_bin,bagging_freq,num_leaves,n_feature), f)
for iter in range(15):
print('max_depth:',max_depth[iter],'random_seed:',random_seed[iter],'feature_fraction:',feature_fraction[iter],
'bagging_fraction:',bagging_fraction[iter],'min_child_weight:',min_child_weight[iter],
'lambd1:',lambd1[iter],'lambd2:',lambd2[iter],'max_bin:',max_bin[iter],'num_leaves:',num_leaves[iter]
,'subsample_for_bin:',subsample_for_bin[iter],'bagging_freq:',bagging_freq[iter],'n_feature:',n_feature[iter])
nround = 5000
for iter in range(15):
if max_depth[iter]==4:
nround = 10000
elif max_depth[iter]==5:
nround = 8000
elif max_depth[iter]==6:
nround = 6000
elif max_depth[iter] == 7:
nround = 5000
X_train_r = X_train[feature_cols[:int(len(feature_cols)*0.7)]+
feature_cols[int(len(feature_cols)*0.7):int(len(feature_cols)*0.7)+int(len(feature_cols)*n_feature[iter])]]
X_test_r = X_test[feature_cols[:int(len(feature_cols) * 0.7)] +
feature_cols[int(len(feature_cols) * 0.7):int(len(feature_cols) * 0.7) + int(
len(feature_cols) * n_feature[iter])]]
scores = 0
threshold = 0
print('start training......')
print('训练集维度：',X_train_r.shape)
print('测试集维度：',X_test_r.shape)
for i, (trn_idx, val_idx) in enumerate(skf.split(X_train_r, y_train)):
clf = lgb.LGBMRegressor(
boosting_type='gbdt',
objective='regression',
n_estimators=nround,
learning_rate=0.08,
num_leaves=num_leaves[iter],
max_bin=max_bin[iter],
max_depth=max_depth[iter],
random_state=random_seed[iter],
subsample_for_bin=subsample_for_bin[iter],
feature_fraction=feature_fraction[iter],
bagging_fraction=bagging_fraction[iter],
bagging_freq=bagging_freq[iter],
min_child_weight=min_child_weight[iter],
lambda_l1=lambd1[iter],
lambda_l2=lambd2[iter],
metric=None,
n_jobs=30,
device='gpu'
)
clf.fit(X_train_r.iloc[trn_idx], y_train.iloc[trn_idx], eval_set=[(X_train_r.iloc[trn_idx], y_train.iloc[trn_idx]), (X_train_r.iloc[val_idx], y_train.iloc[val_idx])],eval_metric='mape',verbose=100, early_stopping_rounds=200)
print('predicting')
val_predict = clf.predict(X_train_r.iloc[val_idx], num_iteration=clf.best_iteration_)
test_predict[:,i] = clf.predict(X_test_r, num_iteration=clf.best_iteration_)
k_fold_mape.append(MAPE(y_train.iloc[val_idx],val_predict))
print("kfold_{}_mape_score:{} ".format(i, k_fold_mape[i]))
print('Train set kfold {} mean mape:'.format(i), np.mean(k_fold_mape))
#display_importances(feature_importance_df)
test_head['result'] = np.mean(test_predict,axis=1)
test_head['id'] = test_head['order_id']
test_head[['id','result']].to_csv('random_result/submission_{}.csv'.format(iter),index=False)
del X_train_r,X_test_r
gc.collect()
#merge
count = 0
result = 1
for name in os.listdir('random_result/'):
tmp = pd.read_csv('random_result/'+name)
if count == 0:
result = tmp[['id']]
tmp = tmp.rename(columns={'result':'result{}'.format(count)})
result = result.merge(tmp,on='id',how='left')
count += 1
result['result'] = result.drop('id',axis=1).sum(axis=1)
result['result'] = result['result']/count
result[['id','result']].to_csv('submission_merge.csv',index=False)

3
机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/LGB_13700/README.MD
Unescape View File

@ -0,0 +1,3 @@
node2vec
1. Recompile from https://github.com/snap-stanford/snap and add node2vec to system path.
2. To grant executable permission, run: chmod +x ./c_exe/node2vec

BIN
机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/LGB_13700/node2vec
View File

Binary file not shown.

214
机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/LGB_13700/utils.py
Unescape View File

@ -0,0 +1,214 @@
#coding=utf-8
"""
Author: Aigege
Code: https://github.com/AiIsBetter
"""
# date 2021.08.01
import multiprocessing as mp
import numpy as np
import pandas as pd
from tqdm import tqdm
from scipy.stats import kurtosis, iqr, skew
import gc
from sklearn.model_selection import KFold
from sklearn.linear_model import Ridge
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
def chunk_groups(groupby_object, chunk_size):
n_groups = groupby_object.ngroups
group_chunk, index_chunk = [], []
for i, (index, df) in enumerate(groupby_object):
group_chunk.append(df)
index_chunk.append(index)
if (i + 1) % chunk_size == 0 or i + 1 == n_groups:
group_chunk_, index_chunk_ = group_chunk.copy(), index_chunk.copy()
group_chunk, index_chunk = [], []
yield index_chunk_, group_chunk_
def parallel_apply(groups, func, index_name='Index', num_workers=1, chunk_size=100000):
n_chunks = np.ceil(1.0 * groups.ngroups / chunk_size)
indeces, features = [], []
for index_chunk, groups_chunk in tqdm(chunk_groups(groups, chunk_size), total=n_chunks):
with mp.pool.Pool(num_workers) as executor:
features_chunk = executor.map(func, groups_chunk)
for i in features_chunk:
features.append(i)
return features
def parallel_apply_fea(groups, func, index_name='Index', num_workers=1, chunk_size=100000):
n_chunks = np.ceil(1.0 * groups.ngroups / chunk_size)
indeces, features = [], []
for index_chunk, groups_chunk in chunk_groups(groups, chunk_size):
with mp.pool.Pool(num_workers) as executor:
features_chunk = executor.map(func, groups_chunk)
features.extend(features_chunk)
indeces.extend(index_chunk)
features = pd.DataFrame(features)
features.index = indeces
features.index.name = index_name
return features
def add_features_in_group(features, gr_, feature_name, aggs, prefix):
for agg in aggs:
if agg == 'sum':
features['{}{}_sum'.format(prefix, feature_name)] = gr_[feature_name].sum()
elif agg == 'mean':
features['{}{}_mean'.format(prefix, feature_name)] = gr_[feature_name].mean()
elif agg == 'max':
features['{}{}_max'.format(prefix, feature_name)] = gr_[feature_name].max()
elif agg == 'min':
features['{}{}_min'.format(prefix, feature_name)] = gr_[feature_name].min()
elif agg == 'std':
features['{}{}_std'.format(prefix, feature_name)] = gr_[feature_name].std()
elif agg == 'count':
features['{}{}_count'.format(prefix, feature_name)] = gr_[feature_name].count()
elif agg == 'skew':
features['{}{}_skew'.format(prefix, feature_name)] = skew(gr_[feature_name])
elif agg == 'kurt':
features['{}{}_kurt'.format(prefix, feature_name)] = kurtosis(gr_[feature_name])
elif agg == 'iqr':
features['{}{}_iqr'.format(prefix, feature_name)] = iqr(gr_[feature_name])
elif agg == 'median':
features['{}{}_median'.format(prefix, feature_name)] = gr_[feature_name].median()
elif agg == 'nunique':
features['{}{}_nunique'.format(prefix, feature_name)] = gr_[feature_name].nunique()
return features
def reduce_mem_usage(df):
# print('reduce_mem_usage_parallel start!')
# chunk_size = df.columns.shape[0]
# start_mem = df.memory_usage().sum() / 1024 ** 2
# print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
for col in df.columns:
col_type = df[col].dtype
if col_type != object:
c_min = df[col].min()
c_max = df[col].max()
if str(col_type)[:3] == 'int':
if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
df[col] = df[col].astype(np.int8)
elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
df[col] = df[col].astype(np.int16)
elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
df[col] = df[col].astype(np.int32)
elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
df[col] = df[col].astype(np.int64)
else:
if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
df[col] = df[col].astype(np.float16)
elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
df[col] = df[col].astype(np.float32)
else:
df[col] = df[col].astype(np.float64)
else:
df[col] = df[col].astype('category')
# end_mem = df.memory_usage().sum() / 1024 ** 2
# print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
# print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
return df
def reduce_mem_usage_parallel(df_original,num_worker):
print('reduce_mem_usage_parallel start!')
# chunk_size = df_original.columns.shape[0]
start_mem = df_original.memory_usage().sum() / 1024 ** 2
print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))
if df_original.columns.shape[0]>500:
group_chunk = []
for name in df_original.columns:
group_chunk.append(df_original[[name]])
with mp.Pool(num_worker) as executor:
df_temp = executor.map(reduce_mem_usage,group_chunk)
del group_chunk
gc.collect()
df_original = pd.concat(df_temp,axis = 1)
end_mem = df_original.memory_usage().sum() / 1024 ** 2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
del df_temp
gc.collect()
else:
df_original = reduce_mem_usage(df_original)
end_mem = df_original.memory_usage().sum() / 1024 ** 2
print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))
print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))
return df_original
# 评估指标
def MAPE(true, pred):
diff = np.abs(np.array(pred) - np.array(true))
return np.mean(diff / true)
# 自定义lgb评估指标
def lgb_score_mape(train_data,preds):
labels = train_data
diff = np.abs(np.array(preds) - np.array(labels))
result = np.mean(diff / labels)
return 'mape',result, False
def ridge_feature_select(X_train, y_train, num_folds):
print("Starting feature select. Train shape: {}".format(X_train.shape))
skf = KFold(n_splits=num_folds, shuffle=True, random_state=2021)
feature_importance_df = pd.DataFrame()
oof_preds = np.zeros(X_train.shape[0])
k_fold_mape = []
for i, (trn_idx, val_idx) in enumerate(skf.split(X_train, y_train)):
clf = Ridge(alpha=1)
clf.fit(X_train.iloc[trn_idx].fillna(0), y_train.iloc[trn_idx])
oof_preds[val_idx] = clf.predict(X_train.iloc[val_idx].fillna(0))
k_fold_mape.append(MAPE(y_train.iloc[val_idx], oof_preds[val_idx]))
# print("kfold_{}_mape_score:{} ".format(i, k_fold_mape[i]))
full_mape = MAPE(y_train, oof_preds)
print("full_mape_score:{} ".format(full_mape))
return k_fold_mape,full_mape
def feature_select(X_train,y_train):
feature_importance_df_ = pd.read_csv('feature_importances.csv')
cols = feature_importance_df_[["feature", "importance"]].groupby("feature").mean().sort_values(by="importance", ascending=False).index
best_features = feature_importance_df_.loc[feature_importance_df_.feature.isin(cols)]
best_features = best_features.groupby('feature',as_index = False)['importance'].mean()
best_features = best_features.sort_values(by = 'importance',ascending=False)
data=best_features.sort_values(by="importance", ascending=False)
feature_select = list(data['feature'].values)
feature_array = []
full_mape_all = 0
count = 0
for fea in feature_select:
print(count)
count = count + 1
feature_array.append(fea)
df_select = X_train[feature_array]
k_fold_mape, full_mape = ridge_feature_select(df_select,y_train, num_folds=5)
if count == 1:
full_mape_all = full_mape
file = open('feature_select_name.txt', 'a')
file.write(fea + '\n')
file.close()
file = open('feature_select_fullauc.txt', 'a')
file.write(str(full_mape_all) + '\n')
file.close()
file = open('feature_select_kfoldauc.txt', 'a')
file.write(str(k_fold_mape) + '\n')
file.close()
del df_select
gc.collect()
continue
if full_mape_all <= full_mape:
feature_array.remove(fea)
else:
full_mape_all = full_mape
file = open('feature_select_name.txt', 'a')
file.write(fea + '\n')
file.close()
file = open('feature_select_fullauc.txt', 'a')
file.write(str(full_mape_all) + '\n')
file.close()
file = open('feature_select_kfoldauc.txt', 'a')
file.write(str(k_fold_mape) + '\n')
file.close()
del df_select
gc.collect()
a = 1
Write Preview
Loading…
Cancel
Save