Create. DCN model & WD model

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/DCN_12953/dcn_model/dcn_model.py

@@ -0,0 +1,644 @@
+import pandas as pd
+import numpy as np
+from tensorflow import keras
+import tensorflow as tf
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+import tensorflow.keras.layers as L
+# import tensorflow.keras.models as M
+import tensorflow.keras.backend as K
+from tensorflow.python.client import device_lib
+from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
+#from keras_radam import RAdam
+from keras_radam.training import RAdamOptimizer
+from tensorflow.keras import initializers, regularizers, constraints, optimizers, layers, callbacks
+from tensorflow.keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D, Conv1D
+from tensorflow.keras.layers import Input, Dense, Lambda, Layer
+from tensorflow.keras.initializers import Constant
+from tensorflow.keras.models import Model
+import os
+from tensorflow.keras.losses import mean_absolute_percentage_error
+#from tensorflow.contrib.opt import AdamWOptimizer
+
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"]='3'
+gamma = 2.0
+alpha=.25
+epsilon = K.epsilon()
+
+
+def mape_2(y_true, y_pred):
+    y_true = y_true[:, :1]
+    y_pred = y_pred[:, :1]
+    return tf.py_function(mean_absolute_percentage_error, (y_true, y_pred), tf.float32)
+
+def mape_3(y_true, y_pred):
+    y_true = y_true[:, :1]
+    y_pred = y_pred[:, 1:]
+    return tf.py_function(mean_absolute_percentage_error, (y_true, y_pred), tf.float32)
+
+
+def knowledge_distillation_loss_withFL(y_true, y_pred, beta=0.1):
+
+    # Extract the groundtruth from dataset and the prediction from teacher model
+    y_true, y_pred_teacher = y_true[: , :1], y_true[: , 1:]
+    
+    # Extract the prediction from student model
+    y_pred, y_pred_stu = y_pred[: , :1], y_pred[: , 1:]
+
+    loss = beta*focal_loss(y_true,y_pred) + (1-beta)*mean_absolute_percentage_error(y_pred_teacher, y_pred_stu)
+
+    return loss
+
+
+def focal_loss(y_true, y_pred):
+    pt_1 = y_pred * y_true
+    pt_1 = K.clip(pt_1, epsilon, 1-epsilon)
+    CE_1 = -K.log(pt_1)
+    FL_1 = alpha* K.pow(1-pt_1, gamma) * CE_1
+    
+    pt_0 = (1-y_pred) * (1-y_true)
+    pt_0 = K.clip(pt_0, epsilon, 1-epsilon)
+    CE_0 = -K.log(pt_0)
+    FL_0 = (1-alpha)* K.pow(1-pt_0, gamma) * CE_0
+    
+    loss = K.sum(FL_1, axis=1) + K.sum(FL_0, axis=1)
+    return loss
+
+
+def knowledge_distillation_loss_withBE(y_true, y_pred, beta=0.6):
+
+    # Extract the groundtruth from dataset and the prediction from teacher model
+    y_true, y_pred_teacher = y_true[: , :1], y_true[: , 1:]
+    
+    # Extract the prediction from student model
+    y_pred, y_pred_stu = y_pred[: , :1], y_pred[: , 1:]
+
+    loss = beta*mean_absolute_percentage_error(y_true,y_pred) + (1-beta)*mean_absolute_percentage_error(y_pred_teacher, y_pred_stu)
+
+    return loss
+
+
+def get_available_gpus():
+    local_device_protos = device_lib.list_local_devices()
+    return [x.name for x in local_device_protos if x.device_type == 'GPU']
+
+
+def gru_layer(hidden_dim, dropout):
+    return L.Bidirectional(L.GRU(
+        hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer='orthogonal'))
+
+
+def lstm_layer(hidden_dim, dropout):
+    return L.Bidirectional(L.LSTM(
+        hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer='orthogonal'))
+
+
+class FeatureDictionary(object):
+    def __init__(self, df=None, numeric_cols=[], ignore_cols=[], cate_cols=[]):
+        self.df = df
+        self.cate_cols = cate_cols
+        self.numeric_cols = numeric_cols
+        self.ignore_cols = ignore_cols
+        self.gen_feat_dict()  # feat_dict 获取cate feature每一列的字典长度。
+
+    def gen_feat_dict(self):
+        self.feat_cate_len = {}
+        tc = 0
+        for col in self.cate_cols:
+            # 获取每一列的类别
+            us = self.df[col].unique()
+            us_len = len(us)
+            # 获取每一列的类别对应的维度
+            self.feat_cate_len[col] = us_len
+
+
+def embedding_layers(fd):
+    # 该函数主要是定义输入和embedding输入的网络层
+    embeddings_tensors = []
+    continus_tensors = []
+    cate_feature = fd.feat_cate_len
+    numeric_feature = fd.numeric_cols
+    for ec in cate_feature:
+        layer_name = ec + '_inp'
+        # for categorical features, embedding特征在维度保持在6×(category cardinality)**(1/4)
+        embed_dim = cate_feature[ec] if int(6 * np.power(cate_feature[ec], 1 / 4)) > cate_feature[ec] else int(
+            6 * np.power(cate_feature[ec], 1 / 4))
+        t_inp, t_embedding = embedding_input(layer_name, cate_feature[ec], embed_dim)
+        embeddings_tensors.append((t_inp, t_embedding))
+        del (t_inp, t_embedding)
+    for cc in numeric_feature:
+        layer_name = cc + '_in'
+        t_inp, t_build = continus_input(layer_name)
+        continus_tensors.append((t_inp, t_build))
+        del (t_inp, t_build)
+    # category feature的输入 这里的输入特征顺序要与xu
+    inp_layer = [et[0] for et in embeddings_tensors]
+    inp_embed = [et[1] for et in embeddings_tensors]
+    # numeric feature的输入
+    inp_layer += [ct[0] for ct in continus_tensors]
+    inp_embed += [ct[1] for ct in continus_tensors]
+
+    return inp_layer, inp_embed
+
+
+def embedding_input(name, input_dim, output_dim):
+    inp = L.Input(shape=(1,), dtype='int64', name=name)
+    embeddings = L.Embedding(input_dim, output_dim, input_length=1)(inp)
+    return inp, embeddings
+
+
+def continus_input(name):
+    inp = L.Input(shape=(1,), dtype='float32', name=name)
+    return inp, L.Reshape((1, 1))(inp)
+
+
+class CrossLayer(L.Layer):
+    def __init__(self, output_dim, num_layer, **kwargs):
+        self.output_dim = output_dim
+        self.num_layer = num_layer
+        super(CrossLayer, self).__init__(**kwargs)
+
+    def get_config(self):
+
+        config = super().get_config().copy()
+        config.update({
+            'vocab_size': self.vocab_size,
+            'num_layers': self.num_layers,
+            'units': self.units,
+            'd_model': self.d_model,
+            'num_heads': self.num_heads,
+            'dropout': self.dropout,
+        })
+        return config
+
+    def build(self, input_shape):
+        self.input_dim = input_shape[2]
+        self.W = []
+        self.bias = []
+        for i in range(self.num_layer):
+            self.W.append(
+                self.add_weight(shape=[1, self.input_dim], initializer='glorot_uniform', name='w_{}'.format(i),
+                                trainable=True))
+            self.bias.append(
+                self.add_weight(shape=[1, self.input_dim], initializer='zeros', name='b_{}'.format(i), trainable=True))
+        self.built = True
+
+    def call(self, input):
+        for i in range(self.num_layer):
+            if i == 0:
+                cross = L.Lambda(lambda x: K.batch_dot(K.dot(x, K.transpose(self.W[i])), x) + self.bias[i] + x)(input)
+            else:
+                cross = L.Lambda(lambda x: K.batch_dot(K.dot(x, K.transpose(self.W[i])), input) + self.bias[i] + x)(
+                    cross)
+        return L.Flatten()(cross)
+
+    def compute_output_shape(self, input_shape):
+        return None, self.output_dim
+
+
+def preprocess(df, cate_cols, numeric_cols):
+    for cl in cate_cols:
+        le = LabelEncoder()
+        df[cl] = le.fit_transform(df[cl])
+    cols = cate_cols + numeric_cols
+    X_train = df[cols]
+    return X_train
+
+
+def DCN_model(inp_layer, inp_embed, link_size, cross_size, slice_size, input_deep_col, input_wide_col,
+              link_nf_size, cross_nf_size, encoder,  link_seqlen=170, cross_seqlen=12, pred_len=1,
+              dropout=0.25, sp_dropout=0.1, embed_dim=64, hidden_dim=128, n_layers=3, lr=0.001, 
+              kernel_size1=3, kernel_size2=2, conv_size=128, conv=False, have_knowledge=True):
+    inp = L.concatenate(inp_embed, axis=-1)
+    link_inputs = L.Input(shape=(link_seqlen, link_nf_size), name='link_inputs')
+    cross_inputs = L.Input(shape=(cross_seqlen, cross_nf_size), name='cross_inputs')
+    deep_inputs = L.Input(shape=(input_deep_col,), name='deep_input')
+    slice_input = L.Input(shape=(1,), name='slice_input')
+    wide_inputs = keras.layers.Input(shape=(input_wide_col,), name='wide_inputs')
+
+    # link----------------------------
+    categorical_link = link_inputs[:, :, :1]
+    embed_link = L.Embedding(input_dim=link_size, output_dim=embed_dim, mask_zero=True)(categorical_link)
+    reshaped_link = tf.reshape(embed_link, shape=(-1, embed_link.shape[1], embed_link.shape[2] * embed_link.shape[3]))
+    reshaped_link = L.SpatialDropout1D(sp_dropout)(reshaped_link)
+    
+    """
+    categorical_slice = link_inputs[:, :, 5:6]
+    embed_slice = L.Embedding(input_dim=289, output_dim=16, mask_zero=True)(categorical_slice)
+    reshaped_slice = tf.reshape(embed_slice, shape=(-1, embed_slice.shape[1], embed_slice.shape[2] * embed_slice.shape[3]))
+    reshaped_slice = L.SpatialDropout1D(sp_dropout)(reshaped_slice)
+
+    categorical_hightemp = link_inputs[:, :, 6:7]
+    embed_hightemp = L.Embedding(input_dim=33, output_dim=8, mask_zero=True)(categorical_hightemp)
+    reshaped_hightemp = tf.reshape(embed_hightemp, shape=(-1, embed_hightemp.shape[1], embed_hightemp.shape[2] * embed_hightemp.shape[3]))
+    reshaped_hightemp = L.SpatialDropout1D(sp_dropout)(reshaped_hightemp)
+
+    categorical_weather = link_inputs[:, :, 7:8]
+    embed_weather = L.Embedding(input_dim=7, output_dim=8, mask_zero=True)(categorical_weather)
+    reshaped_weather = tf.reshape(embed_weather, shape=(-1, embed_weather.shape[1], embed_weather.shape[2] * embed_weather.shape[3]))
+    reshaped_weather = L.SpatialDropout1D(sp_dropout)(reshaped_weather)
+    
+    numerical_fea1 = link_inputs[:, :, 1:5]
+    numerical_fea1 = L.Masking(mask_value=0, name='numerical_fea1')(numerical_fea1)
+    hidden = L.concatenate([reshaped_link, numerical_fea1, reshaped_slice, reshaped_hightemp, reshaped_weather], axis=2)
+    
+    """
+    if have_knowledge:
+        numerical_fea1 = link_inputs[:, :, 1:5]
+        numerical_fea1 = L.Masking(mask_value=0, name='numerical_fea1')(numerical_fea1)
+       
+         
+        categorical_ar_st = link_inputs[:, :, 5:6]
+        categorical_ar_st = L.Masking(mask_value=-1, name='categorical_ar_st')(categorical_ar_st)
+        embed_ar_st = L.Embedding(input_dim=289, output_dim=8)(categorical_ar_st)
+        reshaped_ar_st = tf.reshape(embed_ar_st, shape=(-1, embed_ar_st.shape[1], embed_ar_st.shape[2] * embed_ar_st.shape[3]))
+        reshaped_ar_st = L.SpatialDropout1D(sp_dropout)(reshaped_ar_st)
+
+        categorical_ar_sl = link_inputs[:, :, 6:7]
+        categorical_ar_sl = L.Masking(mask_value=-1, name='categorical_ar_sl')(categorical_ar_sl)
+        embed_ar_sl = L.Embedding(input_dim=289, output_dim=8)(categorical_ar_sl)
+        reshaped_ar_sl = tf.reshape(embed_ar_sl, shape=(-1, embed_ar_sl.shape[1], embed_ar_sl.shape[2] * embed_ar_sl.shape[3]))
+        reshaped_ar_sl = L.SpatialDropout1D(sp_dropout)(reshaped_ar_sl)
+        hidden = L.concatenate([reshaped_link, reshaped_ar_st, reshaped_ar_sl, numerical_fea1],axis=2)
+        
+        #hidden = L.concatenate([reshaped_link, numerical_fea1],axis=2)
+    else:
+        numerical_fea1 = link_inputs[:, :, 1:5]
+        numerical_fea1 = L.Masking(mask_value=0, name='numerical_fea1')(numerical_fea1)    
+        
+        categorical_arrival = link_inputs[:, :, 5:6]
+        categorical_arrival = L.Masking(mask_value=-1, name='categorical_arrival')(categorical_arrival)
+        embed_ar = L.Embedding(input_dim=5, output_dim=16)(categorical_arrival)
+        reshaped_ar = tf.reshape(embed_ar, shape=(-1, embed_ar.shape[1], embed_ar.shape[2] * embed_ar.shape[3]))
+        reshaped_ar = L.SpatialDropout1D(sp_dropout)(reshaped_ar)
+        
+        categorical_ar_st = link_inputs[:, :, 6:7]
+        categorical_ar_st = L.Masking(mask_value=-1, name='categorical_ar_st')(categorical_ar_st)
+        embed_ar_st = L.Embedding(input_dim=289, output_dim=8)(categorical_ar_st)
+        reshaped_ar_st = tf.reshape(embed_ar_st, shape=(-1, embed_ar_st.shape[1], embed_ar_st.shape[2] * embed_ar_st.shape[3]))
+        reshaped_ar_st = L.SpatialDropout1D(sp_dropout)(reshaped_ar_st)
+
+        categorical_ar_sl = link_inputs[:, :, 7:8]
+        categorical_ar_sl = L.Masking(mask_value=-1, name='categorical_ar_sl')(categorical_ar_sl)
+        embed_ar_sl = L.Embedding(input_dim=289, output_dim=8)(categorical_ar_sl)
+        reshaped_ar_sl = tf.reshape(embed_ar_sl, shape=(-1, embed_ar_sl.shape[1], embed_ar_sl.shape[2] * embed_ar_sl.shape[3]))
+        reshaped_ar_sl = L.SpatialDropout1D(sp_dropout)(reshaped_ar_sl)
+        hidden = L.concatenate([reshaped_link, reshaped_ar, reshaped_ar_st, reshaped_ar_sl, numerical_fea1],axis=2)
+        
+        #hidden = L.concatenate([reshaped_link, reshaped_ar, numerical_fea1],axis=2)
+    #hidden = L.Masking(mask_value=0)(hidden)
+    for x in range(n_layers):
+        hidden = gru_layer(hidden_dim, dropout)(hidden)
+
+    if conv:
+        x_conv1 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(hidden)
+        avg_pool1_gru = GlobalAveragePooling1D()(x_conv1)
+        max_pool1_gru = GlobalMaxPooling1D()(x_conv1)
+        #x_conv2 = Conv1D(conv_size, kernel_size=kernel_size2, padding='valid', kernel_initializer='he_uniform')(hidden)
+        #avg_pool2_gru = GlobalAveragePooling1D()(x_conv2)
+        #max_pool2_gru = GlobalMaxPooling1D()(x_conv2)
+        truncated_link = concatenate([avg_pool1_gru, max_pool1_gru])
+    else:
+        truncated_link = hidden[:, :pred_len]
+        truncated_link = L.Flatten()(truncated_link)
+
+    # truncated_link = Attention(256)(hidden)
+    # CROSS----------------------------
+    categorical_fea2 = cross_inputs[:, :, :1]
+    embed2 = L.Embedding(input_dim=cross_size, output_dim=16, mask_zero=True)(categorical_fea2)
+    reshaped2 = tf.reshape(embed2, shape=(-1, embed2.shape[1], embed2.shape[2] * embed2.shape[3]))
+    reshaped2 = L.SpatialDropout1D(sp_dropout)(reshaped2)
+
+    numerical_fea2 = cross_inputs[:, :, 1:]
+    numerical_fea2 = L.Masking(mask_value=0, name='numerical_fea2')(numerical_fea2)
+    hidden2 = L.concatenate([reshaped2, numerical_fea2], axis=2)
+    # hidden2 = L.Masking(mask_value=0)(hidden2)
+    for x in range(n_layers):
+        hidden2 = gru_layer(hidden_dim, dropout)(hidden2)
+
+    if conv:
+        x_conv3 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(hidden2)
+        avg_pool3_gru = GlobalAveragePooling1D()(x_conv3)
+        max_pool3_gru = GlobalMaxPooling1D()(x_conv3)
+        #x_conv4 = Conv1D(conv_size, kernel_size=kernel_size2, padding='valid', kernel_initializer='he_uniform')(hidden2)
+        #avg_pool4_gru = GlobalAveragePooling1D()(x_conv4)
+        #max_pool4_gru = GlobalMaxPooling1D()(x_conv4)
+        truncated_cross = concatenate([avg_pool3_gru, max_pool3_gru])
+    else:
+        truncated_cross = hidden2[:, :pred_len]
+        truncated_cross = L.Flatten()(truncated_cross)
+    
+    # truncated_cross = Attention(256)(hidden2)
+    # SLICE----------------------------
+    embed_slice = L.Embedding(input_dim=slice_size, output_dim=1)(slice_input)
+    embed_slice = L.Flatten()(embed_slice)
+
+    # DEEP_INPUS
+    x = encoder(deep_inputs)
+    x = L.Concatenate()([x, deep_inputs])  # use both raw and encoded features
+    x = L.BatchNormalization()(x)
+    x = L.Dropout(0.25)(x)
+    
+    for i in range(3):
+        x = L.Dense(256)(x)
+        x = L.BatchNormalization()(x)
+        x = L.Lambda(tf.keras.activations.swish)(x)
+        x = L.Dropout(0.25)(x)
+    dense_hidden3 = L.Dense(64,activation='linear')(x)
+
+    # DCN
+    cross = CrossLayer(output_dim=inp.shape[2], num_layer=8, name="cross_layer")(inp)
+
+
+    # MAIN-------------------------------
+    truncated = L.concatenate([truncated_link, truncated_cross, cross, dense_hidden3, wide_inputs, embed_slice])
+    truncated = L.BatchNormalization()(truncated)
+    truncated = L.Dropout(dropout)(L.Dense(512, activation='relu') (truncated))
+    truncated = L.BatchNormalization()(truncated)
+    truncated = L.Dropout(dropout)(L.Dense(256, activation='relu') (truncated))
+
+    if have_knowledge:
+        out = L.Dense(2, activation='linear', name='out')(truncated)
+        model = tf.keras.Model(inputs=[inp_layer, link_inputs, cross_inputs, deep_inputs, wide_inputs, slice_input],
+                               outputs=out)
+        print(model.summary())
+        model.compile(loss=knowledge_distillation_loss_withBE,
+                      optimizer=RAdamOptimizer(learning_rate=1e-3),  # 'adam'  RAdam(warmup_proportion=0.1, min_lr=1e-7)
+                      #metrics={'out':'mape'} # AdamWOptimizer(weight_decay=1e-4)
+                      metrics=[mape_2,mape_3]
+                      )
+    else:
+        out = L.Dense(1, activation='linear', name='out')(truncated)
+        model = tf.keras.Model(inputs=[inp_layer, link_inputs, cross_inputs, deep_inputs, wide_inputs, slice_input],
+                               outputs=out)
+        print(model.summary())
+        model.compile(loss=['mape'],
+                      optimizer=RAdamOptimizer(learning_rate=1e-3),  # 'adam'  RAdam(warmup_proportion=0.1, min_lr=1e-7)
+                      #metrics={'out':'mape'}
+                      metrics=['mape']
+                      )
+
+    return model
+
+
+def arrival_model(inp_layer, inp_embed, link_size, cross_size, slice_size, input_deep_col, input_wide_col,
+              link_nf_size, cross_nf_size,  link_seqlen=170, cross_seqlen=12, pred_len=1,
+              dropout=0.25, sp_dropout=0.1, embed_dim=64, hidden_dim=128, n_layers=3, lr=0.001,
+              kernel_size1=3, kernel_size2=2, conv_size=128, conv=False):
+    inp = L.concatenate(inp_embed, axis=-1)
+    link_inputs = L.Input(shape=(link_seqlen, link_nf_size), name='link_inputs')
+    cross_inputs = L.Input(shape=(cross_seqlen, cross_nf_size), name='cross_inputs')
+    deep_inputs = L.Input(shape=(input_deep_col,), name='deep_input')
+    slice_input = L.Input(shape=(1,), name='slice_input')
+    wide_inputs = keras.layers.Input(shape=(input_wide_col,), name='wide_inputs')
+
+    # link----------------------------
+    categorical_link = link_inputs[:, :, :1]
+    embed_link = L.Embedding(input_dim=link_size, output_dim=embed_dim, mask_zero=True)(categorical_link)
+    reshaped_link = tf.reshape(embed_link, shape=(-1, embed_link.shape[1], embed_link.shape[2] * embed_link.shape[3]))
+    reshaped_link = L.SpatialDropout1D(sp_dropout)(reshaped_link)
+    """ 
+    categorical_slice = link_inputs[:, :, 5:6]
+    embed_slice = L.Embedding(input_dim=289, output_dim=16, mask_zero=True)(categorical_slice)
+    reshaped_slice = tf.reshape(embed_slice, shape=(-1, embed_slice.shape[1], embed_slice.shape[2] * embed_slice.shape[3]))
+    reshaped_slice = L.SpatialDropout1D(sp_dropout)(reshaped_slice)
+
+    categorical_hightemp = link_inputs[:, :, 6:7]
+    embed_hightemp = L.Embedding(input_dim=33, output_dim=8, mask_zero=True)(categorical_hightemp)
+    reshaped_hightemp = tf.reshape(embed_hightemp, shape=(-1, embed_hightemp.shape[1], embed_hightemp.shape[2] * embed_hightemp.shape[3]))
+    reshaped_hightemp = L.SpatialDropout1D(sp_dropout)(reshaped_hightemp)
+
+    categorical_weather = link_inputs[:, :, 7:8]
+    embed_weather = L.Embedding(input_dim=7, output_dim=8, mask_zero=True)(categorical_weather)
+    reshaped_weather = tf.reshape(embed_weather, shape=(-1, embed_weather.shape[1], embed_weather.shape[2] * embed_weather.shape[3]))
+    reshaped_weather = L.SpatialDropout1D(sp_dropout)(reshaped_weather)
+    
+    numerical_fea1 = link_inputs[:, :, 1:5]
+    numerical_fea1 = L.Masking(mask_value=0, name='numerical_fea1')(numerical_fea1)
+    hidden = L.concatenate([reshaped_link, numerical_fea1, reshaped_slice, reshaped_hightemp, reshaped_weather], axis=2)
+    """
+    numerical_fea1 = link_inputs[:, :, 1:]
+    numerical_fea1 = L.Masking(mask_value=0, name='numerical_fea1')(numerical_fea1)
+    hidden = L.concatenate([reshaped_link, numerical_fea1],axis=2)
+    
+    #hidden = L.Masking(mask_value=0)(hidden)
+    for x in range(n_layers):
+        hidden = gru_layer(hidden_dim, dropout)(hidden)
+    if conv:
+        x_conv1 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(hidden)
+        avg_pool1_gru = GlobalAveragePooling1D()(x_conv1)
+        max_pool1_gru = GlobalMaxPooling1D()(x_conv1)
+        #x_conv2 = Conv1D(conv_size, kernel_size=kernel_size2, padding='valid', kernel_initializer='he_uniform')(hidden)
+        #avg_pool2_gru = GlobalAveragePooling1D()(x_conv2)
+        #max_pool2_gru = GlobalMaxPooling1D()(x_conv2)
+        truncated_link = concatenate([avg_pool1_gru, max_pool1_gru])
+    else:
+        truncated_link = hidden[:, :pred_len]
+        truncated_link = L.Flatten()(truncated_link)
+
+    # truncated_link = Attention(256)(hidden)
+    # CROSS----------------------------
+    categorical_fea2 = cross_inputs[:, :, :1]
+    embed2 = L.Embedding(input_dim=cross_size, output_dim=16, mask_zero=True)(categorical_fea2)
+    reshaped2 = tf.reshape(embed2, shape=(-1, embed2.shape[1], embed2.shape[2] * embed2.shape[3]))
+    reshaped2 = L.SpatialDropout1D(sp_dropout)(reshaped2)
+
+    numerical_fea2 = cross_inputs[:, :, 1:]
+    numerical_fea2 = L.Masking(mask_value=0, name='numerical_fea2')(numerical_fea2)
+    hidden2 = L.concatenate([reshaped2, numerical_fea2], axis=2)
+    # hidden2 = L.Masking(mask_value=0)(hidden2)
+    for x in range(n_layers):
+        hidden2 = gru_layer(hidden_dim, dropout)(hidden2)
+
+    if conv:
+        x_conv3 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(hidden2)
+        avg_pool3_gru = GlobalAveragePooling1D()(x_conv3)
+        max_pool3_gru = GlobalMaxPooling1D()(x_conv3)
+        #x_conv4 = Conv1D(conv_size, kernel_size=kernel_size2, padding='valid', kernel_initializer='he_uniform')(hidden2)
+        #avg_pool4_gru = GlobalAveragePooling1D()(x_conv4)
+        #max_pool4_gru = GlobalMaxPooling1D()(x_conv4)
+        truncated_cross = concatenate([avg_pool3_gru, max_pool3_gru])
+    else:
+        truncated_cross = hidden2[:, :pred_len]
+        truncated_cross = L.Flatten()(truncated_cross)
+
+    # truncated_cross = Attention(256)(hidden2)
+    # SLICE----------------------------
+    embed_slice = L.Embedding(input_dim=slice_size, output_dim=1)(slice_input)
+    embed_slice = L.Flatten()(embed_slice)
+
+    # DEEP_INPUS
+    x = L.BatchNormalization()(deep_inputs)
+    x = L.Dropout(0.25)(x)
+
+    for i in range(3):
+        x = L.Dense(256)(x)
+        x = L.BatchNormalization()(x)
+        x = L.Lambda(tf.keras.activations.swish)(x)
+        x = L.Dropout(0.25)(x)
+    dense_hidden3 = L.Dense(64,activation='linear')(x)
+
+    # DCN
+    cross = CrossLayer(output_dim=inp.shape[2], num_layer=8, name="cross_layer")(inp)
+    truncated = L.concatenate([truncated_link, truncated_cross, cross, dense_hidden3, wide_inputs, embed_slice])
+    truncated = L.BatchNormalization()(truncated)
+    truncated = L.Dropout(dropout)(L.Dense(512, activation='relu') (truncated))
+    truncated = L.BatchNormalization()(truncated)
+    truncated = L.Dropout(dropout)(L.Dense(256, activation='relu') (truncated))
+
+    arrival_0 = L.Dense(1, activation='linear', name='arrival_0')(truncated)
+    arrival_1 = L.Dense(1, activation='linear', name='arrival_1')(truncated)
+    arrival_2 = L.Dense(1, activation='linear', name='arrival_2')(truncated)
+    arrival_3 = L.Dense(1, activation='linear', name='arrival_3')(truncated)
+    arrival_4 = L.Dense(1, activation='linear', name='arrival_4')(truncated)
+
+    model = tf.keras.Model(inputs=[inp_layer,link_inputs, cross_inputs, deep_inputs, wide_inputs, slice_input],
+                           outputs=[arrival_0,arrival_1,arrival_2,arrival_3,arrival_4])
+    print(model.summary())
+    model.compile(loss='mse',
+                  optimizer=RAdamOptimizer(learning_rate=1e-3)  # 'adam'  RAdam(warmup_proportion=0.1, min_lr=1e-7)
+                  )
+                  
+    return model
+
+
+def get_mc_es_lr(model_name: str, patience=5, min_delta=1e-4):
+    mc = tf.keras.callbacks.ModelCheckpoint('../model_h5/model_{}.h5'.format(model_name)),
+    es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min',
+                                          restore_best_weights=True, patience=patience)
+    lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=patience-1, mode='min',
+                                              min_delta=min_delta)
+
+    return mc, es, lr
+
+
+def get_mc_es_lr_for_student(model_name: str, patience=5, min_delta=1e-4):
+    mc = tf.keras.callbacks.ModelCheckpoint('../model_h5/model_{}.h5'.format(model_name)),
+    es = tf.keras.callbacks.EarlyStopping(monitor='val_mape_2', mode='min',
+                                          restore_best_weights=True, patience=patience)
+    lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_mape_2', factor=0.8, patience=patience, mode='min',
+                                              min_delta=min_delta)
+
+    return mc, es, lr
+
+
+
+def create_autoencoder(input_dim, output_dim, noise=0.05):
+    i = L.Input(input_dim)
+    encoded = L.BatchNormalization()(i)
+    encoded = L.GaussianNoise(noise)(encoded)
+    encoded = L.Dense(128, activation='relu')(encoded)
+    decoded = L.Dropout(0.2)(encoded)
+    decoded = L.Dense(input_dim,name='decoded')(decoded)
+    x = L.Dense(64, activation='relu')(decoded)
+    x = L.BatchNormalization()(x)
+    x = L.Dropout(0.2)(x)
+    x = L.Dense(output_dim, activation='linear', name='ata_output')(x)
+    
+    encoder = keras.models.Model(inputs=i, outputs=decoded)
+    autoencoder = keras.models.Model(inputs=i, outputs=[decoded, x])
+    
+    autoencoder.compile(optimizer=RAdamOptimizer(learning_rate=1e-3), loss={'decoded':'mse', 'ata_output': 'mape'})
+    return autoencoder, encoder
+
+
+class Attention(L.Layer):
+    def __init__(self, step_dim,
+                 W_regularizer=None, b_regularizer=None,
+                 W_constraint=None, b_constraint=None,
+                 bias=True, **kwargs):
+        self.supports_masking = True
+        self.init = initializers.get('glorot_uniform')
+
+        self.W_regularizer = regularizers.get(W_regularizer)
+        self.b_regularizer = regularizers.get(b_regularizer)
+
+        self.W_constraint = constraints.get(W_constraint)
+        self.b_constraint = constraints.get(b_constraint)
+
+        self.bias = bias
+        self.step_dim = step_dim
+        self.features_dim = 0
+        super(Attention, self).__init__(**kwargs)
+
+    def build(self, input_shape):
+        assert len(input_shape) == 3
+
+        self.W = self.add_weight((input_shape[-1],),
+                                 initializer=self.init,
+                                 name='{}_W'.format(self.name),
+                                 regularizer=self.W_regularizer,
+                                 constraint=self.W_constraint)
+        self.features_dim = input_shape[-1]
+
+        if self.bias:
+            self.b = self.add_weight((input_shape[1],),
+                                     initializer='zero',
+                                     name='{}_b'.format(self.name),
+                                     regularizer=self.b_regularizer,
+                                     constraint=self.b_constraint)
+        else:
+            self.b = None
+
+        self.built = True
+
+    def compute_mask(self, input, input_mask=None):
+        return None
+
+    def call(self, x, mask=None):
+        features_dim = self.features_dim
+        step_dim = self.step_dim
+
+        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),
+                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))
+
+        if self.bias:
+            eij += self.b
+
+        eij = K.tanh(eij)
+
+        a = K.exp(eij)
+
+        if mask is not None:
+            a *= K.cast(mask, K.floatx())
+
+        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())
+
+        a = K.expand_dims(a)
+        weighted_input = x * a
+        return K.sum(weighted_input, axis=1)
+
+    def compute_output_shape(self, input_shape):
+        return input_shape[0],  self.features_dim
+
+class CustomMultiLossLayer(Layer):
+    def __init__(self, nb_outputs=2, **kwargs):
+        self.nb_outputs = nb_outputs
+        self.is_placeholder = True
+        super(CustomMultiLossLayer, self).__init__(**kwargs)
+        
+    def build(self, input_shape=None):
+        # initialise log_vars
+        self.log_vars = []
+        for i in range(self.nb_outputs):
+            self.log_vars += [self.add_weight(name='log_var' + str(i), shape=(1,),
+                                              initializer=Constant(0.), trainable=True)]
+        super(CustomMultiLossLayer, self).build(input_shape)
+
+    def multi_loss(self, ys_true, ys_pred):
+        assert len(ys_true) == self.nb_outputs and len(ys_pred) == self.nb_outputs
+        loss = 0
+        for y_true, y_pred, log_var in zip(ys_true, ys_pred, self.log_vars):
+            precision = K.exp(-log_var[0])
+            loss += K.sum(precision * (y_true - y_pred)**2. + log_var[0], -1)
+        return K.mean(loss)
+
+    def call(self, inputs):
+        ys_true = inputs[:self.nb_outputs]
+        ys_pred = inputs[self.nb_outputs:]
+        loss = self.multi_loss(ys_true, ys_pred)
+        self.add_loss(loss, inputs=inputs)
+        # We won't actually use the output.
+        return K.concatenate(inputs, -1)
+
+

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/DCN_12953/dcn_model/main.py

@@ -0,0 +1,431 @@
+import pandas as pd
+import numpy as np
+import gc
+import tensorflow as tf
+import process
+import dcn_model
+import sys
+import random
+import os
+from sklearn.preprocessing import StandardScaler
+from tensorflow.compat.v1 import ConfigProto
+from tensorflow.compat.v1 import InteractiveSession
+config = ConfigProto()
+config.gpu_options.allow_growth = True
+session = InteractiveSession(config=config)
+tf.random.set_seed(42)
+os.environ["TF_CPP_MIN_LOG_LEVEL"]='3'
+
+RANDOM_SEED = 42
+# types of columns of the data_set DataFrame
+CATEGORICAL_COLS = [
+    'weather_le', 'hightemp', 'lowtemp', 'dayofweek',
+    'slice_id', 'link_current_status_4'
+]
+
+NUMERIC_COLS = [
+    'distance', 'simple_eta', 'link_time_sum', 'link_count',
+    'cr_t_sum', 'link_current_status_4_percent', 'link_current_status_mean',
+    'pr_mean', 'dc_mean','lk_arrival_0_percent', 'lk_arrival_1_percent',
+    'lk_arrival_2_percent', 'lk_arrival_3_percent', 'lk_arrival_4_percent'
+
+]
+
+WIDE_COLS = [
+    'weather_le', 'hightemp', 'lowtemp', 'dayofweek'
+]
+
+IGNORE_COLS = [
+    'order_id', 'ata'
+]
+
+TRAINING = True
+VAL_TO_TEST = False
+
+
+def set_seed(seed=42):
+    random.seed(seed)
+    os.environ["PYTHONHASHSEED"] = str(seed)
+    np.random.seed(seed)
+
+
+if __name__ == '__main__':
+    set_seed(RANDOM_SEED)
+    print(dcn_model.get_available_gpus())  # 返回格式为：['/device:GPU:0', '/device:GPU:1']
+
+    # LOAD DATA
+    print('*-' * 40, 'LOAD DATA')
+    making_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_order_xt/'
+    link_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_170_link_sqe_for_order/'
+    cross_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/for_0714_cross_sqe_for_order/'
+    link_data_other_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/for_0714_link_sqe_for_order_other/'
+    head_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_head_link_data_clear/'
+    win_order_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/win_order_xw/'
+    #pre_arrival_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/final_pre_arrival_data/'
+    arrival_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_link_sqe_for_order_arrival/'
+    zsl_arrival_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/zsl_arrival/'
+    arrival_sqe_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_170_lk_arrival_sqe_for_order/'
+    #h_s_for_link_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_hightmp_slice_for_link_eb/'
+    pre_arrival_sqe_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/sqe_arrival_for_link/'
+    zsl_link_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/zsl_train_link/'
+    data, mk_cols_list, link_cols_list, cross_cols_list = process.load_data(making_data_dir,
+                                                                            link_data_dir,
+                                                                            cross_data_dir,
+                                                                            link_data_other_dir,
+                                                                            head_data_dir,
+                                                                            win_order_data_dir,
+                                                                            pre_arrival_sqe_dir,
+                                                                            zsl_link_data_dir,
+                                                                            #pre_arrival_data_dir,
+                                                                            #h_s_for_link_dir,
+                                                                            arrival_data_dir,
+                                                                            zsl_arrival_data_dir,
+                                                                            arrival_sqe_data_dir)
+    
+    #fd = dcn_model.FeatureDictionary(data, numeric_cols=NUMERIC_COLS, ignore_cols=IGNORE_COLS,
+    #                                 cate_cols=CATEGORICAL_COLS)
+    # PROCESSING DATA
+    data['date_time'] = data['date_time'].astype(int)
+    print("type(data['date_time']):", data['date_time'].dtype)
+    data = data[data['date_time'] != 20200901]
+    print('Here train_test_split..................')
+    # all_train_data, _ = train_test_split(all_train_data, test_size=0.9, random_state=42)
+    data = data.reset_index()
+    del data['index']
+    print('*-' * 40, 'The data.shape:', data.shape)
+    train_data, val_data = train_test_split(data, test_size=0.15, random_state=RANDOM_SEED)
+    train_data = train_data.reset_index()
+    val_data = val_data.reset_index()
+    del train_data['index']
+    del val_data['index']
+    print('Save End.................')
+    fb_list = CATEGORICAL_COLS+NUMERIC_COLS+IGNORE_COLS
+    data_bak = data[fb_list]
+    del data
+    data = data_bak.copy()
+    del data_bak
+    gc.collect()
+
+    print('*-' * 40, 'PROCESSING DATA FOR TRAIN')
+    train_data = process.processing_data(train_data, link_cols_list, cross_cols_list, mk_cols_list, WIDE_COLS)
+    #del data
+    #fb_list = CATEGORICAL_COLS+NUMERIC_COLS+IGNORE_COLS
+    #data = data[fb_list]
+    #gc.collect()
+    # print(train_data.columns.tolist())
+
+    # PROCESSING INPUTS
+    print('*-' * 40, 'PROCESSING INPUTS')
+    # SAVE LIST
+    a = np.array(mk_cols_list)
+    np.save('../model_h5/mk_cols_list_0720_2.npy', a)
+    a = np.array(link_cols_list)
+    np.save('../model_h5/link_cols_list_0720_2.npy', a)
+    a = np.array(cross_cols_list)
+    np.save('../model_h5/cross_cols_list_0720_2.npy', cross_cols_list)
+    a = np.array(CATEGORICAL_COLS)
+    np.save('../model_h5/CATEGORICAL_COLS_0720_2.npy', a)
+    del a
+    pred_cols = ['ata']
+    print('*-' * 40, 'PROCESSING INPUTS FOR TRAIN_DATA', train_data.shape)
+    train_link_inputs, train_cross_inputs, train_deep_input, train_wide_input, \
+        train_inputs_slice, train_labels, train_arrival = process.processing_inputs(
+            train_data, mk_cols_list, link_cols_list, cross_cols_list, WIDE_COLS)
+    X_train = dcn_model.preprocess(train_data, CATEGORICAL_COLS, NUMERIC_COLS)
+    train_pre = train_data[['order_id']]
+    del train_data
+    gc.collect()
+
+    print('*-' * 40, 'PROCESSING DATA FOR TRAIN')
+    val_data = process.processing_data(val_data, link_cols_list, cross_cols_list, mk_cols_list, WIDE_COLS,  is_test=True)
+    print('*-' * 40, 'PROCESSING INPUTS FOR VAL_DATA', val_data.shape)
+    val_link_inputs, val_cross_inputs, val_deep_input, val_wide_input, \
+        val_inputs_slice, val_labels, val_arrival = process.processing_inputs(
+            val_data, mk_cols_list, link_cols_list, cross_cols_list, WIDE_COLS)
+    X_val = dcn_model.preprocess(val_data, CATEGORICAL_COLS, NUMERIC_COLS)
+    # val_data.to_csv('../model_h5/val_data.csv', index=0)  # saving csv for test running
+    val_pre = val_data[['order_id']]
+    del val_data
+    gc.collect()
+
+    # MODEL_INIT
+    print('*-' * 40, 'T_MODEL_INIT')
+    deep_col_len, wide_col_len = train_deep_input.values.shape[1], train_wide_input.shape[1]
+    link_size = 639877 + 2
+    cross_size = 44313 + 2
+    link_nf_size, cross_nf_size = train_link_inputs.shape[2], train_cross_inputs.shape[2]
+    slice_size = 288
+    # link_seqlen, cross_seqlen = 170, 12  # 已默认
+    print("link_size:{},link_nf_size:{},cross_size:{},cross_nf_size:{},slice_size:{}".format(link_size, link_nf_size,
+                                                                                             cross_size, cross_nf_size,
+                                                                                             slice_size))
+    print("deep_col_len:{}, wide_col_len:{}".format(deep_col_len, wide_col_len))
+
+    fd = dcn_model.FeatureDictionary(data, numeric_cols=NUMERIC_COLS, ignore_cols=IGNORE_COLS,
+                                     cate_cols=CATEGORICAL_COLS)
+    inp_layer, inp_embed = dcn_model.embedding_layers(fd)
+    autoencoder, encoder = dcn_model.create_autoencoder(train_deep_input.values.shape[-1], 1, noise=0.1)
+    if TRAINING:
+        autoencoder.fit(train_deep_input.values, (train_deep_input.values, train_labels.values),
+                        epochs=1000,  # 1000
+                        batch_size=2048,  # 1024
+                        validation_split=0.1,
+                        callbacks=[tf.keras.callbacks.EarlyStopping('val_ata_output_loss', patience=10, restore_best_weights=True)])
+        encoder.save_weights('../model_h5/t_encoder.hdf5')
+    else:
+        encoder.load_weights('../model_h5/t_encoder.hdf5')
+    encoder.trainable = False
+    del autoencoder
+
+    t_model = dcn_model.DCN_model(inp_layer, inp_embed, link_size, cross_size, slice_size, deep_col_len, wide_col_len,
+                                link_nf_size, cross_nf_size, encoder, conv=True, have_knowledge=False)    
+    #del encoder
+    gc.collect()
+    
+    mc, es, lr = dcn_model.get_mc_es_lr('0720_2', patience=5, min_delta=1e-4)
+    print('*-' * 40, 'MODEL_INIT END')
+   
+    print('*-' * 40, 'ARRIVAL_MODEL_FIT')
+    t_history = t_model.fit(
+        [
+            X_train['weather_le'], X_train['hightemp'], X_train['lowtemp'], X_train['dayofweek'],
+            X_train['slice_id'], X_train['link_current_status_4'],
+            X_train['distance'], X_train['simple_eta'], X_train['link_time_sum'], X_train['link_count'],
+            X_train['cr_t_sum'], X_train['link_current_status_4_percent'], X_train['link_current_status_mean'],
+            X_train['pr_mean'], X_train['dc_mean'],
+            X_train['lk_arrival_0_percent'], X_train['lk_arrival_1_percent'],X_train['lk_arrival_2_percent'], 
+            X_train['lk_arrival_3_percent'],X_train['lk_arrival_4_percent'],
+            train_link_inputs, train_cross_inputs, train_deep_input.values, train_wide_input, train_inputs_slice],
+        train_labels.values,
+        validation_data=(
+            [
+                X_val['weather_le'], X_val['hightemp'], X_val['lowtemp'], X_val['dayofweek'],
+                X_val['slice_id'], X_val['link_current_status_4'],
+                X_val['distance'], X_val['simple_eta'], X_val['link_time_sum'], X_val['link_count'],
+                X_val['cr_t_sum'], X_val['link_current_status_4_percent'], X_val['link_current_status_mean'],
+                X_val['pr_mean'], X_val['dc_mean'],
+                X_val['lk_arrival_0_percent'], X_val['lk_arrival_1_percent'],X_val['lk_arrival_2_percent'], 
+                X_val['lk_arrival_3_percent'],X_val['lk_arrival_4_percent'],
+                val_link_inputs, val_cross_inputs, val_deep_input.values, val_wide_input, val_inputs_slice],
+                (val_labels.values),),
+        batch_size=2048,  # 2048,1024
+        epochs=100,  # 100
+        verbose=1,
+        # )
+        callbacks=[es])  # lr
+    np.save('../model_h5/t_model_0720_2.npy', t_history.history)
+    t_model.save_weights("../model_h5/t_model_0720_2.h5")
+    print('*-' * 40, 't_MODEL_PREDICT')
+    y_knowledge_train = t_model.predict(
+            [X_train['weather_le'], X_train['hightemp'], X_train['lowtemp'], X_train['dayofweek'],
+            X_train['slice_id'], X_train['link_current_status_4'],
+            X_train['distance'], X_train['simple_eta'], X_train['link_time_sum'], X_train['link_count'],
+            X_train['cr_t_sum'], X_train['link_current_status_4_percent'], X_train['link_current_status_mean'],
+            X_train['pr_mean'], X_train['dc_mean'],
+            X_train['lk_arrival_0_percent'], X_train['lk_arrival_1_percent'],X_train['lk_arrival_2_percent'], 
+            X_train['lk_arrival_3_percent'],X_train['lk_arrival_4_percent'],
+            train_link_inputs, train_cross_inputs, train_deep_input.values, train_wide_input, train_inputs_slice],
+            batch_size=2048)
+    y_knowledge_val = t_model.predict(
+            [
+                X_val['weather_le'], X_val['hightemp'], X_val['lowtemp'], X_val['dayofweek'],
+                X_val['slice_id'], X_val['link_current_status_4'],
+                X_val['distance'], X_val['simple_eta'], X_val['link_time_sum'], X_val['link_count'],
+                X_val['cr_t_sum'], X_val['link_current_status_4_percent'], X_val['link_current_status_mean'],
+                X_val['pr_mean'], X_val['dc_mean'],
+                X_val['lk_arrival_0_percent'], X_val['lk_arrival_1_percent'],X_val['lk_arrival_2_percent'],
+                X_val['lk_arrival_3_percent'],X_val['lk_arrival_4_percent'],
+                val_link_inputs, val_cross_inputs, val_deep_input.values, val_wide_input, val_inputs_slice],
+                batch_size=2048)
+    print('*-'*40, 'TRAINFORME')
+    train_labels = pd.DataFrame(train_labels)
+    train_labels['y_knowledge_train'] = np.squeeze(y_knowledge_train)
+    print(np.squeeze(y_knowledge_train)[:2])
+    print(train_labels['y_knowledge_train'].head(2))
+    val_labels = pd.DataFrame(val_labels) 
+    val_labels['y_knowledge_val'] = np.squeeze(y_knowledge_val)
+    print('*-' * 40, 't_MODEL_END')
+    zsl_arrival_cols = ['zsl_link_arrival_status_mean','zsl_link_arrival_status_nunique','zsl_link_arrival_status0','zsl_link_arrival_status1','zsl_link_arrival_status2','zsl_link_arrival_status3']
+    train_deep_input = train_deep_input.drop(['lk_arrival_0_percent','lk_arrival_1_percent','lk_arrival_2_percent','lk_arrival_3_percent','lk_arrival_4_percent'],axis=1)
+    train_deep_input = train_deep_input.drop(zsl_arrival_cols, axis=1)
+
+    val_deep_input = val_deep_input.drop(['lk_arrival_0_percent','lk_arrival_1_percent','lk_arrival_2_percent','lk_arrival_3_percent','lk_arrival_4_percent'],axis=1)
+    val_deep_input = val_deep_input.drop(zsl_arrival_cols, axis=1)
+
+    if 'ata' in train_deep_input.columns.tolist():
+        print('The ata in the train_deep_input')
+        print('*-' * 40, 'EXIT')
+        sys.exit(0)
+    if 'lk_arrival_0_percent' in train_deep_input.columns.tolist():
+        print('The lk_arrival_0_percent in the train_deep_input')
+        print('*-' * 40, 'EXIT')
+        sys.exit(0)
+    if 'lk_arrival_0_percent' in val_deep_input.columns.tolist():
+        print('The lk_arrival_0_percent in the val_deep_input')
+        print('*-' * 40, 'EXIT')
+        sys.exit(0)
+    if 'zsl_link_arrival_status_mean' in train_deep_input.columns.tolist():
+        print('The zsl_link_arrival_status_mean in the train_deep_input')
+        print('*-' * 40, 'EXIT')
+        sys.exit(0)
+
+    mk_cols_list = train_deep_input.columns.tolist()
+    print('*-' * 40, 'MODEL_FIT')
+    deep_col_len, wide_col_len = train_deep_input.values.shape[1], train_wide_input.shape[1]
+    print("deep_col_len:{}, wide_col_len:{}".format(deep_col_len, wide_col_len))
+    NUMERIC_COLS = list(set(NUMERIC_COLS)-set(['lk_arrival_0_percent','lk_arrival_1_percent','lk_arrival_2_percent',
+                                     'lk_arrival_3_percent','lk_arrival_4_percent']))
+    fb_list = CATEGORICAL_COLS+NUMERIC_COLS+IGNORE_COLS
+    if 'lk_arrival_0_percent' in fb_list:
+        print('The lk_arrival_0_percent in the fb_list')
+        print('*-' * 40, 'EXIT')
+        sys.exit(0)
+    data = data[fb_list]
+    fd = dcn_model.FeatureDictionary(data, numeric_cols=NUMERIC_COLS, ignore_cols=IGNORE_COLS,
+                                     cate_cols=CATEGORICAL_COLS)
+    inp_layer, inp_embed = dcn_model.embedding_layers(fd)
+    autoencoder, encoder = dcn_model.create_autoencoder(train_deep_input.values.shape[-1], 1, noise=0.1)
+    if TRAINING:
+        autoencoder.fit(train_deep_input.values, (train_deep_input.values, train_labels['ata'].values),
+                        epochs=1000,  # 1000
+                        batch_size=2048,  # 1024
+                        validation_split=0.1,
+                        callbacks=[tf.keras.callbacks.EarlyStopping('val_ata_output_loss', patience=10, restore_best_weights=True)])
+        encoder.save_weights('../model_h5/main_encoder.hdf5')
+    else:
+        encoder.load_weights('../model_h5/main_encoder.hdf5')
+    encoder.trainable = False
+    del autoencoder
+  
+    #print(type(train_labels['y_knowledge_train']))
+    #print(type(train_labels))
+    #y_train = np.vstack((train_labels, train_pre['y_knowledge_train'])).T
+    #y_valid = np.vstack((val_labels, val_pre['y_knowledge_val'])).T
+    #print(train_labels.shape)
+    print(train_labels.head(1))
+    print(train_labels.values[0])
+
+    print('*-'*40, 'The shape of train_link_inputs before', train_link_inputs.shape)
+    train_link_inputs = np.concatenate((train_link_inputs[:, :, :5], train_link_inputs[:, :, 6:]), axis=2)
+    
+    print('*-'*40, 'The shape of train_link_inputs after', train_link_inputs.shape)
+    val_link_inputs = np.concatenate((val_link_inputs[:, :, :5], val_link_inputs[:, :, 6:]), axis=2)
+    link_nf_size, cross_nf_size = train_link_inputs.shape[2], train_cross_inputs.shape[2]
+    mc, es, lr = dcn_model.get_mc_es_lr_for_student('0720_2', patience=5, min_delta=1e-4)
+    model = dcn_model.DCN_model(inp_layer, inp_embed, link_size, cross_size, slice_size, deep_col_len, wide_col_len,
+                                link_nf_size, cross_nf_size, encoder, conv=True)
+    history = model.fit(
+        [
+            X_train['weather_le'], X_train['hightemp'], X_train['lowtemp'], X_train['dayofweek'],
+            X_train['slice_id'], X_train['link_current_status_4'],
+            X_train['distance'], X_train['simple_eta'], X_train['link_time_sum'], X_train['link_count'],
+            X_train['cr_t_sum'], X_train['link_current_status_4_percent'], X_train['link_current_status_mean'],
+            X_train['pr_mean'], X_train['dc_mean'],
+            train_link_inputs, train_cross_inputs, train_deep_input.values, train_wide_input, train_inputs_slice],
+        train_labels.values,
+        validation_data=(
+            [
+                X_val['weather_le'], X_val['hightemp'], X_val['lowtemp'], X_val['dayofweek'],
+                X_val['slice_id'], X_val['link_current_status_4'],
+                X_val['distance'], X_val['simple_eta'], X_val['link_time_sum'], X_val['link_count'],
+                X_val['cr_t_sum'], X_val['link_current_status_4_percent'], X_val['link_current_status_mean'],
+                X_val['pr_mean'], X_val['dc_mean'],
+                val_link_inputs, val_cross_inputs, val_deep_input.values, val_wide_input, val_inputs_slice], 
+                (val_labels.values),),
+        batch_size=2048,  # 2048,1024
+        epochs=100,  # 100
+        verbose=1,
+        # )
+        callbacks=[es])  # lr
+    np.save('../model_h5/history_0720_2.npy', history.history)
+    model.save_weights("../model_h5/dcn_model_0720_2.h5")
+    # MODEL_RPEDICT
+    if VAL_TO_TEST:
+        print('*-'*40,'val_to_test')
+        val_pre = val_pre.rename(columns={'order_id': 'id'})
+        print(val_link_inputs.shape, val_cross_inputs.shape, X_val.shape)
+        print('*-' * 40, 'MODEL_RPEDICT')
+        val_pred = model.predict(
+            [
+                X_val['weather_le'], X_val['hightemp'], X_val['lowtemp'], X_val['dayofweek'],
+                X_val['slice_id'], X_val['link_current_status_4'],
+                X_val['distance'], X_val['simple_eta'], X_val['link_time_sum'], X_val['link_count'],
+                X_val['cr_t_sum'], X_val['link_current_status_4_percent'], X_val['link_current_status_mean'],
+                X_val['pr_mean'], X_val['dc_mean'],
+                val_link_inputs, val_cross_inputs, val_deep_input.values, val_wide_input, val_inputs_slice],
+               batch_size=2048)
+        val_pre['val_predict'] = np.squeeze(val_pred[:, 1])
+        val_pre['other_predict'] = np.squeeze(val_pred[:, 0])
+        # val_pre['val_predict'] = val_pre['val_predict'].round(0)
+        val_pre = val_pre.rename(columns={'val_predict': 'result'})  # 更改列名
+        val_pre = val_pre[['id', 'result', 'other_predict']]
+        val_pre['ata'] = val_labels['ata'].values
+        print(val_pre.head())
+        result_save_path = '../result_csv/val_0720_2.csv'
+        print('*-' * 40, 'CSV_SAVE_PATH:', result_save_path)
+        print('..........Finish')
+
+    del X_train, train_link_inputs, train_cross_inputs, train_deep_input, \
+        train_wide_input, train_inputs_slice, train_labels
+    del X_val, val_link_inputs, val_cross_inputs, val_deep_input, val_wide_input, val_inputs_slice, val_labels
+    gc.collect()
+    #print('*-' * 40, 'EXIT')
+    #sys.exit(0)
+    print('*-' * 40, 'LOAD TEST DATA')
+    making_test_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/order_xt/'
+    link_test_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/max_170_link_sqe_for_order/'
+    cross_test_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/cross_sqe_for_order/'
+    link_test_data_other_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/link_sqe_for_order_other/'
+    head_test_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/head_link_data_clear/'
+    win_order_test_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/win_order_xw/'
+    pre_arrival_sqe_test_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/sqe_arrival_for_link/'
+    #h_s_for_test_link_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/max_hightmp_slice_for_link_eb/'
+    #pre_arrival_test_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/final_pre_arrival_data/'
+    zsl_link_test_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/zsl_test_link/'
+    #zsl_cross_test_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/zsl_test_cross_0703/'
+    test_data, _, _, _ = process.load_data(making_test_data_dir,
+                                                                                 link_test_data_dir,
+                                                                                 cross_test_data_dir,
+                                                                                 link_test_data_other_dir,
+                                                                                 head_test_data_dir,
+                                                                                 win_order_test_data_dir,
+                                                                                 pre_arrival_sqe_test_dir,
+                                                                                 zsl_link_test_data_dir) #,
+                                                                                 #h_s_for_test_link_dir)
+                                                                                 #pre_arrival_test_data_dir)
+    print('*-' * 40, 'PROCESSING DATA')
+    link_cols_list.remove('link_arrival_status')
+    test_data = process.processing_data(test_data, link_cols_list, cross_cols_list, mk_cols_list, WIDE_COLS, is_test=True)
+    gc.collect()
+    print('*-' * 40, 'PROCESSING INPUTS FOR TEST_DATA', test_data.shape)
+    test_link_inputs, test_cross_inputs, test_deep_input, test_wide_input, \
+        test_inputs_slice, _ = process.processing_inputs(
+            test_data, mk_cols_list, link_cols_list, cross_cols_list, WIDE_COLS, arrival=False)
+    X_test = dcn_model.preprocess(test_data, CATEGORICAL_COLS, NUMERIC_COLS)
+    test_pre = test_data[['order_id']]
+    test_arrival_pre = test_data[['order_id']]
+    gc.collect()
+
+    test_pre = test_pre.rename(columns={'order_id': 'id'})
+    print(test_link_inputs.shape, test_cross_inputs.shape, X_test.shape, test_deep_input.shape)
+    print('*-' * 40, 'MODEL_RPEDICT')
+    test_pred = model.predict(
+        [
+            X_test['weather_le'], X_test['hightemp'], X_test['lowtemp'], X_test['dayofweek'],
+            X_test['slice_id'], X_test['link_current_status_4'],
+            X_test['distance'], X_test['simple_eta'], X_test['link_time_sum'], X_test['link_count'],
+            X_test['cr_t_sum'], X_test['link_current_status_4_percent'], X_test['link_current_status_mean'],
+            X_test['pr_mean'], X_test['dc_mean'],
+            test_link_inputs, test_cross_inputs, test_deep_input.values, test_wide_input, test_inputs_slice],
+           batch_size=2048)
+    test_pre['test_predict'] = np.squeeze(test_pred[:, 1])
+    test_pre['other_predict'] = np.squeeze(test_pred[:, 0])
+    # test_pre['test_predict'] = test_pre['test_predict'].round(0)
+    test_pre = test_pre.rename(columns={'test_predict': 'result'})  # 更改列名
+    test_pre = test_pre[['id', 'result','other_predict']]
+    print(test_pre.head())
+    result_save_path = '../result_csv/submit_0720_2.csv'
+    print('*-' * 40, 'CSV_SAVE_PATH:', result_save_path)
+    test_pre.to_csv(result_save_path, index=0)  # 保存
+
+    print('..........Finish')

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/DCN_12953/dcn_model/process.py

@@ -0,0 +1,451 @@
+import pandas as pd
+import numpy as np
+import joblib
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from tqdm import tqdm
+from pandarallel import pandarallel
+from sklearn.model_selection import train_test_split
+# import random
+import gc
+import ast
+import os
+import sys
+import warnings
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"]='3'
+warnings.filterwarnings('ignore')
+pd.options.mode.chained_assignment = None
+#pandarallel.initialize(nb_workers=16)
+pandarallel.initialize()
+
+
+def pandas_list_to_array(df):
+    """
+    Input: DataFrame of shape (x, y), containing list of length l
+    Return: np.array of shape (x, l, y)
+    """
+
+    return np.transpose(
+        np.array(df.values.tolist()),
+        (0, 2, 1)
+    )
+
+
+def preprocess_inputs(df, cols: list):
+    return pandas_list_to_array(
+        df[cols]
+    )
+
+
+def append_all_data(files_list, file_head_path):
+    """
+    concat all the data
+    :param files_list: the name of data
+    :param file_head_path: the path of data
+    :return: DataFrame of data for all
+    """
+    data_all_path = file_head_path + files_list[0]
+    data_all = pd.read_csv(data_all_path)
+    data_all = data_all.head(0)
+    try:
+        del data_all['Unnamed: 0']
+    except KeyError as e:
+        pass
+    # 循环添加全部数据
+    for i in files_list:
+        data_path = file_head_path + i
+        print("当前文件为：", data_path)
+        data = pd.read_csv(data_path)
+        try:
+            del data['Unnamed: 0']
+        except KeyError as e:
+            pass
+        data_all = data_all.append(data)
+    return data_all
+
+
+def file_name(file_dir):
+    files_list = []
+    for root, dirs, files in os.walk(file_dir):
+        # print("success")
+        for name in files:
+            files_list.append(name)
+    return files_list
+
+
+def load_data(making_data_dir, link_data_dir, cross_data_dir, link_data_other_dir, head_data_dir, 
+              win_order_data_dir, pre_arrival_sqe_dir,zsl_link_data_dir, arrival_data_dir=None, zsl_arrival_data_dir=None, arrival_sqe_data_dir=None):
+    """
+    loading three path of data, then merge them
+    :return: all data by order_level
+    """
+    print('-------------LOAD DATA for mk_data----------------')
+    mk_list = file_name(making_data_dir)
+    mk_list.sort()
+    mk_data = append_all_data(mk_list, making_data_dir)
+    #mk_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_order_xt/join_20200825.csv')  # for test running
+    mk_data['date_time'] = mk_data['date_time'].astype(str)
+    # print(mk_data['date_time'].head())
+    mk_data['dayofweek'] = pd.to_datetime(mk_data['date_time'])
+    mk_data['dayofweek'] = mk_data['dayofweek'].dt.dayofweek + 1
+    weather_le = LabelEncoder()
+    mk_data['weather_le'] = weather_le.fit_transform(mk_data['weather'])
+    print('Remove the wk2_ and m1_')
+    del_cols = []
+    mk_cols = mk_data.columns.tolist()
+    for i in range(len(mk_cols)):
+        if 'wk2_' in mk_cols[i]:
+            del_cols.append(mk_cols[i])
+        if 'm1_' in mk_cols[i]:
+            del_cols.append(mk_cols[i])
+        if 'ratio' in mk_cols[i]:
+            del_cols.append(mk_cols[i])
+    del_cols = del_cols + ['weather', 'driver_id', 'date_time_dt', 'link_time_sum','date_time_sum']
+    print('*-' * 40, 'Will be drop the list:', del_cols)
+    mk_data.drop(columns=del_cols, axis=1, inplace=True)
+    print('The init shape of mk_data:', mk_data.shape)
+    #if arrival_data_dir:
+    #    mk_data, _ = train_test_split(mk_data, test_size=0.4, random_state=42)
+    #print('*-'*40)
+    #print('The train_test_split shape of mk_data:', mk_data.shape)
+
+    
+    print('-------------LOAD WIN DATA----------------')
+    win_order_list = file_name(win_order_data_dir)
+    win_order_list.sort()
+    win_order_data = append_all_data(win_order_list, win_order_data_dir)
+    #win_order_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/win_order_xw/win_for_slice_20200825.csv')  # for test running
+    del_win_order_cols = []
+    win_order_cols = win_order_data.columns.tolist()
+    for i in range(len(win_order_cols)):
+        if 'last_wk_lk_current' in win_order_cols[i]:
+            del_win_order_cols.append(win_order_cols[i])
+        #if 'distance' in win_order_cols[i]:
+        #    del_win_order_cols.append(win_order_cols[i])
+        #if '1_percent' in win_order_cols[i]:
+        #    del_win_order_cols.append(win_order_cols[i])
+        #if '0_percent' in win_order_cols[i]:
+        #    del_win_order_cols.append(win_order_cols[i])
+    del_win_order_cols = del_win_order_cols + ['slice_id', 'date_time']
+    win_order_data.drop(columns=del_win_order_cols, axis=1, inplace=True)
+    print('win_order_data.shape',win_order_data.shape)
+    mk_data = pd.merge(mk_data, win_order_data, how='left', on='order_id')
+    print('mk_data.shape',mk_data.shape)
+    del win_order_data
+    gc.collect()
+
+
+    """ 
+    print('-------------LOAD ZSL DATA----------------')
+    zsl_link_list = file_name(zsl_link_data_dir)
+    zsl_link_list.sort()
+    zsl_link_data = append_all_data(zsl_link_list, zsl_link_data_dir)
+    #zsl_link_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/zsl_train_link/link_fea_order_id_level_20200825.csv')  # for test running
+    get_zsl_link_cols = []
+    zsl_link_cols = zsl_link_data.columns.tolist()
+    for i in range(len(zsl_link_cols)):
+        if 'eb' in zsl_link_cols[i]:
+            get_zsl_link_cols.append(zsl_link_cols[i])
+    #print(get_zsl_link_cols)
+    get_zsl_link_cols.insert(0, 'order_id')
+    print(zsl_link_data.shape)
+    zsl_link_data = zsl_link_data[get_zsl_link_cols]
+    print('mk_data.shape',mk_data.shape)
+    mk_data = pd.merge(mk_data, zsl_link_data, on='order_id')
+    print('mk_data.shape',mk_data.shape)
+    del zsl_link_data
+    gc.collect()
+    """
+    """
+    #zsl_cross_list = file_name(zsl_cross_data_dir)
+    #zsl_cross_list.sort()
+    #zsl_cross_data = append_all_data(zsl_cross_list, zsl_cross_data_dir)
+    zsl_cross_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/zsl_train_cross_0703/cross_fea_order_id_level_20200825.csv')  # for test running
+    get_zsl_cross_cols = []
+    zsl_cross_cols = zsl_cross_data.columns.tolist()
+    for i in range(len(zsl_cross_cols)):
+        if ('last' or 'div' or 'interval' or 'period') in zsl_cross_cols[i]:
+            get_zsl_cross_cols.append(zsl_cross_cols[i])
+    get_zsl_cross_cols.append('order_id')
+    print(zsl_cross_data.shape)
+    zsl_cross_data = zsl_cross_data[get_zsl_cross_cols]
+    print('mk_data.shape',mk_data.shape)
+    mk_data = pd.merge(mk_data, zsl_cross_data, on='order_id')
+    print('mk_data.shape',mk_data.shape)
+    del zsl_cross_data
+    gc.collect()
+    """
+    
+    print('-------------LOAD HEAD DATA----------------')
+    head_list = file_name(head_data_dir)
+    head_list.sort()
+    head_data = append_all_data(head_list, head_data_dir)
+    #head_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_head_link_data_clear/head_link_20200825.csv')  # for test running
+    get_head_cols = ['len_tmp','status_0','status_1','status_2','status_3','status_4','rate_0','rate_1','rate_2','rate_3','rate_4']
+    get_head_cols.insert(0, 'order_id')
+    print('head_data.shape:',head_data.shape)
+    head_data = head_data[get_head_cols]
+    print('mk_data.shape',mk_data.shape)
+    mk_data = pd.merge(mk_data, head_data, how='left', on='order_id')
+    print('mk_data.shape',mk_data.shape)
+    del head_data
+    gc.collect()
+    
+    print('-------------LOAD DATA for link_data----------------')
+    link_list = file_name(link_data_dir)
+    link_list.sort()
+    link_data = append_all_data(link_list, link_data_dir)
+    # for test running
+    #link_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_170_link_sqe_for_order/sqe_20200825_link.txt')
+    print('The init shape of link_data:', link_data.shape)
+
+    
+    print('-------------LOAD DATA for arrival_sqe_data----------------')
+    arrival_sqe_list = file_name(pre_arrival_sqe_dir)
+    arrival_sqe_list.sort()
+    arrival_sqe_data = append_all_data(arrival_sqe_list, pre_arrival_sqe_dir)
+    #arrival_sqe_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/sqe_arrival_for_link/20200825.csv')  # for test running
+    del arrival_sqe_data['slice_id']
+    arrival_cols = arrival_sqe_data.columns.tolist()
+    new_arrival_cols = ['future_'+i for i in arrival_cols if i != 'order_id']
+    new_arrival_cols.insert(0, 'order_id')
+    arrival_sqe_data.columns = new_arrival_cols
+    print('The init shape of arrival_sqe_data:', arrival_sqe_data.shape)
+    link_data = pd.merge(link_data, arrival_sqe_data, how='left', on='order_id')
+    del arrival_sqe_data
+    gc.collect()
+    
+    """
+    print('-------------LOAD DATA for arrival_link_data----------------')
+    arrival_link_list = file_name(pre_arrival_data_dir)
+    arrival_link_list.sort()
+    arrival_link_data = append_all_data(arrival_link_list, pre_arrival_data_dir)
+    #arrival_link_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/final_pre_arrival_data/sqe_20200825_link.txt')  # for test running
+    print('The init shape of arrival_link_data:', arrival_link_data.shape)
+    link_data = pd.merge(link_data, arrival_link_data, how='left', on='order_id')
+    del arrival_link_data
+    gc.collect()
+    """
+
+    """
+    print('-------------LOAD DATA for h_s_link_data----------------')
+    h_s_link_list = file_name(h_s_for_link_dir)
+    h_s_link_list.sort()
+    h_s_link_data = append_all_data(h_s_link_list,h_s_for_link_dir)
+    #h_s_link_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_hightmp_slice_for_link_eb/20200825_link.txt')  # for test running
+    h_s_link_data = h_s_link_data[['order_id', 'sqe_slice_id', 'sqe_hightemp', 'sqe_weather_le']]
+    print('The init shape of h_s_link_data:', h_s_link_data.shape)
+    link_data = pd.merge(link_data, h_s_link_data, how='left', on='order_id')
+    del h_s_link_data
+    gc.collect()
+    """
+    print('-------------LOAD DATA for link_data_other----------------')
+    link_list_other = file_name(link_data_other_dir)
+    link_list_other.sort()
+    link_data_other = append_all_data(link_list_other, link_data_other_dir)
+    #link_data_other = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/for_0714_link_sqe_for_order_other/sqe_20200825_link.txt')  # for test running
+    print('The init shape of link_data_other:', link_data_other.shape)
+
+    link_data = pd.merge(link_data, link_data_other, on='order_id')
+    # print(link_data.head(0))
+    # del link_data['lk_t_sub_by_min']
+    del_link_cols = ['lk_t_sub_by_min','lk_t_sub_by_q50', 'lk_t_sub_by_min', 'total_linktime_std']
+                      # 'future_pre_arrival_status', 'future_arrive_slice_id']  # 'future_arrive_slice_id'
+    link_data.drop(columns=del_link_cols, axis=1, inplace=True)
+    print('The merge shape of link_data:', link_data.shape)
+    del link_data_other
+    gc.collect()
+
+    print('-------------LOAD DATA for link_data_arrival----------------')
+    if arrival_sqe_data_dir==None:
+        pass
+    else:
+        link_list_arrival = file_name(arrival_sqe_data_dir)
+        link_list_arrival.sort()
+        link_data_arrival = append_all_data(link_list_arrival, arrival_sqe_data_dir)
+        #link_data_arrival = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_170_lk_arrival_sqe_for_order/sqe_20200825_link.txt')  # for test running
+        print('The init shape of link_data_arrival:', link_data_arrival.shape)
+        link_data = pd.merge(link_data, link_data_arrival, on='order_id')
+        print('The merge shape of link_data:', link_data.shape)
+        del link_data_arrival
+        gc.collect()
+
+    link_cols_list = ['link_id', 'link_time', 'link_current_status', 'pr',
+                      'dc', 'link_arrival_status', 'future_pre_arrival_status', 'future_arrive_slice_id']
+
+    data = pd.merge(mk_data, link_data, how='left', on='order_id')
+    del mk_data
+    del link_data
+    gc.collect()
+
+    print('-------------LOAD DATA for arrival_data----------------')
+    if arrival_data_dir==None:
+        pass
+    else:
+        arrival_list = file_name(arrival_data_dir)
+        arrival_list.sort()
+        arrival_data = append_all_data(arrival_list, arrival_data_dir)
+        #arrival_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_link_sqe_for_order_arrival/sqe_20200825_link.txt')
+        arrival_cols = ['order_id', 'lk_arrival_0_percent', 'lk_arrival_1_percent','lk_arrival_2_percent', 'lk_arrival_3_percent', 'lk_arrival_4_percent']
+        #print(arrival_data.head(2))
+        data = pd.merge(data, arrival_data, how='left', on='order_id')
+        del arrival_data
+        gc.collect()
+ 
+    print('-------------LOAD DATA for zsl_arrival_data----------------')
+    if zsl_arrival_data_dir==None:
+        pass
+    else:
+        zsl_arrival_list = file_name(zsl_arrival_data_dir)
+        zsl_arrival_list.sort()
+        zsl_arrival_data = append_all_data(zsl_arrival_list, zsl_arrival_data_dir)
+        #zsl_arrival_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/zsl_arrival/link_fea_arrive_order_id_level_20200818.csv')
+        zsl_arrival_cols = zsl_arrival_data.columns.tolist()
+        zsl_arrival_cols.remove('order_id')
+        #print(zsl_arrival_data.head(2))
+        data = pd.merge(data, zsl_arrival_data, how='left', on='order_id')
+        del zsl_arrival_data
+        gc.collect()
+
+    print('-------------LOAD DATA for cross_data----------------')
+    cross_list = file_name(cross_data_dir)
+    cross_list.sort()
+    cross_data = append_all_data(cross_list, cross_data_dir)
+    # for test running
+    #cross_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/for_0714_cross_sqe_for_order/sqe_20200825_cross.txt')
+    del_cross_cols = ['cr_t_sub_by_min', 'cr_t_sub_by_q50', 'total_crosstime_std']
+    cross_data.drop(columns=del_cross_cols, axis=1, inplace=True)
+    cross_cols_list = ['cross_id', 'cross_time']
+    print('The init shape of cross_data:', cross_data.shape)
+
+    data = pd.merge(data, cross_data, how='left', on='order_id')
+    del cross_data
+    gc.collect()
+    # data['cross_id'] = data['cross_id'].str.replace('nan','0')
+    # print('working..............................')
+
+    mk_cols_list = data.columns.tolist()
+    remove_mk_cols = ['order_id', 'slice_id', 'hightemp', 'lowtemp', 'weather_le', 'dayofweek', 'date_time', 'ata', 'link_arrival_status']
+    mk_cols_list = list(set(mk_cols_list) - set(remove_mk_cols))
+    mk_cols_list = list(set(mk_cols_list) - set(link_cols_list))
+    mk_cols_list = list(set(mk_cols_list) - set(cross_cols_list))
+    if arrival_data_dir==None:
+        pass
+    else:
+        mk_cols_list = list(set(mk_cols_list) - set(arrival_cols))
+        mk_cols_list = list(set(mk_cols_list) - set(zsl_arrival_cols))
+    print('lenght of mk_cols_list', len(mk_cols_list))
+    print('*-' * 40)
+    print('The finish shape of data is:', data.shape)
+
+    return data, mk_cols_list, link_cols_list, cross_cols_list
+
+
+def processing_data(data, link_cols_list, cross_cols_list, mk_cols_list, WIDE_COLS, is_test=False):
+    """
+    fix data, ast.literal_eval + StandardScaler + train_test_split
+    :return: train_data, val_data, test_data
+    """
+    #print('Now, Starting parallel_apply the arrival_status..................')
+    #for i in tqdm(['link_arrival_status']):
+    #    data[i] = data[i].parallel_apply(ast.literal_eval)
+    print('Now, Starting parallel_apply the link..................')
+    for i in tqdm(link_cols_list):
+        data[i] = data[i].parallel_apply(ast.literal_eval)
+    gc.collect()
+    print('Now, Starting parallel_apply the cross..................')
+    for i in tqdm(cross_cols_list):
+        data[i] = data[i].parallel_apply(ast.literal_eval)
+    data = data.fillna(0)
+
+    # train, val
+    if is_test is True:
+        print('is_test is True')
+        ss = joblib.load('../model_h5/ss_scaler')
+        ss_cols = mk_cols_list + WIDE_COLS
+        data[ss_cols] = ss.transform(data[ss_cols])
+        return data
+    else:
+        ss_cols = mk_cols_list + WIDE_COLS
+        ss = StandardScaler()
+        ss.fit(data[ss_cols])
+        data[ss_cols] = ss.transform(data[ss_cols])
+        joblib.dump(ss, '../model_h5/ss_scaler')
+        print('is_test is False')
+        return data
+
+
+def processing_inputs(data, mk_cols_list, link_cols_list, cross_cols_list, WIDE_COLS, arrival=True):
+    """
+    change the data for model
+    :return:
+    """
+    print('*-'*40, processing_inputs)
+    if arrival:
+        mk_cols_list = mk_cols_list +  ['lk_arrival_0_percent', 'lk_arrival_1_percent','lk_arrival_2_percent', 'lk_arrival_3_percent', 'lk_arrival_4_percent']
+        mk_cols_list = mk_cols_list + ['zsl_link_arrival_status_mean','zsl_link_arrival_status_nunique','zsl_link_arrival_status0','zsl_link_arrival_status1','zsl_link_arrival_status2','zsl_link_arrival_status3']
+    if 'lk_arrival_0_percent' in mk_cols_list:
+        print('The lk_arrival_0_percent in the mk_cols_list')
+        #print('*-' * 40, 'EXIT')
+        #sys.exit(0)
+        print('111'*40, 'HAVE FEATURES OF ARRIVAL')
+    else:
+        print('222'*40, 'HAVENOT FEATURES OF ARRIVAL')
+    if 'ata' in mk_cols_list:
+        print('The ata in the mk_cols_list')
+        print('*-' * 40, 'EXIT')
+        sys.exit(0)
+    if 'ata' in link_cols_list:
+        print('The ata in the link_cols_list')
+    if 'ata' in cross_cols_list:
+        print('The ata in the cross_cols_list')
+    if 'ata' in WIDE_COLS:
+        print('The ata in the WIDE_COLS')
+        print('*-' * 40, 'EXIT')
+        sys.exit(0)
+    data_link_inputs = preprocess_inputs(data, cols=link_cols_list)
+    data.drop(columns=link_cols_list, axis=1, inplace=True)
+    gc.collect()
+    print('drop the link_cols_list')
+    # print(data_link_inputs[:, :, :1])
+    # data['cross_id'] = data['cross_id'].str.replace('nan','0')
+    data_cross_inputs = preprocess_inputs(data, cols=cross_cols_list)
+    data.drop(columns=cross_cols_list, axis=1, inplace=True)
+    gc.collect()
+    print('drop the cross_cols_list')
+
+    data_deep_input = data[mk_cols_list]
+    data_wide_input = data[WIDE_COLS].values
+    data_inputs_slice = data['slice_id'].values
+    data_labels = data['ata']
+    if arrival:
+        arrival_col = ['lk_arrival_0_percent', 'lk_arrival_1_percent',
+                        'lk_arrival_2_percent', 'lk_arrival_3_percent', 'lk_arrival_4_percent']
+        data_arrival = data[arrival_col]
+        print('*-'*40, 'data_arrival', data_arrival.shape)
+        return data_link_inputs, data_cross_inputs, data_deep_input, data_wide_input, data_inputs_slice, data_labels, data_arrival
+    else:
+        return data_link_inputs, data_cross_inputs, data_deep_input, data_wide_input, data_inputs_slice, data_labels
+
+
+def split_col(data, columns, fillna=None):
+    '''拆分成列
+
+    :param data: 原始数据
+    :param columns: 拆分的列名
+    :type data: pandas.core.frame.DataFrame
+    :type columns: list
+    '''
+    for c in columns:
+        new_col = data.pop(c)
+        max_len = max(list(map(lambda x:len(x) if isinstance(x, list) else 1, new_col.values)))  # 最大长度
+        new_col = new_col.apply(lambda x: x+[fillna]*(max_len - len(x)) if isinstance(x, list) else [x]+[fillna]*(max_len - 1))  # 补空值，None可换成np.nan
+        new_col = np.array(new_col.tolist()).T  # 转置
+        for i, j in enumerate(new_col):
+            data[c + str(i)] = j
+    return data
+
+def list_to_np(x):
+    return np.array(x)
+
+

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/DCN_12953/dcn_model/test.py

@@ -0,0 +1,3 @@
+import joblib
+cross_le = joblib.load('/data/didi_2021/model_h5/crossid_le')
+print(len(cross_le.classes_.tolist()))

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/DCN_12953/model_for_features/describe_df.csv

@@ -0,0 +1,9 @@
+,order_id,link_current_status_sum,link_current_status_mean,date_time_sum,date_time_mean,wk2_total_linkid_cnt_sum,wk2_total_linkid_cnt_mean,wk2_total_linktime_mean_sum,wk2_total_linktime_mean_mean,wk2_total_linktime_std_sum,wk2_total_linktime_std_mean,wk2_total_linktime_q50_sum,wk2_total_linktime_q50_mean,wk2_total_linktime_skew_sum,wk2_total_linktime_skew_mean,m1_total_linkid_cnt_sum,m1_total_linkid_cnt_mean,m1_total_linktime_mean_sum,m1_total_linktime_mean_mean,m1_total_linktime_std_sum,m1_total_linktime_std_mean,m1_total_linktime_q50_sum,m1_total_linktime_q50_mean,m1_total_linktime_skew_sum,m1_total_linktime_skew_mean,win_6_mean_link_ratio_mean_sum,win_6_mean_link_ratio_mean_mean,win_6_std_link_ratio_mean_sum,win_6_std_link_ratio_mean_mean,win_24_mean_link_ratio_mean_sum,win_24_mean_link_ratio_mean_mean,win_24_std_link_ratio_mean_sum,win_24_std_link_ratio_mean_mean,win_72_mean_link_ratio_mean_sum,win_72_mean_link_ratio_mean_mean,win_72_std_link_ratio_mean_sum,win_72_std_link_ratio_mean_mean,win_6_mean_link_time_std_sum,win_6_mean_link_time_std_mean,win_6_std_link_time_std_sum,win_6_std_link_time_std_mean,win_24_mean_link_time_std_sum,win_24_mean_link_time_std_mean,win_24_std_link_time_std_sum,win_24_std_link_time_std_mean,win_72_mean_link_time_std_sum,win_72_mean_link_time_std_mean,win_72_std_link_time_std_sum,win_72_std_link_time_std_mean,win_6_mean_link_c_status_2_mean_sum,win_6_mean_link_c_status_2_mean_mean,win_6_std_link_c_status_2_mean_sum,win_6_std_link_c_status_2_mean_mean,win_24_mean_link_c_status_2_mean_sum,win_24_mean_link_c_status_2_mean_mean,win_24_std_link_c_status_2_mean_sum,win_24_std_link_c_status_2_mean_mean,win_72_mean_link_c_status_2_mean_sum,win_72_mean_link_c_status_2_mean_mean,win_72_std_link_c_status_2_mean_sum,win_72_std_link_c_status_2_mean_mean,win_6_mean_link_c_status_0_mean_sum,win_6_mean_link_c_status_0_mean_mean,win_6_std_link_c_status_0_mean_sum,win_6_std_link_c_status_0_mean_mean,win_24_mean_link_c_status_0_mean_sum,win_24_mean_link_c_status_0_mean_mean,win_24_std_link_c_status_0_mean_sum,win_24_std_link_c_status_0_mean_mean,win_72_mean_link_c_status_0_mean_sum,win_72_mean_link_c_status_0_mean_mean,win_72_std_link_c_status_0_mean_sum,win_72_std_link_c_status_0_mean_mean,win_6_mean_link_ratio_std_sum,win_6_mean_link_ratio_std_mean,win_6_std_link_ratio_std_sum,win_6_std_link_ratio_std_mean,win_24_mean_link_ratio_std_sum,win_24_mean_link_ratio_std_mean,win_24_std_link_ratio_std_sum,win_24_std_link_ratio_std_mean,win_72_mean_link_ratio_std_sum,win_72_mean_link_ratio_std_mean,win_72_std_link_ratio_std_sum,win_72_std_link_ratio_std_mean,win_6_mean_link_c_status_4_mean_sum,win_6_mean_link_c_status_4_mean_mean,win_6_std_link_c_status_4_mean_sum,win_6_std_link_c_status_4_mean_mean,win_24_mean_link_c_status_4_mean_sum,win_24_mean_link_c_status_4_mean_mean,win_24_std_link_c_status_4_mean_sum,win_24_std_link_c_status_4_mean_mean,win_72_mean_link_c_status_4_mean_sum,win_72_mean_link_c_status_4_mean_mean,win_72_std_link_c_status_4_mean_sum,win_72_std_link_c_status_4_mean_mean,win_6_mean_link_c_status_3_mean_sum,win_6_mean_link_c_status_3_mean_mean,win_6_std_link_c_status_3_mean_sum,win_6_std_link_c_status_3_mean_mean,win_24_mean_link_c_status_3_mean_sum,win_24_mean_link_c_status_3_mean_mean,win_24_std_link_c_status_3_mean_sum,win_24_std_link_c_status_3_mean_mean,win_72_mean_link_c_status_3_mean_sum,win_72_mean_link_c_status_3_mean_mean,win_72_std_link_c_status_3_mean_sum,win_72_std_link_c_status_3_mean_mean,win_6_mean_link_time_mean_sum,win_6_mean_link_time_mean_mean,win_6_std_link_time_mean_sum,win_6_std_link_time_mean_mean,win_24_mean_link_time_mean_sum,win_24_mean_link_time_mean_mean,win_24_std_link_time_mean_sum,win_24_std_link_time_mean_mean,win_72_mean_link_time_mean_sum,win_72_mean_link_time_mean_mean,win_72_std_link_time_mean_sum,win_72_std_link_time_mean_mean,win_6_mean_link_c_status_1_mean_sum,win_6_mean_link_c_status_1_mean_mean,win_6_std_link_c_status_1_mean_sum,win_6_std_link_c_status_1_mean_mean,win_24_mean_link_c_status_1_mean_sum,win_24_mean_link_c_status_1_mean_mean,win_24_std_link_c_status_1_mean_sum,win_24_std_link_c_status_1_mean_mean,win_72_mean_link_c_status_1_mean_sum,win_72_mean_link_c_status_1_mean_mean,win_72_std_link_c_status_1_mean_sum,win_72_std_link_c_status_1_mean_mean,win_6_mean_link_time_mean_skew,win_6_mean_link_time_mean_kurt,win_6_std_link_time_mean_skew,win_6_std_link_time_mean_kurt,ata,distance,simple_eta,slice_id,date_time,link_count,link_time_sum,link_ratio_sum,link_current_status_0,link_current_status_1,link_current_status_2,link_current_status_3,link_current_status_4,link_current_status_0_percent,link_current_status_1_percent,link_current_status_2_percent,link_current_status_3_percent,link_current_status_4_percent,weekday,hightemp,lowtemp,driver_id,pr_sum,top_a_sum,dc_sum,pr_mean,top_a_mean,dc_mean
+count,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158052.0,2158053.0,2158050.0,2158053.0,2158052.0,2158053.0,2158050.0,2158053.0,2158052.0,2158053.0,2158052.0,2158053.0,2158052.0,2158053.0,2158052.0,2158053.0,2158052.0,2158053.0,2158052.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2155184.0,2158053.0,2134872.0,2158053.0,2156917.0,2158053.0,2154391.0,2158053.0,2156924.0,2158053.0,2154496.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2155184.0,2158053.0,2134872.0,2158053.0,2156917.0,2158053.0,2154391.0,2158053.0,2156924.0,2158053.0,2154496.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2158053.0,2158053.0,2158013.0,2158053.0,2157953.0,2157959.0,2157846.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158053.0,2158041.0,2158041.0,2158041.0
+mean,4467661.828779924,89.79694242912477,1.021025083463521,1762768722.0114233,20200827.950593427,624154.5366531777,6499.346030356797,107.18459397707102,1.0457477751229531,595.6869170499464,6.9062763980542075,219.49096299106182,2.500408020714938,553.6435029474947,6.4633324609812535,865959.3644836341,9015.75254131009,93.94376709677661,0.9051677914636824,595.8654238523272,6.907982751186454,220.10615444119156,2.506773281214269,554.0604512002486,6.467212592540564,86.26751497121914,0.9857805874985468,1.629723179224181,0.023088285950048186,86.2615313498214,0.9858256468849965,2.2921654646463776,0.031697170133784404,86.25982621869748,0.9858360552723039,2.6432621611705778,0.03604160623216739,32.34482427864547,0.47375372243773906,28.272010385202297,0.46933956767847557,35.48654766658967,0.47948921217787965,44.80337389038567,0.6112622111731918,35.29664050296828,0.47351498452643714,51.248436166954534,0.6814236392504206,2.1884132478364853,0.02509354505292365,4.772814538391438,0.05525577623637809,2.186381737737402,0.025059645240569674,7.556597415366829,0.08762507453971399,2.1849803960545287,0.025045772120485074,8.914780872226236,0.10319564441022146,2.716482061608397,0.031137421281865082,5.809327599595596,0.06728555413977814,2.7149417001304577,0.031127045930146948,8.925106106240412,0.10366405279263671,2.715227355426387,0.031138485633054862,10.276192924171422,0.1192022983112095,0.9571593590883056,0.015585618096723822,1.0821667471956584,0.019637905293854518,1.0755596533973975,0.015723479661033837,1.9422026272202453,0.027876517711678737,1.074767876113263,0.015616253720558686,2.350282671171,0.032902094523252864,0.3077210024425187,0.0035306000642143153,0.7338806624790072,0.008463081619339036,0.3075325635816437,0.0035297778821450788,1.3670859759231544,0.015708090075820375,0.3068175402582313,0.003524384842123659,1.9657806876813069,0.022415585118994143,1.0691149992787095,0.012251141996505698,2.4505961492000394,0.028280709249226335,1.069187254374715,0.01225896990308865,4.217387437880418,0.04866161994325712,1.0694890817688265,0.012269255753106003,5.415447893201467,0.06227812100125352,599.9173864240867,6.97090541207395,100.1615884459082,1.245840763522761,597.9111310464086,6.935234069860816,137.30607665640738,1.649260435508913,588.8498013828465,6.815035782438041,164.1047804353564,1.9156546651641677,80.98047031226596,0.9279872916047708,11.687974849491923,0.13609502119880584,80.98415836758727,0.9280245610440933,15.4935217431259,0.1810222312837765,80.98568725002217,0.9280221016527614,16.56558713599247,0.19335887261307302,2.33536485789304,9.078896086518606,4.059868542449378,23.06302537211353,842.6362225580187,5284.369708884213,752.9432414310492,110.46978874012825,20200827.950593427,87.26220162340776,599.1405837957179,86.27031101650424,2.718074115881306,80.97677211820098,2.1895407573400654,1.0701697316979704,0.307644900287435,0.03948589002252532,0.9192209479067776,0.025716458780200093,0.011935595165643501,0.0036411081248533426,3.0047084107758244,32.7198660088515,26.59008791721056,40661.525553357584,0.00010176174541111462,2.2065755012762456e-08,0.00031956071163570196,1.1823368755668215e-06,3.343314851055103e-10,3.857914258387011e-06
+std,2581237.303292554,56.22209139934151,0.14625504972700396,1054901630.9379686,2.0000954741654255,614629.7589985116,3961.9966697974405,148.725612987228,0.8555934720159407,388.69000240560234,1.5592031037418341,174.80145586966398,1.0101654877402293,349.48867688903704,1.4093281373151436,871458.025808236,5696.244141676745,137.7234202831471,0.7973411613413249,388.92896117766634,1.558271449873249,175.82555249542474,1.0120971386706274,349.9333556207701,1.4099565180802753,52.05461100411868,0.010548153310221232,0.8888492277686458,0.015946012863514315,52.040192401656256,0.01020160642323602,1.1730516582882629,0.018790068605554978,52.035786639133114,0.01013113503731739,1.3282663524398155,0.019660090990265614,22.876310833480222,0.4010438535858314,23.378189120649033,0.4299086554957893,21.64705520790038,0.300803565109309,30.246784913041736,0.3599028383141747,20.442873524700083,0.2794611107443965,33.85473696451345,0.3513061713738496,1.9524313397950857,0.01760596520970008,4.085913142210754,0.0364826797843522,1.5639936578300029,0.01049823579155561,5.050676161845756,0.02973368685134998,1.4881521347861946,0.008893365188365114,5.659572465965043,0.02657607246857608,2.3285704517373973,0.020968564097641883,4.784833427619452,0.04250484893064781,1.872734261093236,0.012147236656869835,5.819429945417835,0.032874660519638485,1.7856592840709156,0.01013879359298155,6.400675198973619,0.028591846014648172,0.6367079111344864,0.017026743290626038,0.8168701201349867,0.02039527985911189,0.6277664194106983,0.01252746189605709,1.1562911159393385,0.0167885288108576,0.6144523343093371,0.012134985331597715,1.3803105223039622,0.017738039160800124,0.5443486629757853,0.006422758922209785,1.2603117780499207,0.014910705764192856,0.354978371177164,0.0038579807121240425,1.4231160954287998,0.014532789266245983,0.31047767869002585,0.003274224837347187,1.6756464566603981,0.014547393992850013,1.2726971788461279,0.013396180349810088,2.816569882444775,0.029570862252902484,0.9197660977852785,0.008147282752955901,3.328667547641834,0.026734637611970674,0.8433565866740418,0.006956191886678332,3.81740731640289,0.024870714495344774,403.23658305367667,1.9320675477872138,78.92258028084426,0.7321167744332737,400.1521481344652,1.8306190051352047,117.23688513082699,0.9472035308645873,392.28458679314593,1.6797448874169334,141.93131140776893,1.0188544628898286,48.5480025817039,0.03401883140845069,8.235677786351873,0.05754907233850395,48.49756542651297,0.02024678050847428,9.569537353339724,0.041187950002267104,48.49434258263024,0.017135690063167088,9.965567727400979,0.03543626885777821,1.3316270272939412,12.903202176440441,1.8747926394961851,22.844461620805273,539.209212443296,4421.50107737492,471.7511945641532,70.75252019456302,2.0000954741654255,52.220712556557764,405.57210046623237,52.21783274310701,4.876658271111734,50.252202999919845,4.347795477801296,3.007631459171947,1.534675171866052,0.08009974606950314,0.1064887060113878,0.05153732956191985,0.03475095978324956,0.019943464660937056,1.9894980687530581,1.2752993259756793,0.9129176751379382,23436.39340102212,6.0652077088920155e-05,3.241527063004052e-05,0.00016999403323419705,3.0072626231287654e-08,4.911418296071371e-07,5.784095168169497e-07
+min,4.0,0.0,0.0,60602475.0,20200825.0,0.0,1.0,0.0,0.0,0.0,2.17470555291436,0.0,0.3458229496396688,0.0,2.095389473684211,0.0,2.0,0.0,0.0,0.0,2.1397137454977075,0.0,0.0,0.0,2.0856052631578947,1.3359999999997514,0.4453333333332505,0.0,0.0,1.335999999999535,0.4453333333331782,0.0,0.0,1.3359999999981595,0.4453333333327199,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,5.921189464667503e-16,1.1842378929335003e-16,0.0,0.0,5.282811225508035e-15,1.0565622451016074e-15,0.0,0.0,1.2209688683898407e-14,3.0524221709746018e-15,0.0,0.0,0.0,0.0,0.0,0.0,4.403884664346455e-15,1.4679615547821515e-15,0.0,0.0,3.069043335966454e-14,8.505012636996306e-15,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,6.664001874625056e-08,0.0,0.0,0.0,0.0,1.3061447348531252e-17,4.353815782843749e-18,0.0,0.0,0.0,0.0,0.0,1.7277923356419887e-08,0.0,0.0,0.0,0.0,2.411445230458202e-15,7.986824149649325e-16,0.0,0.0,6.637086666699785,1.897275000010844,0.0,9.282598202400428e-06,6.364615902667396,1.9266249999507856,0.0,1.8778265732536302e-05,6.278358393207272,1.9266249999811784,0.0,0.0,0.5000000000052536,0.05555555555613929,0.0,0.0,0.4999999999998703,0.05555555555554115,0.0,0.0,0.5000000000199729,0.05555555555777477,0.0,0.0,-2.1337138450061683,-5.830538975815136,-1.8952390224981912,-5.8487303528367995,20.0,23.8588,4.0,0.0,20200825.0,3.0,3.8263,1.1001,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,31.0,25.0,0.0,0.0,0.0,0.0,4.912669039568308e-07,8.1860558413547e-129,1.1328659128576882e-06
+25%,2230851.0,51.0,0.9767441860465116,1030242279.0,20200826.0,205685.0,3474.857142857143,28.3363,0.47232611019736837,339.6437964072864,5.815868689923237,108.20547817336457,1.777201964361006,321.82744999999994,5.487931228813561,277216.0,4659.159314997405,22.762,0.3774791698042168,339.77624420734804,5.817776693333093,108.43692552317074,1.780004878591055,322.0099,5.4912554484657745,50.27611428571368,0.9819328703703776,0.979388859137162,0.012403903130703563,50.278291147342145,0.9820690775187172,1.4381333820002995,0.01870932904183898,50.280706485370274,0.98210970555968,1.6806382799676816,0.02230874852100711,16.709101973327776,0.27330065019641125,12.619085375513382,0.26335494007425786,20.451513166820085,0.3068681602520748,24.414814997477627,0.3938367197009818,20.90553806416662,0.31753550224208715,28.114965955950705,0.46315697269937695,0.8611111111113469,0.013509517044876835,1.9832817933757283,0.031105321937720105,1.1180555555558611,0.018494392865468345,4.083578218715672,0.06792716694009349,1.173866642616915,0.019990287425783824,5.029220056904682,0.0866661496934262,1.1027777777780414,0.01740855762595669,2.468917609115848,0.03928003877000085,1.4234508547027809,0.023897058823549155,4.912895352750648,0.0819732404849539,1.4932573082220677,0.02603677289782749,5.879612063044936,0.10178526423769467,0.4882117784758322,0.00705133094029778,0.4726768690452236,0.008937249467187117,0.6163292081421146,0.00836302235936949,1.0999065231759817,0.016644730887115884,0.6259505925262197,0.008526191491351198,1.349065381931176,0.020749189063212055,4.6259292692714846e-18,7.542275982507856e-20,2.271794648428364e-05,3.0798244857427137e-07,0.08333333333333791,0.0012856606606607606,0.408248670398549,0.006103385039394396,0.11653777333890715,0.0018164036852762481,0.7989411439158044,0.012747767055173342,0.2500000000000033,0.0037165920499253853,0.6094989545187619,0.008980303892072869,0.4525793650795844,0.007108815748523822,1.9153313781097687,0.03048351476135985,0.5091275542784822,0.008239781021901555,2.7856478817563377,0.04630535845257015,337.2363405556818,5.660815226339265,52.96136168335824,0.7702210480120035,336.15935905744567,5.675979704802313,67.96799929420226,1.0270927477588976,331.61806441653164,5.640583239532125,78.3928237540531,1.2339830600323882,47.37619047620034,0.9110566448803502,6.014553635301671,0.09754040508672274,47.40624999999854,0.9176829443535556,8.951570748208589,0.15256983405572813,47.416555939452856,0.9201269859575428,9.737989182900733,0.17051456854060848,1.4685889212871015,2.126323009284821,2.6925383548655297,8.068737159435525,478.0,2605.3329,436.0,53.0,20200826.0,51.0,335.04010000000005,50.1132,0.0,46.0,0.0,0.0,0.0,0.0,0.8842105263157894,0.0,0.0,0.0,1.0,31.0,26.0,20260.0,5.954728164955193e-05,9.086239813771446e-57,0.0001993844006629532,1.1629437413702528e-06,1.3168462605025118e-58,3.4767264222184227e-06
+50%,4467339.0,76.0,1.0,1515061950.0,20200828.0,433256.0,5767.864156904041,58.05329999999998,0.7957164215686274,494.2322703222725,6.67347392710202,168.78651287466317,2.31247321696397,464.407,6.250633848110464,590755.0,7844.014312001635,48.49329999999999,0.6671349159663865,494.4706774436566,6.676522370589291,169.24212567143655,2.320119195107122,464.68620000000016,6.254179732142859,73.76020833333409,0.9883962779323932,1.4867696109808102,0.01929285119279588,73.75434187514881,0.9883425051824336,2.1158552945347022,0.027700763064297508,73.75386837085969,0.9883385481145052,2.4444900653626003,0.03224175617064338,27.13279513640625,0.3896772348330073,22.542426345425945,0.3799487938948173,30.94151554768452,0.414667523444552,37.84928774276318,0.5353952632390584,31.23884428401885,0.416266193551009,43.59718783480334,0.6147924701915504,1.6722222222223957,0.021666666666674387,3.7309301692769887,0.04841752740281116,1.8128472222232368,0.023814304656720325,6.389866001708138,0.0843178833818178,1.8358346223088944,0.0241726051643738,7.631691634423714,0.10203241694035343,2.117063492063893,0.027276658526667046,4.617662884007031,0.05955081524398804,2.2809576023402616,0.02985585418840136,7.601259256814535,0.09996899620616988,2.30828905116646,0.030360417332279413,8.839805871550805,0.11812998990196735,0.8546704224558319,0.012231256351527569,0.9315375377958406,0.015967809648766637,0.9772547209040668,0.013157311667436423,1.75773729253201,0.025255066754827332,0.9807085813349536,0.013167377676561301,2.130451596259153,0.030561507570116903,0.12135225885225875,0.0014367816091954025,0.2907728532024817,0.0035103362400936517,0.20833333333333456,0.002635458388963082,0.9568523689316,0.012222438887353445,0.2306769722814549,0.002937687176992789,1.552145060781108,0.02013406092866357,0.670634920634921,0.008662280701754389,1.6246951927926214,0.02043738244014944,0.8333333333355979,0.010817519671692971,3.403859455166467,0.04446319249791636,0.8632975933900222,0.011234769725676684,4.543957209511522,0.06020609326712119,493.7307075135082,6.618523599037324,79.67887168625684,1.0650262246483748,492.1270475296155,6.620199074036398,104.1307423983003,1.4113494937854725,485.07732112027253,6.548122094642067,122.4743623818542,1.6651788473658569,69.37142857144767,0.9326538022621322,9.765195482927588,0.12781605355585496,69.38073306524858,0.9297594997595512,13.267676552206972,0.17776051580059546,69.39029087460779,0.9294056585462414,14.269547942587895,0.19262633511139346,1.9902920495165213,4.8701848327743775,3.687167184165265,15.971061036266175,711.0,3998.0657,638.0,108.0,20200828.0,75.0,493.26730000000015,73.7887,0.0,69.0,0.0,0.0,0.0,0.0,0.9553571428571428,0.0,0.0,0.0,3.0,33.0,27.0,40720.0,8.738930245060761e-05,1.694235286004553e-52,0.00028434934412727966,1.181542843062831e-06,2.0660297849084206e-54,3.7222737136752616e-06
+75%,6702292.0,113.0,1.0517241379310345,2201890252.0,20200830.0,836075.0,8755.672791023842,121.34719999999996,1.3508233840090091,726.9779839729974,7.735142528142925,269.6808714312547,3.0083073198680204,676.3673000000006,7.188994403549806,1151752.0,12208.812500000002,105.0834,1.172454580965909,727.157983600573,7.734032176380113,270.20524531723754,3.020564790610264,676.8649000000001,7.191899318181816,107.9454466666678,0.9926959603174672,2.122802364669244,0.02937599210285965,107.9281410702594,0.9925254853692914,2.9482030921100244,0.0401390437966665,107.9237978946198,0.992487276487212,3.3828684785704284,0.04547914478255236,41.91231846263072,0.5666838023646997,37.15657896674668,0.5591796791052369,45.22371102810958,0.5714149543580508,56.97615683653269,0.737291421500644,45.018300517202455,0.5568752610074897,65.46225668232057,0.8188830643927043,2.9189935064939045,0.032406204906211086,6.36786862407014,0.07093333836130881,2.827876984128165,0.03004906204908607,9.691017330345383,0.10381027420926367,2.794971174388203,0.028983861187507388,11.334631443672912,0.11885620014477424,3.6527777777780552,0.03991317324651103,7.780316871226907,0.0857625904492023,3.5119093631380625,0.03662080170234478,11.418501380687182,0.12158474194211545,3.4681295015619487,0.035023591103002745,13.02561015204931,0.13574932260386796,1.3134547131880725,0.019627548660024093,1.5244120054914725,0.02523329018374914,1.4268346460672607,0.0198531458053726,2.5793708996515687,0.03584161222977891,1.419124025783959,0.019628545228104914,3.1041159421573816,0.042090987299988614,0.3793650793650793,0.004356725146198829,0.9153470348998308,0.010531944125844487,0.4145066738816767,0.0046483650278293384,1.8663900011448225,0.02108061764957301,0.4067881316616704,0.004457311542052089,2.6607921592049735,0.02933698228580314,1.4343434343434482,0.016330631174381202,3.305088829999343,0.03789201197922795,1.4139767237599106,0.0156510114208365,5.58418045097852,0.062020888542653085,1.3921941677154566,0.015007658638304847,7.049747314390178,0.07621419026443862,733.1517577781668,7.854667261909086,121.41262767730957,1.5045892172513544,730.5865643407174,7.815524434993155,164.50988647992375,1.9867848111973208,719.0092481294794,7.672567858933575,197.7154213533789,2.3044918541650605,101.25000000002508,0.950670498084474,15.155434542593962,0.16561187213529918,101.23043311464345,0.9405446023380144,19.45937513233536,0.20623813509317893,101.22696716681679,0.9375770547807696,20.759019725979766,0.21519788472673207,2.790041195854423,10.468515762031824,5.050355672251803,30.15906892256812,1053.0,6314.265,932.0,156.0,20200830.0,109.0,734.1738999999999,108.0498,4.0,102.0,3.0,0.0,0.0,0.04444444444444445,1.0,0.031746031746031744,0.0,0.0,5.0,34.0,27.0,61048.0,0.00012750419506523227,1.6675588380191983e-38,0.0003987688013259063,1.200405034422123e-06,3.2696647555142363e-40,4.095745992639334e-06
+max,8939076.0,886.0,4.0,17251507966.0,20200831.0,9655801.0,36682.90909090909,3758.3051000000014,52.56945,7286.840813058084,62.63686448863637,3381.0825514783064,30.102298973558373,6563.775749999998,65.1287,11567813.0,44427.0,3762.912499999999,34.30173333333333,7265.973156940879,46.03208501725014,3344.796271598372,26.750360388532,6587.964799999998,42.63668333333335,852.6967666667238,1.0000000000000226,12.397266095578713,0.7071067811866291,852.7670052081507,1.0000000000000546,16.586744465229767,0.7071067811865516,852.7295362648924,0.999994359461974,18.97063621955413,0.7071067811864706,621.2501128994347,92.09507210597572,449.7250809669818,55.42933510377009,398.77833800087006,62.890854936191616,558.9131250501301,42.146949351928185,297.86909315126974,62.89085493619334,564.8715556410666,42.146949351928185,76.900000000001,0.6266666666667025,146.38085593246842,0.7071067811854831,75.0267586580335,0.6250000000003051,105.5159934472998,0.7071067811858912,75.02675865808105,0.6250000000006145,98.57628707455412,0.7071067811869653,82.44444444446353,0.9444444444444748,120.51594431648823,0.7071067811865089,83.4133370286567,0.9444444444447532,102.83997572680616,0.7071067811867108,83.47885744386271,0.9444444444445558,104.20795475175535,0.7071067811871112,7.45498616664199,0.7071067811865834,10.520400683882352,0.5000000000000001,7.523452550093473,0.7071067811866325,15.110032660324507,0.5000000000000001,7.969073249295636,0.7071067811867913,18.31832179490479,0.5,67.73333333333335,0.5480769230769232,76.91332319931284,0.5474634450698099,54.08564997804141,0.5271978021978022,57.98518361298847,0.5195088810097401,53.926670237654626,0.5271978021978027,55.104165813215005,0.5195088810092535,62.9166666666671,0.6666666666666744,76.58503721269176,0.655330085890308,60.0863663812121,0.6666666666668801,60.20179293528384,0.6553300858903782,60.061490187167614,0.6666666666670619,65.81164496039031,0.6553300858898297,10189.878026115346,64.6140030555572,2783.2529650442625,35.14264133965888,9917.094191361286,57.5364663657382,3061.2219767913593,35.14264133921829,9679.55107990508,51.043200988758926,3232.9093910871306,35.142641340007216,765.9777777784097,1.0000000000033051,174.1045305461332,0.7071067811866446,775.2302723589679,1.0000000000014495,150.53067156830278,0.7071067811864671,779.13733759429,1.000000000114923,154.65445308213145,0.7071067811897359,17.599990578162238,382.0302403042582,23.62526066059123,566.2255389010735,10012.0,117909.7931,10601.0,287.0,20200831.0,854.0,10209.1048,852.9918,187.0,848.0,76.0,81.0,55.0,1.0,1.0,1.0,1.0,1.0,6.0,34.0,28.0,80885.0,0.0008704737550512393,0.047619047590635885,0.0024333959808183142,1.775180150693991e-06,0.0007215007210702408,9.818171244766632e-06

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/DCN_12953/model_for_features/missing_values_table.csv

@@ -0,0 +1,50 @@
+,Missing Values,% of Total Values
+win_6_std_link_ratio_std_mean,23181,1.1
+win_6_std_link_time_std_mean,23181,1.1
+win_24_std_link_ratio_std_mean,3662,0.2
+win_24_std_link_time_std_mean,3662,0.2
+win_72_std_link_ratio_std_mean,3557,0.2
+win_72_std_link_time_std_mean,3557,0.2
+win_6_mean_link_time_std_mean,2869,0.1
+win_6_mean_link_ratio_std_mean,2869,0.1
+win_24_mean_link_time_std_mean,1136,0.1
+win_24_mean_link_ratio_std_mean,1136,0.1
+win_72_mean_link_ratio_std_mean,1129,0.1
+win_72_mean_link_time_std_mean,1129,0.1
+win_6_std_link_time_mean_kurt,207,0.0
+win_6_mean_link_time_mean_kurt,100,0.0
+win_6_std_link_time_mean_skew,94,0.0
+win_72_std_link_c_status_4_mean_mean,40,0.0
+win_24_std_link_c_status_4_mean_mean,40,0.0
+win_6_std_link_c_status_4_mean_mean,40,0.0
+win_72_std_link_time_mean_mean,40,0.0
+win_6_std_link_c_status_3_mean_mean,40,0.0
+win_24_std_link_c_status_3_mean_mean,40,0.0
+win_72_std_link_c_status_3_mean_mean,40,0.0
+win_72_std_link_c_status_0_mean_mean,40,0.0
+win_6_std_link_c_status_2_mean_mean,40,0.0
+win_24_std_link_c_status_0_mean_mean,40,0.0
+win_6_std_link_c_status_0_mean_mean,40,0.0
+win_72_std_link_c_status_2_mean_mean,40,0.0
+win_24_std_link_c_status_2_mean_mean,40,0.0
+win_24_std_link_time_mean_mean,40,0.0
+win_72_std_link_ratio_mean_mean,40,0.0
+win_24_std_link_ratio_mean_mean,40,0.0
+win_6_std_link_ratio_mean_mean,40,0.0
+win_6_std_link_c_status_1_mean_mean,40,0.0
+win_24_std_link_c_status_1_mean_mean,40,0.0
+win_72_std_link_c_status_1_mean_mean,40,0.0
+win_6_std_link_time_mean_mean,40,0.0
+pr_mean,12,0.0
+top_a_mean,12,0.0
+dc_mean,12,0.0
+wk2_total_linktime_mean_mean,3,0.0
+wk2_total_linktime_q50_mean,3,0.0
+m1_total_linktime_skew_mean,1,0.0
+m1_total_linktime_q50_mean,1,0.0
+m1_total_linktime_std_mean,1,0.0
+m1_total_linktime_mean_mean,1,0.0
+m1_total_linkid_cnt_mean,1,0.0
+wk2_total_linktime_skew_mean,1,0.0
+wk2_total_linktime_std_mean,1,0.0
+wk2_total_linkid_cnt_mean,1,0.0

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/DCN_12953/model_for_features/model_lgb_for_features.py

@@ -0,0 +1,181 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import gc
+import os
+import lightgbm as lgb
+from sklearn.model_selection import StratifiedKFold,KFold
+from sklearn.model_selection import train_test_split
+
+
+def append_all_data(files_list, file_head_path):
+    """
+    concat all the data
+    :param files_list: the name of data
+    :param file_head_path: the path of data
+    :return: DataFrame of data for all
+    """
+    data_all_path = file_head_path + files_list[0]
+    data_all = pd.read_csv(data_all_path)
+    data_all = data_all.head(0)
+    try:
+        del data_all['Unnamed: 0']
+    except KeyError as e:
+        pass
+    # 循环添加全部数据
+    for i in files_list:
+        data_path = file_head_path + i
+        print("当前文件为：", data_path)
+        data = pd.read_csv(data_path)
+        try:
+            del data['Unnamed: 0']
+        except KeyError as e:
+            pass
+        data_all = data_all.append(data)
+    return data_all
+
+
+def file_name(file_dir):
+    files_list = []
+    for root, dirs, files in os.walk(file_dir):
+        # print("success")
+        for name in files:
+            files_list.append(name)
+    return files_list
+
+
+def del_str_in_list(lst, del_str):
+    a = []
+    for i in range(len(lst)):
+        if del_str not in lst[i]:
+            a.append(lst[i])
+    return a
+
+
+# 自定义lgb评估指标
+def lgb_score_mape(preds, train_data):
+    labels = train_data.get_label()
+    diff = np.abs(np.array(preds) - np.array(labels))
+    result = np.mean(diff / labels)
+    return 'mape',result, False
+
+# 评估指标
+def MAPE(true, pred):
+    diff = np.abs(np.array(pred) - np.array(true))
+    return np.mean(diff / true)
+
+# Function to calculate missing values by column
+def missing_values_table(df):
+        # Total missing values
+    mis_val = df.isnull().sum()
+        
+        # Percentage of missing values
+    mis_val_percent = 100 * df.isnull().sum() / len(df)
+   
+        # Make a table with the results
+    mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)
+        
+        # Rename the columns
+    mis_val_table_ren_columns = mis_val_table.rename(
+    columns = {0 : 'Missing Values', 1 : '% of Total Values'})
+        
+        # Sort the table by percentage of missing descending
+    mis_val_table_ren_columns = mis_val_table_ren_columns[
+        mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(
+        '% of Total Values', ascending=False).round(1)
+        
+        # Print some summary information
+    #print ("Your selected dataframe has " + str(df.shape[1]) + " columns.\n"      
+    #    "There are " + str(mis_val_table_ren_columns.shape[0]) +
+    #      " columns that have missing values.")
+        
+       
+        # Return the dataframe with missing information
+    return mis_val_table_ren_columns
+
+
+def model_fit(train_x, train_y):
+    evals_result = {}
+    params = {
+            'boosting_type': 'gbdt',
+            'objective': 'regression',  # 回归目标
+            #'metric': {'binary_logloss,auc'},
+            #'max_depth':-1,
+            'num_leaves': 30,
+            'learning_rate': 0.07,
+            #'min_child_samples':21,
+            #'min_child_weight':0.001,
+            #'feature_fraction': 0.7,
+            #'bagging_fraction': 0.6,
+            #'bagging_freq': 2,
+            #'min_split_gain':0.5,
+            'verbose': 0,
+            #'is_unbalenced':True,
+        }
+
+
+    n_fold=5
+    folds = StratifiedKFold(n_splits=n_fold, shuffle=True, random_state=42)
+    # gkf = GroupKFold(n_splits=n_fold)
+    toof = np.zeros((train_x.shape[0], ))
+    for fold_, (trn_idx, val_idx) in enumerate(folds.split(train_x,train_y)):  # 5折训练
+        print("fold {}".format(fold_ + 1))
+        trn_data = lgb.Dataset(train_x.iloc[trn_idx], label=train_y.iloc[trn_idx])
+        val_data = lgb.Dataset(train_x.iloc[val_idx], label=train_y.iloc[val_idx])
+
+        clf = lgb.train(params,
+                        trn_data,
+                        valid_sets=[trn_data, val_data],
+                        valid_names=['train', 'val'],
+                        verbose_eval=10,
+                        feval=lgb_score_mape,
+                        #categorical_feature=[],
+                        evals_result=evals_result,
+                        early_stopping_rounds=20,
+                        num_boost_round = 1000
+                        )
+        toof[val_idx] = clf.predict(train_x.iloc[val_idx], num_iteration=clf.best_iteration)
+
+        #print('拟合情况:')
+        #lgb.plot_metric(evals_result)
+        #plt.show()
+        mape_vale = MAPE(train_y.iloc[val_idx],toof[val_idx])
+        print("当前MAPE值为：",mape_vale)
+
+        print('画特征重要性排序...')
+        plt.figure(figsize=(10, 30))
+        ax = lgb.plot_importance(clf, figsize=(10,30))#max_features表示最多展示出前10个重要性特征，可以自行设置
+        plt.savefig("features_importance.png", dpi=500, bbox_inches='tight') 
+
+    return clf
+
+
+if __name__=='__main__':
+    making_data_dir = '/home/didi2021/didi2021/giscup_2021/order_xt/'
+    mk_list = file_name(making_data_dir)
+    mk_list.sort()
+    mk_data = append_all_data(mk_list, making_data_dir)
+    print(mk_data.shape)
+    mk_data['date_time'] = mk_data['date_time'].astype(int)
+    mk_data = mk_data[mk_data['date_time']!=20200901]
+    print(mk_data.shape)
+    describe_df = mk_data.describe()
+    describe_df.to_csv('describe_df.csv')
+    print('*-'*40, 'missing_values_table')
+    ms_table = missing_values_table(mk_data)
+    ms_table.to_csv('missing_values_table.csv')
+    train_y = mk_data['ata']
+    train_x = mk_data.drop(['ata','weather','date_time_dt','order_id','driver_id','date_time'],axis=1)
+    print(train_y)
+    print('*-'*40)
+    print(train_x.head(5))
+    print('*-'*40, 'model_fit')
+    model = model_fit(train_x, train_y)
+    print('................FINISH')
+
+
+
+
+
+
+

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/README.md

@@ -0,0 +1,33 @@
+# README
+
+7th/Top1%，提供答疑
+
+![1628602069041](assets/1628602069041.png)
+
+也能做到前5，但是没必要
+
+![1628602545539](assets/1628602545539.png)
+
+<https://www.biendata.xyz/competition/didi-eta/>
+
+### 推荐工具
+
+- [智能钛Notebook-2.4.0-tf](https://console.cloud.tencent.com/tione/notebook/instance)
+- [腾讯云服务器](https://console.cloud.tencent.com/cvm/instance/index)
+
+
+
+### How to run the code
+
+- DCN模型
+- WDR模型
+- LGB模型
+
+
+
+### 说明
+
+- 数据来自滴滴出行，英文（Data source: Didi Chuxing），数据出处：[https://gaia.didichuxing.com](https://gaia.didichuxing.com/)
+- 代码属于公司所有，不能提供最优代码
+
+### 

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/WD_128544/wd_model/main.py

@@ -0,0 +1,154 @@
+import pandas as pd
+import numpy as np
+import gc
+import process
+import wd_model
+import time
+
+
+RANDOM_SEED = 42
+
+# types of columns of the data_set DataFrame
+WIDE_COLS = [
+    'weather_le', 'hightemp', 'lowtemp', 'dayofweek'
+]
+
+if __name__ == '__main__':
+    t1 = time.time()
+    print(wd_model.get_available_gpus())  # 返回格式为：['/device:GPU:0', '/device:GPU:1']
+
+    # LOAD DATA
+    print('*-' * 40, 'LOAD DATA')
+    making_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_order_xt/'
+    link_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_170_link_sqe_for_order/'
+    cross_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/for_0714_cross_sqe_for_order/'
+    head_link_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_head_link_data_clear/'
+    win_order_data_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/win_order_xw/'
+    pre_arrival_sqe_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/sqe_arrival_for_link/'
+    data_for_driver_xw = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/data_for_driver_xw/'
+    downstream_status_dir = '/home/didi2021/didi2021/giscup_2021/final_train_data_0703/downstream_status_for_order/'
+    data, mk_cols_list, link_cols_list, cross_cols_list = process.load_data(making_data_dir,
+                                                                            link_data_dir,
+                                                                            cross_data_dir,
+                                                                            head_link_dir,
+                                                                            win_order_data_dir,
+                                                                            pre_arrival_sqe_dir,
+                                                                            data_for_driver_xw,
+                                                                            downstream_status_dir)
+
+    # PROCESSING DATA
+    print('*-' * 40, 'PROCESSING DATA')
+    train_data, val_data = process.processing_data(data, mk_cols_list, link_cols_list, cross_cols_list,
+                                                              WIDE_COLS)
+    del data
+    gc.collect()
+    # print(train_data.columns.tolist())
+
+    # PROCESSING INPUTS
+    print('*-' * 40, 'PROCESSING INPUTS')
+    # SAVE LIST
+    a = np.array(mk_cols_list)
+    np.save('../model_h5/wd_mk_cols_list_0730_5.npy', a)
+    a = np.array(link_cols_list)
+    np.save('../model_h5/wd_link_cols_list_0730_5.npy', a)
+    a = np.array(cross_cols_list)
+    np.save('../model_h5/wd_cross_cols_list_0730_5.npy', cross_cols_list)
+    pred_cols = ['ata']
+    print('*-' * 40, 'PROCESSING INPUTS FOR TRAIN_DATA', train_data.shape)
+    train_link_inputs, train_cross_inputs, train_deep_input, train_wide_input, \
+    train_inputs_slice, train_labels = process.processing_inputs(
+        train_data, mk_cols_list, link_cols_list, cross_cols_list, WIDE_COLS)
+    del train_data
+    gc.collect()
+
+    print('*-' * 40, 'PROCESSING INPUTS FOR VAL_DATA', val_data.shape)
+    val_link_inputs, val_cross_inputs, val_deep_input, val_wide_input, \
+    val_inputs_slice, val_labels = process.processing_inputs(
+        val_data, mk_cols_list, link_cols_list, cross_cols_list, WIDE_COLS)
+    del val_data
+    gc.collect()
+
+
+    # MODEL_INIT
+    print('*-' * 40, 'MODEL_INIT')
+    deep_col_len, wide_col_len = train_deep_input.shape[1], train_wide_input.shape[1]
+    link_nf_size, cross_nf_size = train_link_inputs.shape[2], train_cross_inputs.shape[2]
+    link_size = 639877 + 2
+    cross_size = 44313 + 2
+    slice_size = 288
+    # link_seqlen, cross_seqlen = 170, 12  # 已默认
+    print("link_size:{},link_nf_size:{},cross_size:{},cross_nf_size:{},slice_size:{}".format(link_size, link_nf_size,
+                                                                                             cross_size, cross_nf_size,
+                                                                                             slice_size))
+    print("deep_col_len:{}, wide_col_len:{}".format(deep_col_len, wide_col_len))
+
+    model = wd_model.wd_model(link_size, cross_size, slice_size, deep_col_len, wide_col_len,
+                              link_nf_size, cross_nf_size, conv='conv')
+
+    mc, es, lr = wd_model.get_mc_es_lr('0730_5', patience=4, min_delta=1e-4)
+    print('*-' * 40, 'MODEL_INIT END')
+    # MODEL_FIT
+    print('*-' * 40, 'MODEL_FIT_PREDICT')
+    history = model.fit(
+        [train_link_inputs, train_cross_inputs, train_deep_input, train_wide_input, train_inputs_slice], train_labels,
+        validation_data=(
+            [val_link_inputs, val_cross_inputs, val_deep_input, val_wide_input, val_inputs_slice], val_labels),
+        batch_size=2048,  # 2048,256
+        epochs=100,
+        verbose=1,
+        callbacks=[es])
+    np.save('../model_h5/history_0730_5.npy', history.history)
+    model.save_weights("../model_h5/wd_model_0730_5.h5")
+
+    del train_link_inputs, train_cross_inputs, train_deep_input, \
+        train_wide_input, train_inputs_slice, train_labels
+    del val_link_inputs, val_cross_inputs, val_deep_input, val_wide_input, val_inputs_slice, val_labels
+    gc.collect()
+
+    print('*-' * 40, 'LOAD TEST DATA')
+    making_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/order_xt/'
+    link_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/max_170_link_sqe_for_order/'
+    cross_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/cross_sqe_for_order/'
+    head_link_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/head_link_data_clear/'
+    win_order_test_data_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/win_order_xw/'
+    pre_arrival_sqe_test_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/sqe_arrival_for_link/'
+    data_test_for_driver_xw = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/data_for_driver_xw/'
+    downstream_status_test_dir = '/home/didi2021/didi2021/giscup_2021/final_test_data_0703/downstream_status_for_order/'
+    test_data, _, _, _ = process.load_data(making_data_dir,
+                                                                            link_data_dir,
+                                                                            cross_data_dir,
+                                                                            head_link_dir,
+                                                                            win_order_test_data_dir,
+                                                                            pre_arrival_sqe_test_dir,
+                                                                            data_test_for_driver_xw,
+                                                                            downstream_status_test_dir)
+
+    # PROCESSING DATA
+    print('*-' * 40, 'PROCESSING DATA')
+    test_data = process.processing_data(test_data, mk_cols_list, link_cols_list, cross_cols_list,
+                                                              WIDE_COLS, is_test=True)
+    print('*-' * 40, 'PROCESSING INPUTS FOR TEST_DATA', test_data.shape)
+    test_link_inputs, test_cross_inputs, test_deep_input, test_wide_input, \
+    test_inputs_slice, test_labels = process.processing_inputs(
+        test_data, mk_cols_list, link_cols_list, cross_cols_list, WIDE_COLS)
+    test_pre = test_data[['order_id']]
+    del test_data
+    gc.collect()
+
+    # MODEL_RPEDICT
+    print('*-' * 40, 'MODEL_RPEDICT')
+    test_pre = test_pre.rename(columns={'order_id': 'id'})
+    test_pred = model.predict(
+        [test_link_inputs, test_cross_inputs, test_deep_input, test_wide_input, test_inputs_slice],
+        batch_size=2048)
+    test_pre['test_predict'] = test_pred
+    # test_pre['test_predict'] = test_pre['test_predict'].round(0)
+    test_pre = test_pre.rename(columns={'test_predict': 'result'})  # 更改列名
+    test_pre = test_pre[['id', 'result']]
+    print(test_pre.head())
+    result_save_path = '../result_csv/submit_w_0730_5.csv'
+    print('*-' * 40, 'CSV_SAVE_PATH:', result_save_path)
+    test_pre.to_csv(result_save_path, index=0)  # 保存
+    print('..........Finish')
+    t2 = time.time()
+    print("Total time spent: {:.4f}".format((t2-t1)/3600))

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/WD_128544/wd_model/process.py

@@ -0,0 +1,312 @@
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+from tqdm import tqdm
+from pandarallel import pandarallel
+from sklearn.model_selection import train_test_split
+# import random
+import gc
+import ast
+import os
+import warnings
+import joblib
+
+
+warnings.filterwarnings('ignore')
+pd.options.mode.chained_assignment = None
+pandarallel.initialize()
+
+
+def pandas_list_to_array(df):
+    """
+    Input: DataFrame of shape (x, y), containing list of length l
+    Return: np.array of shape (x, l, y)
+    """
+
+    return np.transpose(
+        np.array(df.values.tolist()),
+        (0, 2, 1)
+    )
+
+
+def preprocess_inputs(df, cols: list):
+    return pandas_list_to_array(
+        df[cols]
+    )
+
+
+def append_all_data(files_list, file_head_path):
+    """
+    concat all the data
+    :param files_list: the name of data
+    :param file_head_path: the path of data
+    :return: DataFrame of data for all
+    """
+    data_all_path = file_head_path + files_list[0]
+    data_all = pd.read_csv(data_all_path)
+    data_all = data_all.head(0)
+    try:
+        del data_all['Unnamed: 0']
+    except KeyError as e:
+        pass
+    # 循环添加全部数据
+    for i in files_list:
+        data_path = file_head_path + i
+        print("当前文件为：", data_path)
+        data = pd.read_csv(data_path)
+        try:
+            del data['Unnamed: 0']
+        except KeyError as e:
+            pass
+        data_all = data_all.append(data)
+    return data_all
+
+
+def file_name(file_dir):
+    files_list = []
+    for root, dirs, files in os.walk(file_dir):
+        # print("success")
+        for name in files:
+            files_list.append(name)
+    return files_list
+
+
+def load_data(making_data_dir, link_data_dir, cross_data_dir, head_link_dir,
+             win_order_data_dir, pre_arrival_sqe_dir, data_for_driver_xw, downstream_status_dir):
+    """
+    loading three path of data, then merge them
+    :return: all data by order_level
+    """
+    print('-------------LOAD DATA for mk_data----------------')
+    mk_list = file_name(making_data_dir)
+    mk_list.sort()
+    mk_data = append_all_data(mk_list, making_data_dir)
+    #mk_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_order_xt/join_20200825.csv')  # for test running
+    mk_data['date_time'] = mk_data['date_time'].astype(str)
+    mk_data['dayofweek'] = pd.to_datetime(mk_data['date_time'])
+    mk_data['dayofweek'] = mk_data['dayofweek'].dt.dayofweek+1
+
+    weather_le = LabelEncoder()
+    mk_data['weather_le'] = weather_le.fit_transform(mk_data['weather'])
+    mk_data['driver_id'] = mk_data['driver_id'].astype(str)
+
+    """
+    print('-------------LOAD DATA for driver_data----------------')
+    driver_list = file_name(data_for_driver_xw)
+    driver_list.sort()
+    driver_data = append_all_data(driver_list, data_for_driver_xw)
+    #driver_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/data_for_driver_xw/driver_20200825_head.txt')
+    driver_data = driver_data[['driver_id','date_time','entropy','hour_mean','workday_order','weekend_order']]
+    driver_data['date_time'] = driver_data['date_time'].astype(str)
+    driver_data['driver_id'] = driver_data['driver_id'].astype(str)
+    mk_data = mk_data.merge(driver_data, on=['driver_id', 'date_time'], how='left')
+    del driver_data
+    """
+
+    """
+    print('-------------LOAD DATA for downstream_status_for_order----------------')
+    ds_data_list = file_name(downstream_status_dir)
+    ds_data_list.sort()
+    ds_link_data = append_all_data(ds_data_list, downstream_status_dir)
+    #ds_link_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/downstream_status_for_order/ds_for_order_20200825.csv')
+    mk_data = mk_data.merge(ds_link_data, on=['order_id'], how='left')
+    del ds_link_data
+    """
+
+
+    """
+    print('-------------LOAD DATA for rate_status_for_order----------------')
+    #rate_data_list = file_name(rate_status_for_order)
+    #rate_data_list.sort()
+    #rate_data = append_all_data(rate_data_list, rate_status_for_order)
+    rate_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/rate_status_for_order/rate_for_order_20200825.csv')
+    mk_data = mk_data.merge(rate_data, on=['order_id'], how='left')
+    del rate_data
+    """
+
+
+    print('Remove the wk2_ and m1_ and ratio')
+    del_cols = []
+    mk_cols = mk_data.columns.tolist()
+    for i in range(len(mk_cols)):
+        if 'wk2_' in mk_cols[i]:
+            del_cols.append(mk_cols[i])
+        if 'm1_' in mk_cols[i]:
+            del_cols.append(mk_cols[i])
+        if 'ratio' in mk_cols[i]:
+            del_cols.append(mk_cols[i])
+    del_cols = del_cols + ['date_time_mean','weather', 'driver_id', 'date_time_dt', 'link_time_sum','date_time_sum']
+    print('*-' * 40, 'Will be drop the list:', del_cols)
+    mk_data.drop(columns=del_cols, axis=1, inplace=True)
+    print('The init shape of mk_data:', mk_data.shape)
+
+
+    print('-------------LOAD WIN DATA----------------')
+    win_order_list = file_name(win_order_data_dir)
+    win_order_list.sort()
+    win_order_data = append_all_data(win_order_list, win_order_data_dir)
+    #win_order_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/win_order_xw/win_for_slice_20200825.csv')  # for test running
+    del_win_order_cols = []
+    win_order_cols = win_order_data.columns.tolist()
+    for i in range(len(win_order_cols)):
+        if 'last_wk_lk_current' in win_order_cols[i]:
+            del_win_order_cols.append(win_order_cols[i])
+        #if 'distance' in win_order_cols[i]:
+        #    del_win_order_cols.append(win_order_cols[i])
+        #if '1_percent' in win_order_cols[i]:
+        #    del_win_order_cols.append(win_order_cols[i])
+        #if '0_percent' in win_order_cols[i]:
+        #    del_win_order_cols.append(win_order_cols[i])
+    del_win_order_cols = del_win_order_cols + ['slice_id', 'date_time']
+    win_order_data.drop(columns=del_win_order_cols, axis=1, inplace=True)
+    print('win_order_data.shape',win_order_data.shape)
+    mk_data = pd.merge(mk_data, win_order_data, how='left', on='order_id')
+    print('mk_data.shape',mk_data.shape)
+    del win_order_data
+    gc.collect()
+
+
+    print('-------------LOAD HEAD DATA----------------')
+    head_list = file_name(head_link_dir)
+    head_list.sort()
+    head_data = append_all_data(head_list, head_link_dir)
+    #head_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/head_link_data_clear/head_link_20200825.csv')  # for test running
+    get_head_cols = ['len_tmp','status_0','status_1','status_2','status_3','status_4','rate_0','rate_1','rate_2','rate_3','rate_4']
+    get_head_cols.insert(0, 'order_id')
+    print('head_data.shape:',head_data.shape)
+    head_data = head_data[get_head_cols]
+    print('mk_data.shape',mk_data.shape)
+    mk_data = pd.merge(mk_data, head_data, how='left', on='order_id')
+    print('mk_data.shape',mk_data.shape)
+    del head_data
+    gc.collect()
+
+
+    print('-------------LOAD DATA for link_data----------------')
+    link_list = file_name(link_data_dir)
+    link_list.sort()
+    link_data = append_all_data(link_list, link_data_dir)
+    #link_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/max_170_link_sqe_for_order/sqe_20200825_link.txt')  # for test running
+    #del_link_cols = ['link_time_sub','link_time_sub_sum','link_time_sub_mean', 'link_time_sub_std','link_time_sub_skew']
+    #link_data.drop(del_link_cols, axis=1, inplace=True)
+    print('The init shape of link_data:', link_data.shape)
+    gc.collect()
+
+
+    print('-------------LOAD DATA for arrival_sqe_data----------------')
+    arrival_sqe_list = file_name(pre_arrival_sqe_dir)
+    arrival_sqe_list.sort()
+    arrival_sqe_data = append_all_data(arrival_sqe_list, pre_arrival_sqe_dir)
+    #arrival_sqe_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/sqe_arrival_for_link/20200825.csv')  # for test running
+    del arrival_sqe_data['slice_id']
+    del arrival_sqe_data['pre_arrival_status']
+    del arrival_sqe_data['arrive_slice_id']
+    arrival_cols = arrival_sqe_data.columns.tolist()
+    new_arrival_cols = ['future_'+i for i in arrival_cols if i != 'order_id']
+    new_arrival_cols.insert(0, 'order_id')
+    arrival_sqe_data.columns = new_arrival_cols
+    print('The init shape of arrival_sqe_data:', arrival_sqe_data.shape)
+    link_data = pd.merge(link_data, arrival_sqe_data, how='left', on='order_id')
+    del arrival_sqe_data
+    gc.collect()
+    link_cols_list = ['link_id', 'link_time', 'link_current_status', 'pr','dc']
+
+
+
+    print('-------------LOAD DATA for cross_data----------------')
+    cross_list = file_name(cross_data_dir)
+    cross_list.sort()
+    cross_data = append_all_data(cross_list, cross_data_dir)
+    #cross_data = pd.read_csv('/home/didi2021/didi2021/giscup_2021/final_train_data_0703/for_0714_cross_sqe_for_order/sqe_20200825_cross.txt')  # for test running
+    del_cross_cols = ['cr_t_sub_by_min', 'cr_t_sub_by_q50', 'total_crosstime_std']
+    cross_data.drop(columns=del_cross_cols, axis=1, inplace=True)
+    print('The init shape of cross_data:', cross_data.shape)
+    cross_cols_list = ['cross_id', 'cross_time']
+
+
+    data = pd.merge(mk_data, link_data, how='left', on='order_id')
+    del mk_data
+    del link_data
+    gc.collect()
+    data = pd.merge(data, cross_data, how='left', on='order_id')
+    del cross_data
+    gc.collect()
+
+    # remove the class type and id and label, for deep inputs
+    mk_cols_list = data.columns.tolist()
+    remove_mk_cols = ['order_id', 'slice_id', 'hightemp', 'lowtemp', 'weather_le', 'dayofweek', 'date_time', 'ata']
+    mk_cols_list = list(set(mk_cols_list) - set(remove_mk_cols))
+    mk_cols_list = list(set(mk_cols_list) - set(link_cols_list))
+    mk_cols_list = list(set(mk_cols_list) - set(cross_cols_list))
+    print('lenght of mk_cols_list', len(mk_cols_list))
+    print('*-' * 40)
+    print('The finish shape of data is:', data.shape)
+
+    return data, mk_cols_list, link_cols_list, cross_cols_list
+
+
+def processing_data(data, mk_cols_list, link_cols_list, cross_cols_list, WIDE_COLS, is_test=False):
+    """
+    fix data, ast.literal_eval + StandardScaler + train_test_split
+    :return: train_data, val_data, test_data
+    """
+    print('Now, Starting parallel_apply the link..................')
+    for i in tqdm(link_cols_list):
+        data[i] = data[i].parallel_apply(ast.literal_eval)
+    print('Now, Starting parallel_apply the cross..................')
+    for i in tqdm(cross_cols_list):
+        data[i] = data[i].parallel_apply(ast.literal_eval)
+    # data = data.fillna(0)
+    data.fillna(data.median(),inplace=True)
+    ss_cols = mk_cols_list + WIDE_COLS
+    
+        # train, val
+    if is_test is True:
+        print('is_test is True')
+        ss = joblib.load('../model_h5/ss_scaler')
+        data[ss_cols] = ss.transform(data[ss_cols])
+        return data
+    else:
+        ss = StandardScaler()
+        ss.fit(data[ss_cols])
+        data[ss_cols] = ss.transform(data[ss_cols])
+        joblib.dump(ss, '../model_h5/ss_scaler')
+        print('is_test is False')
+        data['date_time'] = data['date_time'].astype(int)
+        print("type(data['date_time']):", data['date_time'].dtype)
+        # print('Here train_test_split..................')
+        # all_train_data, _ = train_test_split(all_train_data, test_size=0.9, random_state=42)
+        print('*-' * 40, 'The data.shape:', data.shape)
+        train_data, val_data = train_test_split(data, test_size=0.15, random_state=42)
+        train_data = train_data.reset_index()
+        val_data = val_data.reset_index()
+        del train_data['index']
+        del val_data['index']
+        return train_data, val_data
+
+
+def processing_inputs(data, mk_cols_list, link_cols_list, cross_cols_list, WIDE_COLS):
+    """
+    change the data for model
+    :return:
+    """
+    if 'ata' in mk_cols_list:
+        print('The ata in the mk_cols_list')
+    if 'ata' in link_cols_list:
+        print('The ata in the link_cols_list')
+    if 'ata' in cross_cols_list:
+        print('The ata in the cross_cols_list')
+    if 'ata' in WIDE_COLS:
+        print('The ata in the WIDE_COLS')
+    #link_cols_list = ['link_id', 'link_time','link_id_count','pr','dc',
+    #                                                   'top_a','link_current_status','link_ratio']
+    #cross_cols_list = ['cross_id', 'cross_time']
+    data_link_inputs = preprocess_inputs(data, cols=link_cols_list)
+    data_cross_inputs = preprocess_inputs(data, cols=cross_cols_list)
+    data_deep_input = data[mk_cols_list].values
+    data_wide_input = data[WIDE_COLS].values
+    data_inputs_slice = data['slice_id'].values
+    # print('--------------------------------test, ', min(data['slice_id'].values.tolist()))
+    data_labels = data['ata'].values
+
+    return data_link_inputs, data_cross_inputs, data_deep_input, data_wide_input, data_inputs_slice, data_labels

机器学习竞赛实战_优胜解决方案/滴滴——预估到达时间/WD_128544/wd_model/wd_model.py

@@ -0,0 +1,198 @@
+import pandas as pd
+import numpy as np
+from tensorflow import keras
+import tensorflow as tf
+from sklearn.preprocessing import StandardScaler, LabelEncoder
+import tensorflow.keras.layers as L
+# import tensorflow.keras.models as M
+import tensorflow.keras.backend as K
+from tensorflow.python.client import device_lib
+from tensorflow.keras.callbacks import ReduceLROnPlateau, ModelCheckpoint, EarlyStopping
+from keras_radam.training import RAdamOptimizer
+from tensorflow.keras import initializers, regularizers, constraints, optimizers, layers, callbacks
+from tensorflow.keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D, Conv1D
+from tensorflow.keras.layers import Input, Dense, Lambda, Layer
+from tensorflow.keras.initializers import Constant
+from tensorflow.keras.models import Model
+
+
+def get_available_gpus():
+    local_device_protos = device_lib.list_local_devices()
+    return [x.name for x in local_device_protos if x.device_type == 'GPU']
+
+
+def gru_layer(hidden_dim, dropout):
+    return L.Bidirectional(L.GRU(
+        hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer='orthogonal'))
+
+
+def lstm_layer(hidden_dim, dropout):
+    return L.Bidirectional(L.LSTM(
+        hidden_dim, dropout=dropout, return_sequences=True, kernel_initializer='orthogonal'))
+
+
+def preprocess(df, cate_cols, numeric_cols):
+    for cl in cate_cols:
+        le = LabelEncoder()
+        df[cl] = le.fit_transform(df[cl])
+    cols = cate_cols + numeric_cols
+    X_train = df[cols]
+    return X_train
+
+
+def wd_model(link_size, cross_size, slice_size, input_deep_col, input_wide_col,
+              link_nf_size, cross_nf_size, link_seqlen=170, cross_seqlen=12, pred_len=1,
+              dropout=0.25, sp_dropout=0.1, embed_dim=64, hidden_dim=128, n_layers=3, lr=0.001,
+              kernel_size1=3, kernel_size2=2, conv_size=128, conv='conv'):
+    link_inputs = L.Input(shape=(link_seqlen, link_nf_size))
+    cross_inputs = L.Input(shape=(cross_seqlen, cross_nf_size))
+    deep_inputs = L.Input(shape=(input_deep_col,), name='deep_input')
+    slice_input = L.Input(shape=(1,))
+    wide_inputs = keras.layers.Input(shape=(input_wide_col,), name='wide_input')
+
+    # link----------------------------
+    categorical_fea1 = link_inputs[:, :, :1]
+    numerical_fea1 = link_inputs[:, :, 1:5]
+
+    embed = L.Embedding(input_dim=link_size, output_dim=embed_dim)(categorical_fea1)
+    reshaped = tf.reshape(embed, shape=(-1, embed.shape[1], embed.shape[2] * embed.shape[3]))
+    #reshaped = L.SpatialDropout1D(sp_dropout)(reshaped)
+
+    hidden = L.concatenate([reshaped, numerical_fea1], axis=2)
+    hidden = L.SpatialDropout1D(sp_dropout)(hidden)
+    """
+    categorical_ar_st = link_inputs[:, :, 5:6]
+    categorical_ar_st = L.Masking(mask_value=-1, name='categorical_ar_st')(categorical_ar_st)
+    embed_ar_st = L.Embedding(input_dim=(-1,289), output_dim=8)(categorical_ar_st)
+    reshaped_ar_st = tf.reshape(embed_ar_st, shape=(-1, embed_ar_st.shape[1], embed_ar_st.shape[2] * embed_ar_st.shape[3]))
+    reshaped_ar_st = L.SpatialDropout1D(sp_dropout)(reshaped_ar_st)
+
+    categorical_ar_sl = link_inputs[:, :, 6:7]
+    categorical_ar_sl = L.Masking(mask_value=-1, name='categorical_ar_sl')(categorical_ar_sl)
+    embed_ar_sl = L.Embedding(input_dim=(-1, 289), output_dim=8)(categorical_ar_sl)
+    reshaped_ar_sl = tf.reshape(embed_ar_sl, shape=(-1, embed_ar_sl.shape[1], embed_ar_sl.shape[2] * embed_ar_sl.shape[3]))
+    reshaped_ar_sl = L.SpatialDropout1D(sp_dropout)(reshaped_ar_sl)
+    hidden = L.concatenate([reshaped, reshaped_ar_st, reshaped_ar_sl, numerical_fea1],axis=2)
+    """
+    for x in range(n_layers):
+        hidden = lstm_layer(hidden_dim, dropout)(hidden)
+
+    if conv=='conv':
+        #x_conv1 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(hidden)
+        avg_pool1_gru = GlobalAveragePooling1D()(hidden)
+        max_pool1_gru = GlobalMaxPooling1D()(hidden)
+        truncated_link = concatenate([avg_pool1_gru, max_pool1_gru])
+    elif conv=='resnet50':
+        truncated_link = ResNet50(include_top=False, pooling='max', weights=None)(hidden)
+    else:
+        truncated_link = hidden[:, :pred_len]
+        truncated_link = L.Flatten()(truncated_link)
+
+    # cross----------------------------
+    categorical_fea2 = cross_inputs[:, :, :1]
+    numerical_fea2 = cross_inputs[:, :, 1:]
+    embed2 = L.Embedding(input_dim=cross_size, output_dim=embed_dim)(categorical_fea2)
+    reshaped2 = tf.reshape(embed2, shape=(-1, embed2.shape[1], embed2.shape[2] * embed2.shape[3]))
+    #reshaped2 = L.SpatialDropout1D(sp_dropout)(reshaped2)
+
+    hidden2 = L.concatenate([reshaped2, numerical_fea2], axis=2)
+    hidden2 = L.SpatialDropout1D(sp_dropout)(hidden2)
+    for x in range(n_layers):
+        hidden2 = lstm_layer(hidden_dim, dropout)(hidden2)
+
+    if conv=='conv':
+        #x_conv3 = Conv1D(conv_size, kernel_size=kernel_size1, padding='valid', kernel_initializer='he_uniform')(hidden2)
+        avg_pool3_gru = GlobalAveragePooling1D()(hidden2)
+        max_pool3_gru = GlobalMaxPooling1D()(hidden2)
+        truncated_cross = concatenate([avg_pool3_gru, max_pool3_gru])
+    elif conv=='resnet50':
+        truncated_cross = ResNet50(include_top=False, pooling='max', weights=None)(hidden2)
+    else:
+        truncated_cross = hidden2[:, :pred_len]
+        truncated_cross = L.Flatten()(truncated_cross)
+
+    # slice----------------------------
+    embed_slice = L.Embedding(input_dim=slice_size, output_dim=1)(slice_input)
+    embed_slice = L.Flatten()(embed_slice)
+
+    # deep_inputs
+    """
+    dense_hidden1 = L.Dense(256, activation="relu")(deep_inputs)
+    dense_hidden1 = L.Dropout(dropout)(dense_hidden1)
+    dense_hidden2 = L.Dense(256, activation="relu")(dense_hidden1)
+    dense_hidden2 = L.Dropout(dropout)(dense_hidden2)
+    dense_hidden3 = L.Dense(128, activation="relu")(dense_hidden2)
+    """
+    x = L.Dense(512, activation="relu")(deep_inputs)
+    x = L.BatchNormalization()(x)
+    x = L.Lambda(tf.keras.activations.swish)(x)
+    x = L.Dropout(0.25)(x)
+    for i in range(2):
+        x = L.Dense(256)(x)
+        x = L.BatchNormalization()(x)
+        x = L.Lambda(tf.keras.activations.swish)(x)
+        x = L.Dropout(0.25)(x)
+    dense_hidden3 = L.Dense(64,activation='linear')(x)
+    # main-------------------------------
+    truncated = L.concatenate([truncated_link, truncated_cross, dense_hidden3, wide_inputs, embed_slice])  # WD
+    """
+    truncated = L.BatchNormalization()(truncated)
+    truncated = L.Dropout(dropout)(L.Dense(512, activation='relu') (truncated))
+    truncated = L.BatchNormalization()(truncated)
+    truncated = L.Dropout(dropout)(L.Dense(256, activation='relu') (truncated))
+    """
+    truncated = L.BatchNormalization()(truncated)
+    truncated = L.Dropout(dropout)(L.Dense(1024, activation='relu') (truncated))
+    truncated = L.Dropout(dropout)(truncated)
+
+    for i in range(2):
+        truncated = L.Dense(512)(truncated)
+        truncated = L.BatchNormalization()(truncated)
+        truncated = L.Lambda(tf.keras.activations.swish)(truncated)
+        truncated = L.Dropout(dropout)(truncated)
+
+    out = L.Dense(1, activation='linear')(truncated)
+
+
+    model = tf.keras.Model(inputs=[link_inputs, cross_inputs, deep_inputs, wide_inputs, slice_input],
+                           outputs=out)  # WD
+    print(model.summary())
+    model.compile(loss='mape',
+                  optimizer=RAdamOptimizer(learning_rate=1e-3),
+                  metrics=['mape'])
+
+    return model
+
+
+def get_mc_es_lr(model_name: str, patience=5, min_delta=1e-4):
+    mc = tf.keras.callbacks.ModelCheckpoint('../model_h5/model_{}.h5'.format(model_name)),
+    es = tf.keras.callbacks.EarlyStopping(monitor='val_loss', mode='min',
+                                          restore_best_weights=True, patience=patience)
+    lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.8, patience=patience, mode='min',
+                                              min_delta=min_delta)
+
+    return mc, es, lr
+
+
+class Mish(tf.keras.layers.Layer):
+
+    def __init__(self, **kwargs):
+        super(Mish, self).__init__(**kwargs)
+        self.supports_masking = True
+
+    def call(self, inputs):
+        return inputs * K.tanh(K.softplus(inputs))
+
+    def get_config(self):
+        base_config = super(Mish, self).get_config()
+        return dict(list(base_config.items()) + list(config.items()))
+
+    def compute_output_shape(self, input_shape):
+        return input_shape
+
+
+def mish(x):
+        return tf.keras.layers.Lambda(lambda x: x*K.tanh(K.softplus(x)))(x)
+
+
+tf.keras.utils.get_custom_objects().update({'mish': tf.keras.layers.Activation(mish)})