|
|
@ -59,50 +59,53 @@ def conv_bn_layer(input, filter_size, num_channels_in, num_channels_out, stride,
|
|
|
|
return padding_reset
|
|
|
|
return padding_reset
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def simple_rnn(input, size, param_attr=None, bias_attr=None, is_reverse=False):
|
|
|
|
class RNNCell(fluid.layers.RNNCell):
|
|
|
|
'''A simple rnn layer.
|
|
|
|
'''A simple rnn cell.
|
|
|
|
:param input: input layer.
|
|
|
|
:param hidden_size: Dimension of RNN cells.
|
|
|
|
:type input: Variable
|
|
|
|
:type hidden_size: int
|
|
|
|
:param size: Dimension of RNN cells.
|
|
|
|
|
|
|
|
:type size: int
|
|
|
|
|
|
|
|
:param param_attr: Parameter properties of hidden layer weights that
|
|
|
|
:param param_attr: Parameter properties of hidden layer weights that
|
|
|
|
can be learned
|
|
|
|
can be learned
|
|
|
|
:type param_attr: ParamAttr
|
|
|
|
:type param_attr: ParamAttr
|
|
|
|
:param bias_attr: Bias properties of hidden layer weights that can be learned
|
|
|
|
:param bias_attr: Bias properties of hidden layer weights that can be learned
|
|
|
|
:type bias_attr: ParamAttr
|
|
|
|
:type bias_attr: ParamAttr
|
|
|
|
:param is_reverse: Whether to calculate the inverse RNN
|
|
|
|
:param hidden_activation: Activation for hidden cell
|
|
|
|
:type is_reverse: bool
|
|
|
|
:type hidden_activation: Activation
|
|
|
|
:return: A simple RNN layer.
|
|
|
|
:param activation: Activation for output
|
|
|
|
:rtype: Variable
|
|
|
|
:type activation: Activation
|
|
|
|
|
|
|
|
:param name: Name of cell
|
|
|
|
|
|
|
|
:type name: string
|
|
|
|
'''
|
|
|
|
'''
|
|
|
|
if is_reverse:
|
|
|
|
|
|
|
|
input = fluid.layers.sequence_reverse(x=input)
|
|
|
|
def __init__(self,
|
|
|
|
|
|
|
|
hidden_size,
|
|
|
|
pad_value = fluid.layers.assign(input=np.array([0.0], dtype=np.float32))
|
|
|
|
param_attr=None,
|
|
|
|
input, length = fluid.layers.sequence_pad(input, pad_value)
|
|
|
|
bias_attr=None,
|
|
|
|
rnn = fluid.layers.StaticRNN()
|
|
|
|
hidden_activation=None,
|
|
|
|
input = fluid.layers.transpose(input, [1, 0, 2])
|
|
|
|
activation=None,
|
|
|
|
with rnn.step():
|
|
|
|
dtype="float32",
|
|
|
|
in_ = rnn.step_input(input)
|
|
|
|
name="RNNCell"):
|
|
|
|
mem = rnn.memory(shape=[-1, size], batch_ref=in_)
|
|
|
|
self.hidden_size = hidden_size
|
|
|
|
out = fluid.layers.fc(
|
|
|
|
self.param_attr = param_attr
|
|
|
|
input=mem,
|
|
|
|
self.bias_attr = bias_attr
|
|
|
|
size=size,
|
|
|
|
self.hidden_activation = hidden_activation
|
|
|
|
act=None,
|
|
|
|
self.activation = activation or fluid.layers.brelu
|
|
|
|
param_attr=param_attr,
|
|
|
|
self.name = name
|
|
|
|
bias_attr=bias_attr)
|
|
|
|
|
|
|
|
out = fluid.layers.elementwise_add(out, in_)
|
|
|
|
def call(self, inputs, states):
|
|
|
|
out = fluid.layers.brelu(out)
|
|
|
|
new_hidden = fluid.layers.fc(
|
|
|
|
rnn.update_memory(mem, out)
|
|
|
|
input=states,
|
|
|
|
rnn.output(out)
|
|
|
|
size=self.hidden_size,
|
|
|
|
|
|
|
|
act=self.hidden_activation,
|
|
|
|
out = rnn()
|
|
|
|
param_attr=self.param_attr,
|
|
|
|
out = fluid.layers.transpose(out, [1, 0, 2])
|
|
|
|
bias_attr=self.bias_attr)
|
|
|
|
out = fluid.layers.sequence_unpad(x=out, length=length)
|
|
|
|
new_hidden = fluid.layers.elementwise_add(new_hidden, inputs)
|
|
|
|
|
|
|
|
new_hidden = self.activation(new_hidden)
|
|
|
|
if is_reverse:
|
|
|
|
|
|
|
|
out = fluid.layers.sequence_reverse(x=out)
|
|
|
|
return new_hidden, new_hidden
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|
@property
|
|
|
|
|
|
|
|
def state_shape(self):
|
|
|
|
|
|
|
|
return [self.hidden_size]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def bidirectional_simple_rnn_bn_layer(name, input, size, share_weights):
|
|
|
|
def bidirectional_simple_rnn_bn_layer(name, input, size, share_weights):
|
|
|
@ -138,19 +141,31 @@ def bidirectional_simple_rnn_bn_layer(name, input, size, share_weights):
|
|
|
|
moving_mean_name=name + '_batch_norm_moving_mean',
|
|
|
|
moving_mean_name=name + '_batch_norm_moving_mean',
|
|
|
|
moving_variance_name=name + '_batch_norm_moving_variance')
|
|
|
|
moving_variance_name=name + '_batch_norm_moving_variance')
|
|
|
|
#forward and backword in time
|
|
|
|
#forward and backword in time
|
|
|
|
forward_rnn = simple_rnn(
|
|
|
|
forward_cell = RNNCell(
|
|
|
|
input=input_proj_bn,
|
|
|
|
hidden_size=size,
|
|
|
|
size=size,
|
|
|
|
activation=fluid.layers.brelu,
|
|
|
|
param_attr=fluid.ParamAttr(name=name + '_forward_rnn_weight'),
|
|
|
|
param_attr=fluid.ParamAttr(name=name + '_forward_rnn_weight'),
|
|
|
|
bias_attr=fluid.ParamAttr(name=name + '_forward_rnn_bias'),
|
|
|
|
bias_attr=fluid.ParamAttr(name=name + '_forward_rnn_bias'))
|
|
|
|
is_reverse=False)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
reverse_rnn = simple_rnn(
|
|
|
|
pad_value = fluid.layers.assign(input=np.array([0.0], dtype=np.float32))
|
|
|
|
input=input_proj_bn,
|
|
|
|
input, length = fluid.layers.sequence_pad(input_proj_bn, pad_value)
|
|
|
|
size=size,
|
|
|
|
forward_rnn, _ = fluid.layers.rnn(
|
|
|
|
|
|
|
|
cell=forward_cell, inputs=input, time_major=False, is_reverse=False)
|
|
|
|
|
|
|
|
forward_rnn = fluid.layers.sequence_unpad(x=forward_rnn, length=length)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
reverse_cell = RNNCell(
|
|
|
|
|
|
|
|
hidden_size=size,
|
|
|
|
|
|
|
|
activation=fluid.layers.brelu,
|
|
|
|
param_attr=fluid.ParamAttr(name=name + '_reverse_rnn_weight'),
|
|
|
|
param_attr=fluid.ParamAttr(name=name + '_reverse_rnn_weight'),
|
|
|
|
bias_attr=fluid.ParamAttr(name=name + '_reverse_rnn_bias'),
|
|
|
|
bias_attr=fluid.ParamAttr(name=name + '_reverse_rnn_bias'))
|
|
|
|
|
|
|
|
input, length = fluid.layers.sequence_pad(input_proj_bn, pad_value)
|
|
|
|
|
|
|
|
reverse_rnn, _ = fluid.layers.rnn(
|
|
|
|
|
|
|
|
cell=reverse_cell,
|
|
|
|
|
|
|
|
inputs=input,
|
|
|
|
|
|
|
|
sequence_length=length,
|
|
|
|
|
|
|
|
time_major=False,
|
|
|
|
is_reverse=True)
|
|
|
|
is_reverse=True)
|
|
|
|
|
|
|
|
reverse_rnn = fluid.layers.sequence_unpad(x=reverse_rnn, length=length)
|
|
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
input_proj_forward = fluid.layers.fc(
|
|
|
|
input_proj_forward = fluid.layers.fc(
|
|
|
@ -183,18 +198,32 @@ def bidirectional_simple_rnn_bn_layer(name, input, size, share_weights):
|
|
|
|
moving_mean_name=name + '_reverse_batch_norm_moving_mean',
|
|
|
|
moving_mean_name=name + '_reverse_batch_norm_moving_mean',
|
|
|
|
moving_variance_name=name + '_reverse_batch_norm_moving_variance')
|
|
|
|
moving_variance_name=name + '_reverse_batch_norm_moving_variance')
|
|
|
|
# forward and backward in time
|
|
|
|
# forward and backward in time
|
|
|
|
forward_rnn = simple_rnn(
|
|
|
|
forward_cell = RNNCell(
|
|
|
|
input=input_proj_bn_forward,
|
|
|
|
hidden_size=size,
|
|
|
|
size=size,
|
|
|
|
activation=fluid.layers.brelu,
|
|
|
|
param_attr=fluid.ParamAttr(name=name + '_forward_rnn_weight'),
|
|
|
|
param_attr=fluid.ParamAttr(name=name + '_forward_rnn_weight'),
|
|
|
|
bias_attr=fluid.ParamAttr(name=name + '_forward_rnn_bias'),
|
|
|
|
bias_attr=fluid.ParamAttr(name=name + '_forward_rnn_bias'))
|
|
|
|
is_reverse=False)
|
|
|
|
|
|
|
|
reverse_rnn = simple_rnn(
|
|
|
|
pad_value = fluid.layers.assign(input=np.array([0.0], dtype=np.float32))
|
|
|
|
input=input_proj_bn_backward,
|
|
|
|
input, length = fluid.layers.sequence_pad(input_proj_bn, pad_value)
|
|
|
|
size=size,
|
|
|
|
forward_rnn, _ = fluid.layers.rnn(
|
|
|
|
|
|
|
|
cell=forward_cell, inputs=input, time_major=False, is_reverse=False)
|
|
|
|
|
|
|
|
forward_rnn = fluid.layers.sequence_unpad(x=forward_rnn, length=length)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
reverse_cell = RNNCell(
|
|
|
|
|
|
|
|
hidden_size=size,
|
|
|
|
|
|
|
|
activation=fluid.layers.brelu,
|
|
|
|
param_attr=fluid.ParamAttr(name=name + '_reverse_rnn_weight'),
|
|
|
|
param_attr=fluid.ParamAttr(name=name + '_reverse_rnn_weight'),
|
|
|
|
bias_attr=fluid.ParamAttr(name=name + '_reverse_rnn_bias'),
|
|
|
|
bias_attr=fluid.ParamAttr(name=name + '_reverse_rnn_bias'))
|
|
|
|
|
|
|
|
input, length = fluid.layers.sequence_pad(input_proj_bn, pad_value)
|
|
|
|
|
|
|
|
reverse_rnn, _ = fluid.layers.rnn(
|
|
|
|
|
|
|
|
cell=reverse_cell,
|
|
|
|
|
|
|
|
inputs=input,
|
|
|
|
|
|
|
|
sequence_length=length,
|
|
|
|
|
|
|
|
time_major=False,
|
|
|
|
is_reverse=True)
|
|
|
|
is_reverse=True)
|
|
|
|
|
|
|
|
reverse_rnn = fluid.layers.sequence_unpad(x=reverse_rnn, length=length)
|
|
|
|
|
|
|
|
|
|
|
|
out = fluid.layers.concat(input=[forward_rnn, reverse_rnn], axis=1)
|
|
|
|
out = fluid.layers.concat(input=[forward_rnn, reverse_rnn], axis=1)
|
|
|
|
return out
|
|
|
|
return out
|
|
|
|
|
|
|
|
|
|
|
|