add float converter for min_value in attention

pull/3075/head
TianYuan 3 years ago
parent 0b3ca1fc08
commit 2b6f0a9d6c

@ -103,7 +103,7 @@ class MultiHeadedAttention(nn.Layer):
mask = paddle.logical_not(mask)
# assume scores.dtype==paddle.float32, we only use "float32" here
dtype = str(scores.dtype).split(".")[-1]
min_value = numpy.finfo(dtype).min
min_value = float(numpy.finfo(dtype).min)
scores = masked_fill(scores, mask, min_value)
# (batch, head, time1, time2)
self.attn = softmax(scores)
@ -194,11 +194,9 @@ class RelPositionMultiHeadedAttention(MultiHeadedAttention):
# only keep the positions from 0 to time2
new_t = paddle.cast(paddle.floor(t2 / 2) + 1, dtype='int32')
x = x_padded[:, :, 1:].reshape([b, h, t1, t2])[:, :, :, :new_t]
if self.zero_triu:
ones = paddle.ones((t1, t2))
x = x * paddle.tril(ones, t2 - t1)[None, None, :, :]
return x
def forward(self, query, key, value, pos_emb, mask):

Loading…
Cancel
Save