|
|
### BERT源码工作流解读
|
|
|
|
|
|
#### 数据读取模块
|
|
|
|
|
|
处理MRPC数据的类
|
|
|
|
|
|
~~~python
|
|
|
class MrpcProcessor(DataProcessor):
|
|
|
"""Processor for the MRPC data set (GLUE version)."""
|
|
|
|
|
|
def get_train_examples(self, data_dir):
|
|
|
"""See base class."""
|
|
|
return self._create_examples(
|
|
|
self._read_tsv(os.path.join(data_dir, "train.tsv")), "train")
|
|
|
|
|
|
def get_dev_examples(self, data_dir):
|
|
|
"""See base class."""
|
|
|
return self._create_examples(
|
|
|
self._read_tsv(os.path.join(data_dir, "dev.tsv")), "dev")
|
|
|
|
|
|
def get_test_examples(self, data_dir):
|
|
|
"""See base class."""
|
|
|
return self._create_examples(
|
|
|
self._read_tsv(os.path.join(data_dir, "test.tsv")), "test")
|
|
|
|
|
|
def get_labels(self):
|
|
|
"""See base class."""
|
|
|
return ["0", "1"] # 是否是二分类
|
|
|
|
|
|
def _create_examples(self, lines, set_type):
|
|
|
"""Creates examples for the training and dev sets."""
|
|
|
examples = []
|
|
|
for (i, line) in enumerate(lines):
|
|
|
if i == 0:
|
|
|
continue
|
|
|
guid = "%s-%s" % (set_type, i)
|
|
|
text_a = tokenization.convert_to_unicode(line[3]) # 相关的test_a和b怎么切分
|
|
|
text_b = tokenization.convert_to_unicode(line[4])
|
|
|
if set_type == "test":
|
|
|
label = "0"
|
|
|
else:
|
|
|
label = tokenization.convert_to_unicode(line[0])
|
|
|
examples.append(
|
|
|
InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
|
|
|
return examples
|
|
|
~~~
|
|
|
|
|
|
|
|
|
|
|
|
读取训练数据代码：
|
|
|
|
|
|
~~~python
|
|
|
if FLAGS.do_train:
|
|
|
train_examples = processor.get_train_examples(FLAGS.data_dir)
|
|
|
num_train_steps = int(
|
|
|
len(train_examples) / FLAGS.train_batch_size * FLAGS.num_train_epochs) # 得到需要迭代的次数，len(train_examples)计算出多少数据量 除以 我们设置的train_batch_size，再乘上epochs次数。
|
|
|
num_warmup_steps = int(num_train_steps * FLAGS.warmup_proportion) # 在刚开始时，让学习率偏小，经过warmup的百分比后，再还原回原始的学习率
|
|
|
~~~
|
|
|
|
|
|
|
|
|
|
|
|
#### 数据预处理模块
|
|
|
|
|
|
~~~python
|
|
|
# 衔接上一个
|
|
|
file_based_convert_examples_to_features(
|
|
|
train_examples, label_list, FLAGS.max_seq_length, tokenizer, train_file)
|
|
|
|
|
|
# ctrl点击file_based_xxx函数跳转
|
|
|
def file_based_convert_examples_to_features(
|
|
|
examples, label_list, max_seq_length, tokenizer, output_file):
|
|
|
"""Convert a set of `InputExample`s to a TFRecord file."""
|
|
|
|
|
|
writer = tf.python_io.TFRecordWriter(output_file) # TFRecord读取数据块，在bert中要求数据是TFRecord的形式。
|
|
|
|
|
|
for (ex_index, example) in enumerate(examples):
|
|
|
if ex_index % 10000 == 0:
|
|
|
tf.logging.info("Writing example %d of %d" % (ex_index, len(examples))) # for循环变量取数据
|
|
|
feature = convert_single_example(ex_index, example, label_list,
|
|
|
max_seq_length, tokenizer) # ctrl点击convert_xxx跳转
|
|
|
|
|
|
def convert_single_example(ex_index, example, label_list, max_seq_length,
|
|
|
tokenizer):
|
|
|
"""Converts a single `InputExample` into a single `InputFeatures`."""
|
|
|
|
|
|
if isinstance(example, PaddingInputExample):
|
|
|
return InputFeatures(
|
|
|
input_ids=[0] * max_seq_length,
|
|
|
input_mask=[0] * max_seq_length,
|
|
|
segment_ids=[0] * max_seq_length,
|
|
|
label_id=0,
|
|
|
is_real_example=False)
|
|
|
|
|
|
label_map = {} # 构建标签0, 1
|
|
|
for (i, label) in enumerate(label_list):
|
|
|
label_map[label] = i
|
|
|
|
|
|
tokens_a = tokenizer.tokenize(example.text_a) # ctrl点击tokenize，对第一句话分词
|
|
|
tokens_b = None
|
|
|
if example.text_b: # 第二句话分词
|
|
|
tokens_b = tokenizer.tokenize(example.text_b)
|
|
|
if tokens_b:
|
|
|
# Modifies `tokens_a` and `tokens_b` in place so that the total
|
|
|
# length is less than the specified length.
|
|
|
# Account for [CLS], [SEP], [SEP] with "- 3" # 保留3个特殊字符
|
|
|
_truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3) # 如果太长就截断的操作
|
|
|
else: # 没有b的时候保留两个字符
|
|
|
# Account for [CLS] and [SEP] with "- 2"
|
|
|
if len(tokens_a) > max_seq_length - 2:
|
|
|
tokens_a = tokens_a[0:(max_seq_length - 2)]
|
|
|
|
|
|
# The convention in BERT is:
|
|
|
# (a) For sequence pairs: # 将下面一对话，CLS开始，SEP断点，变成type_ids的0/1形式，0表示前一句，1表示后一句
|
|
|
# tokens: [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
|
|
|
# type_ids: 0 0 0 0 0 0 0 0 1 1 1 1 1 1
|
|
|
def tokenize(self, text):
|
|
|
split_tokens = []
|
|
|
for token in self.basic_tokenizer.tokenize(text): # 词切片，将一个词切片成多个小段，让表达的含义更丰富
|
|
|
for sub_token in self.wordpiece_tokenizer.tokenize(token):
|
|
|
split_tokens.append(sub_token)
|
|
|
|
|
|
return split_tokens
|
|
|
~~~
|
|
|
|
|
|
|
|
|
|
|
|
#### tfrecord制作
|
|
|
|
|
|
~~~~python
|
|
|
# 延续上面的convert_single_example模块
|
|
|
# 开始构建，创建两个列表来承接
|
|
|
tokens = []
|
|
|
segment_ids = []
|
|
|
tokens.append("[CLS]") # 第一个词是CLS
|
|
|
segment_ids.append(0) # 第一个的编码也肯定是0
|
|
|
for token in tokens_a:
|
|
|
tokens.append(token)
|
|
|
segment_ids.append(0) # 遍历获取，a（第一句话）都是0
|
|
|
tokens.append("[SEP]") # 遍历完增加个SEP连接符/断电
|
|
|
segment_ids.append(0) # tokens添加完SEP后，ids也添加对应的0
|
|
|
|
|
|
if tokens_b:
|
|
|
for token in tokens_b:
|
|
|
tokens.append(token)
|
|
|
segment_ids.append(1) # b和a一样，唯一不同的是添加的是1
|
|
|
tokens.append("[SEP]")
|
|
|
segment_ids.append(1)
|
|
|
|
|
|
input_ids = tokenizer.convert_tokens_to_ids(tokens) # 转成ID的映射，就是vocab语料库索引
|
|
|
|
|
|
# The mask has 1 for real tokens and 0 for padding tokens. Only real
|
|
|
# tokens are attended to.
|
|
|
input_mask = [1] * len(input_ids)
|
|
|
|
|
|
# Zero-pad up to the sequence length. 保证输入的长度是一样的，多退少补
|
|
|
while len(input_ids) < max_seq_length: # PAD的长度取决于设置的最大长度，小于全补0
|
|
|
input_ids.append(0)
|
|
|
input_mask.append(0)
|
|
|
segment_ids.append(0)
|
|
|
|
|
|
assert len(input_ids) == max_seq_length
|
|
|
assert len(input_mask) == max_seq_length
|
|
|
assert len(segment_ids) == max_seq_length
|
|
|
|
|
|
label_id = label_map[example.label]
|
|
|
if ex_index < 5:
|
|
|
tf.logging.info("*** Example ***") # 打印结果，这时候预处理的部分大致完成
|
|
|
...
|
|
|
return feature
|
|
|
~~~~
|
|
|
|
|
|
> 将数据制作成tfcord的形式，以便除了速度更快
|
|
|
|
|
|
|
|
|
|
|
|
返回原先的convert_single_example
|
|
|
|
|
|
~~~python
|
|
|
for (ex_index, example) in enumerate(examples): # 不断遍历处理数据
|
|
|
if ex_index % 10000 == 0:
|
|
|
tf.logging.info("Writing example %d of %d" % (ex_index, len(examples)))
|
|
|
|
|
|
feature = convert_single_example(ex_index, example, label_list,
|
|
|
max_seq_length, tokenizer) # ctrl点击convert_xxx跳
|
|
|
|
|
|
def create_int_feature(values):
|
|
|
f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))
|
|
|
return f
|
|
|
|
|
|
features = collections.OrderedDict() # 下面执行格式处理，处理成模型所需的格式
|
|
|
features["input_ids"] = create_int_feature(feature.input_ids)
|
|
|
features["input_mask"] = create_int_feature(feature.input_mask)
|
|
|
features["segment_ids"] = create_int_feature(feature.segment_ids)
|
|
|
features["label_ids"] = create_int_feature([feature.label_id])
|
|
|
features["is_real_example"] = create_int_feature(
|
|
|
[int(feature.is_real_example)])
|
|
|
|
|
|
tf_example = tf.train.Example(features=tf.train.Features(feature=features)) # 最后转换成tf的数据格式
|
|
|
writer.write(tf_example.SerializeToString())
|
|
|
writer.close()
|
|
|
~~~
|
|
|
|
|
|
|
|
|
|
|
|
#### Embedding层的作用
|
|
|
|
|
|
~~~python
|
|
|
def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,
|
|
|
labels, num_labels, use_one_hot_embeddings):
|
|
|
"""Creates a classification model."""
|
|
|
model = modeling.BertModel( # ctrl点击BertModel跳转
|
|
|
config=bert_config, # 配置
|
|
|
is_training=is_training,
|
|
|
input_ids=input_ids, # 特征
|
|
|
input_mask=input_mask, # 特征0/1
|
|
|
token_type_ids=segment_ids, # 特征维度表示第一句话还是第二句
|
|
|
use_one_hot_embeddings=use_one_hot_embeddings)
|
|
|
。。。
|
|
|
|
|
|
class BertModel(object):
|
|
|
"""BERT model ("Bidirectional Encoder Representations from Transformers").
|
|
|
|
|
|
Example usage:
|
|
|
|
|
|
```python
|
|
|
# Already been converted into WordPiece token ids
|
|
|
input_ids = tf.constant([[31, 51, 99], [15, 5, 0]])
|
|
|
input_mask = tf.constant([[1, 1, 1], [1, 1, 0]])
|
|
|
token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]])
|
|
|
|
|
|
config = modeling.BertConfig(vocab_size=32000, hidden_size=512,
|
|
|
num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024)
|
|
|
|
|
|
model = modeling.BertModel(config=config, is_training=True,
|
|
|
input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids)
|
|
|
|
|
|
label_embeddings = tf.get_variable(...)
|
|
|
pooled_output = model.get_pooled_output()
|
|
|
logits = tf.matmul(pooled_output, label_embeddings)
|
|
|
...
|
|
|
```
|
|
|
"""
|
|
|
|
|
|
def __init__(self,
|
|
|
config,
|
|
|
is_training,
|
|
|
input_ids,
|
|
|
input_mask=None,
|
|
|
token_type_ids=None,
|
|
|
use_one_hot_embeddings=False,
|
|
|
scope=None):
|
|
|
"""Constructor for BertModel.
|
|
|
|
|
|
Args:
|
|
|
config: `BertConfig` instance.
|
|
|
is_training: bool. true for training model, false for eval model. Controls
|
|
|
whether dropout will be applied.
|
|
|
input_ids: int32 Tensor of shape [batch_size, seq_length].
|
|
|
input_mask: (optional) int32 Tensor of shape [batch_size, seq_length].
|
|
|
token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length].
|
|
|
use_one_hot_embeddings: (optional) bool. Whether to use one-hot word
|
|
|
embeddings or tf.embedding_lookup() for the word embeddings.
|
|
|
scope: (optional) variable scope. Defaults to "bert".
|
|
|
|
|
|
Raises:
|
|
|
ValueError: The config is invalid or one of the input tensor shapes
|
|
|
is invalid.
|
|
|
"""
|
|
|
config = copy.deepcopy(config)
|
|
|
if not is_training:
|
|
|
config.hidden_dropout_prob = 0.0
|
|
|
config.attention_probs_dropout_prob = 0.0
|
|
|
|
|
|
input_shape = get_shape_list(input_ids, expected_rank=2)
|
|
|
batch_size = input_shape[0]
|
|
|
seq_length = input_shape[1]
|
|
|
|
|
|
if input_mask is None: # 如果没设置mask，默认都是1
|
|
|
input_mask = tf.ones(shape=[batch_size, seq_length], dtype=tf.int32)
|
|
|
|
|
|
if token_type_ids is None: # 没设置就默认一句话
|
|
|
token_type_ids = tf.zeros(shape=[batch_size, seq_length], dtype=tf.int32)
|
|
|
|
|
|
with tf.variable_scope(scope, default_name="bert"):
|
|
|
with tf.variable_scope("embeddings"):
|
|
|
# Perform embedding lookup on the word ids. 词的embeddings
|
|
|
(self.embedding_output, self.embedding_table) = embedding_lookup( # ctrl点击embedding_lookup跳转
|
|
|
input_ids=input_ids, # 词
|
|
|
vocab_size=config.vocab_size, # 语料库
|
|
|
embedding_size=config.hidden_size, # 编码映射成多少维
|
|
|
initializer_range=config.initializer_range, # 初始化范围
|
|
|
word_embedding_name="word_embeddings",
|
|
|
use_one_hot_embeddings=use_one_hot_embeddings)
|
|
|
~~~
|
|
|
|
|
|
|
|
|
|
|
|
~~~python
|
|
|
def embedding_lookup(input_ids,
|
|
|
vocab_size,
|
|
|
embedding_size=128,
|
|
|
initializer_range=0.02,
|
|
|
word_embedding_name="word_embeddings",
|
|
|
use_one_hot_embeddings=False):
|
|
|
"""Looks up words embeddings for id tensor.
|
|
|
|
|
|
Args:
|
|
|
input_ids: int32 Tensor of shape [batch_size, seq_length] containing word
|
|
|
ids.
|
|
|
vocab_size: int. Size of the embedding vocabulary.
|
|
|
embedding_size: int. Width of the word embeddings.
|
|
|
initializer_range: float. Embedding initialization range.
|
|
|
word_embedding_name: string. Name of the embedding table.
|
|
|
use_one_hot_embeddings: bool. If True, use one-hot method for word
|
|
|
embeddings. If False, use `tf.gather()`.
|
|
|
|
|
|
Returns:
|
|
|
float Tensor of shape [batch_size, seq_length, embedding_size].
|
|
|
"""
|
|
|
# This function assumes that the input is of shape [batch_size, seq_length,
|
|
|
# num_inputs].
|
|
|
#
|
|
|
# If the input is a 2D tensor of shape [batch_size, seq_length], we
|
|
|
# reshape to [batch_size, seq_length, 1].
|
|
|
if input_ids.shape.ndims == 2:
|
|
|
input_ids = tf.expand_dims(input_ids, axis=[-1])
|
|
|
|
|
|
embedding_table = tf.get_variable( # 词映射矩阵
|
|
|
name=word_embedding_name, # 词向量
|
|
|
shape=[vocab_size, embedding_size], # 获取语料库大表vovab.txt
|
|
|
initializer=create_initializer(initializer_range))
|
|
|
|
|
|
flat_input_ids = tf.reshape(input_ids, [-1])
|
|
|
if use_one_hot_embeddings:
|
|
|
one_hot_input_ids = tf.one_hot(flat_input_ids, depth=vocab_size) # 查出所有词做one_hot
|
|
|
output = tf.matmul(one_hot_input_ids, embedding_table) # 运算一个batch里所有的映射结果
|
|
|
else:
|
|
|
output = tf.gather(embedding_table, flat_input_ids)
|
|
|
|
|
|
input_shape = get_shape_list(input_ids)
|
|
|
|
|
|
output = tf.reshape(output,
|
|
|
input_shape[0:-1] + [input_shape[-1] * embedding_size]) # 制作返回结果
|
|
|
return (output, embedding_table) # 返回，词变成了向量
|
|
|
~~~
|
|
|
|
|
|
> 给数据做Embedding，再加入位置编码
|
|
|
|
|
|
|
|
|
|
|
|
#### 位置编码
|
|
|
|
|
|
~~~python
|
|
|
class BertModel(object):
|
|
|
"""BERT model ("Bidirectional Encoder Representations from Transformers").
|
|
|
|
|
|
Example usage:
|
|
|
|
|
|
```python
|
|
|
# Already been converted into WordPiece token ids
|
|
|
input_ids = tf.constant([[31, 51, 99], [15, 5, 0]])
|
|
|
input_mask = tf.constant([[1, 1, 1], [1, 1, 0]])
|
|
|
token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]])
|
|
|
|
|
|
config = modeling.BertConfig(vocab_size=32000, hidden_size=512,
|
|
|
num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024)
|
|
|
|
|
|
model = modeling.BertModel(config=config, is_training=True,
|
|
|
input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids)
|
|
|
|
|
|
label_embeddings = tf.get_variable(...)
|
|
|
pooled_output = model.get_pooled_output()
|
|
|
logits = tf.matmul(pooled_output, label_embeddings)
|
|
|
...
|
|
|
```
|
|
|
"""
|
|
|
|
|
|
def __init__(self,
|
|
|
config,
|
|
|
is_training,
|
|
|
input_ids,
|
|
|
input_mask=None,
|
|
|
token_type_ids=None,
|
|
|
use_one_hot_embeddings=False,
|
|
|
scope=None):
|
|
|
"""Constructor for BertModel.
|
|
|
|
|
|
Args:
|
|
|
config: `BertConfig` instance.
|
|
|
is_training: bool. true for training model, false for eval model. Controls
|
|
|
whether dropout will be applied.
|
|
|
input_ids: int32 Tensor of shape [batch_size, seq_length].
|
|
|
input_mask: (optional) int32 Tensor of shape [batch_size, seq_length].
|
|
|
token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length].
|
|
|
use_one_hot_embeddings: (optional) bool. Whether to use one-hot word
|
|
|
embeddings or tf.embedding_lookup() for the word embeddings.
|
|
|
scope: (optional) variable scope. Defaults to "bert".
|
|
|
|
|
|
Raises:
|
|
|
ValueError: The config is invalid or one of the input tensor shapes
|
|
|
is invalid.
|
|
|
"""
|
|
|
...
|
|
|
|
|
|
# Add positional embeddings and token type embeddings, then layer
|
|
|
# normalize and perform dropout.
|
|
|
self.embedding_output = embedding_postprocessor( # 制作位置编码，ctrl点击embedding_postprocessor
|
|
|
input_tensor=self.embedding_output,
|
|
|
use_token_type=True,
|
|
|
token_type_ids=token_type_ids,
|
|
|
token_type_vocab_size=config.type_vocab_size,
|
|
|
token_type_embedding_name="token_type_embeddings",
|
|
|
use_position_embeddings=True,
|
|
|
position_embedding_name="position_embeddings",
|
|
|
initializer_range=config.initializer_range,
|
|
|
max_position_embeddings=config.max_position_embeddings,
|
|
|
dropout_prob=config.hidden_dropout_prob)
|
|
|
~~~
|
|
|
|
|
|
|
|
|
|
|
|
~~~python
|
|
|
def embedding_postprocessor(input_tensor,
|
|
|
use_token_type=False,
|
|
|
token_type_ids=None,
|
|
|
token_type_vocab_size=16,
|
|
|
token_type_embedding_name="token_type_embeddings",
|
|
|
use_position_embeddings=True,
|
|
|
position_embedding_name="position_embeddings",
|
|
|
initializer_range=0.02,
|
|
|
max_position_embeddings=512,
|
|
|
dropout_prob=0.1):
|
|
|
"""Performs various post-processing on a word embedding tensor.
|
|
|
|
|
|
Args:
|
|
|
input_tensor: float Tensor of shape [batch_size, seq_length,
|
|
|
embedding_size].
|
|
|
use_token_type: bool. Whether to add embeddings for `token_type_ids`.
|
|
|
token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length].
|
|
|
Must be specified if `use_token_type` is True.
|
|
|
token_type_vocab_size: int. The vocabulary size of `token_type_ids`.
|
|
|
token_type_embedding_name: string. The name of the embedding table variable
|
|
|
for token type ids.
|
|
|
use_position_embeddings: bool. Whether to add position embeddings for the
|
|
|
position of each token in the sequence.
|
|
|
position_embedding_name: string. The name of the embedding table variable
|
|
|
for positional embeddings.
|
|
|
initializer_range: float. Range of the weight initialization.
|
|
|
max_position_embeddings: int. Maximum sequence length that might ever be
|
|
|
used with this model. This can be longer than the sequence length of
|
|
|
input_tensor, but cannot be shorter.
|
|
|
dropout_prob: float. Dropout probability applied to the final output tensor.
|
|
|
|
|
|
Returns:
|
|
|
float tensor with same shape as `input_tensor`.
|
|
|
|
|
|
Raises:
|
|
|
ValueError: One of the tensor shapes or input values is invalid.
|
|
|
"""
|
|
|
input_shape = get_shape_list(input_tensor, expected_rank=3)
|
|
|
batch_size = input_shape[0]
|
|
|
seq_length = input_shape[1]
|
|
|
width = input_shape[2]
|
|
|
|
|
|
output = input_tensor
|
|
|
|
|
|
if use_token_type: # 判断是第一句还是第二句，再做相应处理
|
|
|
if token_type_ids is None:
|
|
|
raise ValueError("`token_type_ids` must be specified if"
|
|
|
"`use_token_type` is True.")
|
|
|
token_type_table = tf.get_variable(
|
|
|
name=token_type_embedding_name,
|
|
|
shape=[token_type_vocab_size, width],
|
|
|
initializer=create_initializer(initializer_range))
|
|
|
# This vocab will be small so we always do one-hot here, since it is always
|
|
|
# faster for a small vocabulary.
|
|
|
flat_token_type_ids = tf.reshape(token_type_ids, [-1])
|
|
|
one_hot_ids = tf.one_hot(flat_token_type_ids, depth=token_type_vocab_size)
|
|
|
token_type_embeddings = tf.matmul(one_hot_ids, token_type_table)
|
|
|
token_type_embeddings = tf.reshape(token_type_embeddings,
|
|
|
[batch_size, seq_length, width])
|
|
|
output += token_type_embeddings
|
|
|
|
|
|
if use_position_embeddings: # 判断是否要做位置编码信息
|
|
|
assert_op = tf.assert_less_equal(seq_length, max_position_embeddings)
|
|
|
with tf.control_dependencies([assert_op]):
|
|
|
full_position_embeddings = tf.get_variable(
|
|
|
name=position_embedding_name,
|
|
|
shape=[max_position_embeddings, width],
|
|
|
initializer=create_initializer(initializer_range))
|
|
|
# Since the position embedding table is a learned variable, we create it
|
|
|
# using a (long) sequence length `max_position_embeddings`. The actual
|
|
|
# sequence length might be shorter than this, for faster training of
|
|
|
# tasks that do not have long sequences.
|
|
|
#
|
|
|
# So `full_position_embeddings` is effectively an embedding table
|
|
|
# for position [0, 1, 2, ..., max_position_embeddings-1], and the current
|
|
|
# sequence has positions [0, 1, 2, ... seq_length-1], so we can just
|
|
|
# perform a slice.
|
|
|
position_embeddings = tf.slice(full_position_embeddings, [0, 0],
|
|
|
[seq_length, -1]) # 如果位置编码给的过大，为了加速只需取出部分
|
|
|
num_dims = len(output.shape.as_list())
|
|
|
|
|
|
# Only the last two dimensions are relevant (`seq_length` and `width`), so
|
|
|
# we broadcast among the first dimensions, which is typically just
|
|
|
# the batch size.
|
|
|
position_broadcast_shape = []
|
|
|
for _ in range(num_dims - 2):
|
|
|
position_broadcast_shape.append(1)
|
|
|
position_broadcast_shape.extend([seq_length, width])
|
|
|
position_embeddings = tf.reshape(position_embeddings,
|
|
|
position_broadcast_shape)
|
|
|
output += position_embeddings
|
|
|
|
|
|
output = layer_norm_and_dropout(output, dropout_prob)
|
|
|
return output
|
|
|
~~~
|
|
|
|
|
|
> 给数据加入位置编码
|
|
|
|
|
|
|
|
|
|
|
|
#### mask机制
|
|
|
|
|
|
~~~python
|
|
|
class BertModel(object):
|
|
|
"""BERT model ("Bidirectional Encoder Representations from Transformers").
|
|
|
|
|
|
Example usage:
|
|
|
|
|
|
```python
|
|
|
# Already been converted into WordPiece token ids
|
|
|
input_ids = tf.constant([[31, 51, 99], [15, 5, 0]])
|
|
|
input_mask = tf.constant([[1, 1, 1], [1, 1, 0]])
|
|
|
token_type_ids = tf.constant([[0, 0, 1], [0, 2, 0]])
|
|
|
|
|
|
config = modeling.BertConfig(vocab_size=32000, hidden_size=512,
|
|
|
num_hidden_layers=8, num_attention_heads=6, intermediate_size=1024)
|
|
|
|
|
|
model = modeling.BertModel(config=config, is_training=True,
|
|
|
input_ids=input_ids, input_mask=input_mask, token_type_ids=token_type_ids)
|
|
|
|
|
|
label_embeddings = tf.get_variable(...)
|
|
|
pooled_output = model.get_pooled_output()
|
|
|
logits = tf.matmul(pooled_output, label_embeddings)
|
|
|
...
|
|
|
```
|
|
|
"""
|
|
|
|
|
|
def __init__(self,
|
|
|
config,
|
|
|
is_training,
|
|
|
input_ids,
|
|
|
input_mask=None,
|
|
|
token_type_ids=None,
|
|
|
use_one_hot_embeddings=False,
|
|
|
scope=None):
|
|
|
"""Constructor for BertModel.
|
|
|
|
|
|
Args:
|
|
|
config: `BertConfig` instance.
|
|
|
is_training: bool. true for training model, false for eval model. Controls
|
|
|
whether dropout will be applied.
|
|
|
input_ids: int32 Tensor of shape [batch_size, seq_length].
|
|
|
input_mask: (optional) int32 Tensor of shape [batch_size, seq_length].
|
|
|
token_type_ids: (optional) int32 Tensor of shape [batch_size, seq_length].
|
|
|
use_one_hot_embeddings: (optional) bool. Whether to use one-hot word
|
|
|
embeddings or tf.embedding_lookup() for the word embeddings.
|
|
|
scope: (optional) variable scope. Defaults to "bert".
|
|
|
|
|
|
Raises:
|
|
|
ValueError: The config is invalid or one of the input tensor shapes
|
|
|
is invalid.
|
|
|
"""
|
|
|
|
|
|
with tf.variable_scope("encoder"):
|
|
|
# This converts a 2D mask of shape [batch_size, seq_length] to a 3D
|
|
|
# mask of shape [batch_size, seq_length, seq_length] which is used
|
|
|
# for the attention scores.
|
|
|
attention_mask = create_attention_mask_from_input_mask(
|
|
|
input_ids, input_mask) # 创建mask矩阵
|
|
|
# 比如一个矩阵:[45，54，85，...，0，0，0]
|
|
|
# [12，31，11，...，0，0，0]
|
|
|
# [91，51，18，...，12，21，0]
|
|
|
# 后面长度不足的都补0，mask后，有信息的变1，无信息的变0
|
|
|
# [1，1，1，...，0，0，0]
|
|
|
# [1，1，1，...，0，0，0]
|
|
|
# [1，1，1，...，1，1，0]
|
|
|
# 不管要知道二维的，还要知道三维的，如开头这句话This converts a 2D mask of shape [batch_size, seq_length] to a 3D
|
|
|
# 把里面的维度再分一个维度，如左上角的45
|
|
|
# [1，1，1，...，0，0，0] ， 这里的1是指45能看到的信息是那些，有的则为1，并与其计算，为0则不与其进行计算
|
|
|
# Run the stacked transformer.
|
|
|
# `sequence_output` shape = [batch_size, seq_length, hidden_size].
|
|
|
self.all_encoder_layers = transformer_model( # Ctrl点击跳转transformer_model
|
|
|
input_tensor=self.embedding_output, # 3种embedding
|
|
|
attention_mask=attention_mask, # 上面的需不需要计算的0，1，1则是要计算
|
|
|
hidden_size=config.hidden_size, # 特征结果
|
|
|
num_hidden_layers=config.num_hidden_layers, # Transformer中的隐层神经元个数
|
|
|
num_attention_heads=config.num_attention_heads, # 多头机制，在bert的图解中有讲解
|
|
|
intermediate_size=config.intermediate_size, # 全连接层神经元个数
|
|
|
intermediate_act_fn=get_activation(config.hidden_act),
|
|
|
hidden_dropout_prob=config.hidden_dropout_prob,
|
|
|
attention_probs_dropout_prob=config.attention_probs_dropout_prob,
|
|
|
initializer_range=config.initializer_range,
|
|
|
do_return_all_layers=True)
|
|
|
~~~
|
|
|
|
|
|
> 对数据进行mask，此时数据部分已加工完，开始做QKV计算
|
|
|
|
|
|
|
|
|
|
|
|
#### 构建QKV矩阵
|
|
|
|
|
|
~~~python
|
|
|
# 通过上面的点击函数跳转到transformer_model
|
|
|
def transformer_model(input_tensor,
|
|
|
attention_mask=None,
|
|
|
hidden_size=768,
|
|
|
num_hidden_layers=12,
|
|
|
num_attention_heads=12,
|
|
|
intermediate_size=3072,
|
|
|
intermediate_act_fn=gelu,
|
|
|
hidden_dropout_prob=0.1,
|
|
|
attention_probs_dropout_prob=0.1,
|
|
|
initializer_range=0.02,
|
|
|
do_return_all_layers=False):
|
|
|
"""Multi-headed, multi-layer Transformer from "Attention is All You Need".
|
|
|
|
|
|
This is almost an exact implementation of the original Transformer encoder.
|
|
|
|
|
|
See the original paper:
|
|
|
https://arxiv.org/abs/1706.03762
|
|
|
|
|
|
Also see:
|
|
|
https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/models/transformer.py
|
|
|
|
|
|
Args:
|
|
|
input_tensor: float Tensor of shape [batch_size, seq_length, hidden_size].
|
|
|
attention_mask: (optional) int32 Tensor of shape [batch_size, seq_length,
|
|
|
seq_length], with 1 for positions that can be attended to and 0 in
|
|
|
positions that should not be.
|
|
|
hidden_size: int. Hidden size of the Transformer.
|
|
|
num_hidden_layers: int. Number of layers (blocks) in the Transformer.
|
|
|
num_attention_heads: int. Number of attention heads in the Transformer.
|
|
|
intermediate_size: int. The size of the "intermediate" (a.k.a., feed
|
|
|
forward) layer.
|
|
|
intermediate_act_fn: function. The non-linear activation function to apply
|
|
|
to the output of the intermediate/feed-forward layer.
|
|
|
hidden_dropout_prob: float. Dropout probability for the hidden layers.
|
|
|
attention_probs_dropout_prob: float. Dropout probability of the attention
|
|
|
probabilities.
|
|
|
initializer_range: float. Range of the initializer (stddev of truncated
|
|
|
normal).
|
|
|
do_return_all_layers: Whether to also return all layers or just the final
|
|
|
layer.
|
|
|
|
|
|
Returns:
|
|
|
float Tensor of shape [batch_size, seq_length, hidden_size], the final
|
|
|
hidden layer of the Transformer.
|
|
|
|
|
|
Raises:
|
|
|
ValueError: A Tensor shape or parameter is invalid.
|
|
|
"""
|
|
|
if hidden_size % num_attention_heads != 0: # 判断是否能整除，否则后面会报错
|
|
|
raise ValueError(
|
|
|
"The hidden size (%d) is not a multiple of the number of attention "
|
|
|
"heads (%d)" % (hidden_size, num_attention_heads))
|
|
|
|
|
|
attention_head_size = int(hidden_size / num_attention_heads)
|
|
|
input_shape = get_shape_list(input_tensor, expected_rank=3)
|
|
|
batch_size = input_shape[0]
|
|
|
seq_length = input_shape[1]
|
|
|
input_width = input_shape[2]
|
|
|
|
|
|
# The Transformer performs sum residuals on all layers so the input needs
|
|
|
# to be the same as the hidden size.
|
|
|
if input_width != hidden_size:
|
|
|
raise ValueError("The width of the input tensor (%d) != hidden size (%d)" %
|
|
|
(input_width, hidden_size))
|
|
|
|
|
|
# We keep the representation as a 2D tensor to avoid re-shaping it back and
|
|
|
# forth from a 3D tensor to a 2D tensor. Re-shapes are normally free on
|
|
|
# the GPU/CPU but may not be free on the TPU, so we want to minimize them to
|
|
|
# help the optimizer.
|
|
|
prev_output = reshape_to_matrix(input_tensor)
|
|
|
|
|
|
all_layer_outputs = []
|
|
|
for layer_idx in range(num_hidden_layers): # 遍历层数，这层结果是下一层的输入
|
|
|
with tf.variable_scope("layer_%d" % layer_idx):
|
|
|
layer_input = prev_output
|
|
|
|
|
|
with tf.variable_scope("attention"):
|
|
|
attention_heads = []
|
|
|
with tf.variable_scope("self"):
|
|
|
attention_head = attention_layer( # Ctrl点击attention_layer跳转
|
|
|
from_tensor=layer_input,
|
|
|
to_tensor=layer_input, # from和to都是self_tensor，即自己和自己本句的关联
|
|
|
attention_mask=attention_mask, # 0/1
|
|
|
num_attention_heads=num_attention_heads, # 多头参数
|
|
|
size_per_head=attention_head_size, # 头大小
|
|
|
attention_probs_dropout_prob=attention_probs_dropout_prob, # 丢弃
|
|
|
initializer_range=initializer_range, # 初始化位置
|
|
|
do_return_2d_tensor=True, # 是否返回2D特征
|
|
|
batch_size=batch_size,
|
|
|
from_seq_length=seq_length,
|
|
|
to_seq_length=seq_length)
|
|
|
attention_heads.append(attention_head)
|
|
|
...
|
|
|
|
|
|
def create_attention_mask_from_input_mask(from_tensor, to_mask):
|
|
|
"""Create 3D attention mask from a 2D tensor mask.
|
|
|
|
|
|
Args:
|
|
|
from_tensor: 2D or 3D Tensor of shape [batch_size, from_seq_length, ...].
|
|
|
to_mask: int32 Tensor of shape [batch_size, to_seq_length].
|
|
|
|
|
|
Returns:
|
|
|
float Tensor of shape [batch_size, from_seq_length, to_seq_length].
|
|
|
"""
|
|
|
from_shape = get_shape_list(from_tensor, expected_rank=[2, 3])
|
|
|
batch_size = from_shape[0]
|
|
|
from_seq_length = from_shape[1]
|
|
|
|
|
|
to_shape = get_shape_list(to_mask, expected_rank=2)
|
|
|
to_seq_length = to_shape[1]
|
|
|
|
|
|
to_mask = tf.cast(
|
|
|
tf.reshape(to_mask, [batch_size, 1, to_seq_length]), tf.float32)
|
|
|
|
|
|
# We don't assume that `from_tensor` is a mask (although it could be). We
|
|
|
# don't actually care if we attend *from* padding tokens (only *to* padding)
|
|
|
# tokens so we create a tensor of all ones.
|
|
|
#
|
|
|
# `broadcast_ones` = [batch_size, from_seq_length, 1]
|
|
|
broadcast_ones = tf.ones(
|
|
|
shape=[batch_size, from_seq_length, 1], dtype=tf.float32)
|
|
|
|
|
|
# Here we broadcast along two dimensions to create the mask.
|
|
|
mask = broadcast_ones * to_mask
|
|
|
|
|
|
return mask
|
|
|
|
|
|
|
|
|
def attention_layer(from_tensor,
|
|
|
to_tensor,
|
|
|
attention_mask=None,
|
|
|
num_attention_heads=1,
|
|
|
size_per_head=512,
|
|
|
query_act=None,
|
|
|
key_act=None,
|
|
|
value_act=None,
|
|
|
attention_probs_dropout_prob=0.0,
|
|
|
initializer_range=0.02,
|
|
|
do_return_2d_tensor=False,
|
|
|
batch_size=None,
|
|
|
from_seq_length=None,
|
|
|
to_seq_length=None):
|
|
|
"""Performs multi-headed attention from `from_tensor` to `to_tensor`.
|
|
|
|
|
|
This is an implementation of multi-headed attention based on "Attention
|
|
|
is all you Need". If `from_tensor` and `to_tensor` are the same, then
|
|
|
this is self-attention. Each timestep in `from_tensor` attends to the
|
|
|
corresponding sequence in `to_tensor`, and returns a fixed-with vector.
|
|
|
|
|
|
This function first projects `from_tensor` into a "query" tensor and
|
|
|
`to_tensor` into "key" and "value" tensors. These are (effectively) a list
|
|
|
of tensors of length `num_attention_heads`, where each tensor is of shape
|
|
|
[batch_size, seq_length, size_per_head].
|
|
|
|
|
|
Then, the query and key tensors are dot-producted and scaled. These are
|
|
|
softmaxed to obtain attention probabilities. The value tensors are then
|
|
|
interpolated by these probabilities, then concatenated back to a single
|
|
|
tensor and returned.
|
|
|
|
|
|
In practice, the multi-headed attention are done with transposes and
|
|
|
reshapes rather than actual separate tensors.
|
|
|
|
|
|
Args:
|
|
|
from_tensor: float Tensor of shape [batch_size, from_seq_length,
|
|
|
from_width].
|
|
|
to_tensor: float Tensor of shape [batch_size, to_seq_length, to_width].
|
|
|
attention_mask: (optional) int32 Tensor of shape [batch_size,
|
|
|
from_seq_length, to_seq_length]. The values should be 1 or 0. The
|
|
|
attention scores will effectively be set to -infinity for any positions in
|
|
|
the mask that are 0, and will be unchanged for positions that are 1.
|
|
|
num_attention_heads: int. Number of attention heads.
|
|
|
size_per_head: int. Size of each attention head.
|
|
|
query_act: (optional) Activation function for the query transform.
|
|
|
key_act: (optional) Activation function for the key transform.
|
|
|
value_act: (optional) Activation function for the value transform.
|
|
|
attention_probs_dropout_prob: (optional) float. Dropout probability of the
|
|
|
attention probabilities.
|
|
|
initializer_range: float. Range of the weight initializer.
|
|
|
do_return_2d_tensor: bool. If True, the output will be of shape [batch_size
|
|
|
* from_seq_length, num_attention_heads * size_per_head]. If False, the
|
|
|
output will be of shape [batch_size, from_seq_length, num_attention_heads
|
|
|
* size_per_head].
|
|
|
batch_size: (Optional) int. If the input is 2D, this might be the batch size
|
|
|
of the 3D version of the `from_tensor` and `to_tensor`.
|
|
|
from_seq_length: (Optional) If the input is 2D, this might be the seq length
|
|
|
of the 3D version of the `from_tensor`.
|
|
|
to_seq_length: (Optional) If the input is 2D, this might be the seq length
|
|
|
of the 3D version of the `to_tensor`.
|
|
|
|
|
|
Returns:
|
|
|
float Tensor of shape [batch_size, from_seq_length,
|
|
|
num_attention_heads * size_per_head]. (If `do_return_2d_tensor` is
|
|
|
true, this will be of shape [batch_size * from_seq_length,
|
|
|
num_attention_heads * size_per_head]).
|
|
|
|
|
|
Raises:
|
|
|
ValueError: Any of the arguments or tensor shapes are invalid.
|
|
|
"""
|
|
|
|
|
|
def transpose_for_scores(input_tensor, batch_size, num_attention_heads,
|
|
|
seq_length, width):
|
|
|
output_tensor = tf.reshape(
|
|
|
input_tensor, [batch_size, seq_length, num_attention_heads, width])
|
|
|
|
|
|
output_tensor = tf.transpose(output_tensor, [0, 2, 1, 3])
|
|
|
return output_tensor
|
|
|
|
|
|
from_shape = get_shape_list(from_tensor, expected_rank=[2, 3])
|
|
|
to_shape = get_shape_list(to_tensor, expected_rank=[2, 3])
|
|
|
|
|
|
if len(from_shape) != len(to_shape):
|
|
|
raise ValueError(
|
|
|
"The rank of `from_tensor` must match the rank of `to_tensor`.")
|
|
|
|
|
|
if len(from_shape) == 3:
|
|
|
batch_size = from_shape[0]
|
|
|
from_seq_length = from_shape[1]
|
|
|
to_seq_length = to_shape[1]
|
|
|
elif len(from_shape) == 2:
|
|
|
if (batch_size is None or from_seq_length is None or to_seq_length is None):
|
|
|
raise ValueError(
|
|
|
"When passing in rank 2 tensors to attention_layer, the values "
|
|
|
"for `batch_size`, `from_seq_length`, and `to_seq_length` "
|
|
|
"must all be specified.")
|
|
|
|
|
|
# Scalar dimensions referenced here:
|
|
|
# B = batch size (number of sequences)
|
|
|
# F = `from_tensor` sequence length
|
|
|
# T = `to_tensor` sequence length
|
|
|
# N = `num_attention_heads`
|
|
|
# H = `size_per_head`
|
|
|
|
|
|
from_tensor_2d = reshape_to_matrix(from_tensor)
|
|
|
to_tensor_2d = reshape_to_matrix(to_tensor)
|
|
|
|
|
|
# `query_layer` = [B*F, N*H]
|
|
|
query_layer = tf.layers.dense(
|
|
|
from_tensor_2d,
|
|
|
num_attention_heads * size_per_head,
|
|
|
activation=query_act,
|
|
|
name="query",
|
|
|
kernel_initializer=create_initializer(initializer_range))
|
|
|
|
|
|
# `key_layer` = [B*T, N*H]
|
|
|
key_layer = tf.layers.dense(
|
|
|
to_tensor_2d,
|
|
|
num_attention_heads * size_per_head,
|
|
|
activation=key_act,
|
|
|
name="key",
|
|
|
kernel_initializer=create_initializer(initializer_range))
|
|
|
|
|
|
# `value_layer` = [B*T, N*H]
|
|
|
value_layer = tf.layers.dense(
|
|
|
to_tensor_2d,
|
|
|
num_attention_heads * size_per_head,
|
|
|
activation=value_act,
|
|
|
name="value",
|
|
|
kernel_initializer=create_initializer(initializer_range))
|
|
|
|
|
|
# `query_layer` = [B, N, F, H] 加速内积计算
|
|
|
query_layer = transpose_for_scores(query_layer, batch_size,
|
|
|
num_attention_heads, from_seq_length,
|
|
|
size_per_head)
|
|
|
|
|
|
# `key_layer` = [B, N, T, H] 加速内积计算
|
|
|
key_layer = transpose_for_scores(key_layer, batch_size, num_attention_heads,
|
|
|
to_seq_length, size_per_head)
|
|
|
~~~
|
|
|
|
|
|
> 此时完成QKV的计算，接下来消除维度影响、softmax
|
|
|
|
|
|
|
|
|
|
|
|
#### 完成Transformer模块构建
|
|
|
|
|
|
~~~python
|
|
|
# 衔接上面的
|
|
|
def attention_layer(from_tensor,
|
|
|
to_tensor,
|
|
|
attention_mask=None,
|
|
|
num_attention_heads=1,
|
|
|
size_per_head=512,
|
|
|
query_act=None,
|
|
|
key_act=None,
|
|
|
value_act=None,
|
|
|
attention_probs_dropout_prob=0.0,
|
|
|
initializer_range=0.02,
|
|
|
do_return_2d_tensor=False,
|
|
|
batch_size=None,
|
|
|
from_seq_length=None,
|
|
|
to_seq_length=None):
|
|
|
...
|
|
|
# Take the dot product between "query" and "key" to get the raw
|
|
|
# attention scores.
|
|
|
# `attention_scores` = [B, N, F, T]
|
|
|
attention_scores = tf.matmul(query_layer, key_layer, transpose_b=True)
|
|
|
attention_scores = tf.multiply(attention_scores,
|
|
|
1.0 / math.sqrt(float(size_per_head))) # 消除维度对结果的影响
|
|
|
|
|
|
if attention_mask is not None:
|
|
|
# `attention_mask` = [B, 1, F, T]
|
|
|
attention_mask = tf.expand_dims(attention_mask, axis=[1])
|
|
|
|
|
|
# Since attention_mask is 1.0 for positions we want to attend and 0.0 for
|
|
|
# masked positions, this operation will create a tensor which is 0.0 for
|
|
|
# positions we want to attend and -10000.0 for masked positions.
|
|
|
adder = (1.0 - tf.cast(attention_mask, tf.float32)) * -10000.0 # mask为1时结果为0，mask为0时结果为非常大的负数
|
|
|
|
|
|
# Since we are adding it to the raw scores before the softmax, this is
|
|
|
# effectively the same as removing these entirely.
|
|
|
attention_scores += adder # 把上面的值加入原始得分里相当于mask为1不变，mask为0则变成很大的负数
|
|
|
|
|
|
# Normalize the attention scores to probabilities.
|
|
|
# `attention_probs` = [B, N, F, T]
|
|
|
attention_probs = tf.nn.softmax(attention_scores) # 再softmax时，非常大的负数则无限接近于0，就相当于不考虑
|
|
|
|
|
|
# This is actually dropping out entire tokens to attend to, which might
|
|
|
# seem a bit unusual, but is taken from the original Transformer paper.
|
|
|
attention_probs = dropout(attention_probs, attention_probs_dropout_prob)
|
|
|
|
|
|
# `value_layer` = [B, T, N, H]
|
|
|
value_layer = tf.reshape(
|
|
|
value_layer,
|
|
|
[batch_size, to_seq_length, num_attention_heads, size_per_head])
|
|
|
|
|
|
# `value_layer` = [B, N, T, H]
|
|
|
value_layer = tf.transpose(value_layer, [0, 2, 1, 3])
|
|
|
|
|
|
# `context_layer` = [B, N, F, H]
|
|
|
context_layer = tf.matmul(attention_probs, value_layer)
|
|
|
|
|
|
# `context_layer` = [B, F, N, H]
|
|
|
context_layer = tf.transpose(context_layer, [0, 2, 1, 3])
|
|
|
|
|
|
if do_return_2d_tensor: # 返回结果前判断维度是否一样，因为连接了很多层，会不断输入输出
|
|
|
# `context_layer` = [B*F, N*H]
|
|
|
context_layer = tf.reshape(
|
|
|
context_layer,
|
|
|
[batch_size * from_seq_length, num_attention_heads * size_per_head])
|
|
|
else:
|
|
|
# `context_layer` = [B, F, N*H]
|
|
|
context_layer = tf.reshape(
|
|
|
context_layer,
|
|
|
[batch_size, from_seq_length, num_attention_heads * size_per_head])
|
|
|
|
|
|
return context_layer
|
|
|
~~~
|
|
|
|
|
|
> 上面处理完后，还有残差连接，防止训练结果比不训练的更差
|
|
|
|
|
|
|
|
|
|
|
|
#### 训练BERT模型
|
|
|
|
|
|
~~~python
|
|
|
# 回到transformer_model
|
|
|
def transformer_model(input_tensor,
|
|
|
attention_mask=None,
|
|
|
hidden_size=768,
|
|
|
num_hidden_layers=12,
|
|
|
num_attention_heads=12,
|
|
|
intermediate_size=3072,
|
|
|
intermediate_act_fn=gelu,
|
|
|
hidden_dropout_prob=0.1,
|
|
|
attention_probs_dropout_prob=0.1,
|
|
|
initializer_range=0.02,
|
|
|
do_return_all_layers=False):
|
|
|
|
|
|
# Run a linear projection of `hidden_size` then add a residual
|
|
|
# with `layer_input`.
|
|
|
with tf.variable_scope("output"): # 残差连接
|
|
|
attention_output = tf.layers.dense(
|
|
|
attention_output,
|
|
|
hidden_size,
|
|
|
kernel_initializer=create_initializer(initializer_range))
|
|
|
attention_output = dropout(attention_output, hidden_dropout_prob)
|
|
|
attention_output = layer_norm(attention_output + layer_input)
|
|
|
|
|
|
# The activation is only applied to the "intermediate" hidden layer.
|
|
|
with tf.variable_scope("intermediate"):
|
|
|
intermediate_output = tf.layers.dense(
|
|
|
attention_output,
|
|
|
intermediate_size,
|
|
|
activation=intermediate_act_fn,
|
|
|
kernel_initializer=create_initializer(initializer_range))
|
|
|
|
|
|
# Down-project back to `hidden_size` then add the residual.
|
|
|
with tf.variable_scope("output"): # 残差连接完，数据维度会增大，需要变回一直的维度
|
|
|
layer_output = tf.layers.dense(
|
|
|
intermediate_output,
|
|
|
hidden_size,
|
|
|
kernel_initializer=create_initializer(initializer_range))
|
|
|
layer_output = dropout(layer_output, hidden_dropout_prob)
|
|
|
layer_output = layer_norm(layer_output + attention_output)
|
|
|
prev_output = layer_output
|
|
|
all_layer_outputs.append(layer_output)
|
|
|
|
|
|
if do_return_all_layers:
|
|
|
final_outputs = []
|
|
|
for layer_output in all_layer_outputs:
|
|
|
final_output = reshape_from_matrix(layer_output, input_shape)
|
|
|
final_outputs.append(final_output)
|
|
|
return final_outputs
|
|
|
else:
|
|
|
final_output = reshape_from_matrix(prev_output, input_shape)
|
|
|
return final_output
|
|
|
~~~
|
|
|
|
|
|
> 最终，所有的结果已处理完成并输出向量，这样BertModel模块已经讲完，modeling.py的部分也完成了，我们再回到run_classifier.py
|
|
|
|
|
|
~~~python
|
|
|
def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,
|
|
|
labels, num_labels, use_one_hot_embeddings):
|
|
|
"""Creates a classification model."""
|
|
|
...
|
|
|
# 前面的modeling.BertModel已经看过了，最终我们得到了QVK计算后的softemax层和残差连接后的结果
|
|
|
|
|
|
# In the demo, we are doing a simple classification task on the entire
|
|
|
# segment.
|
|
|
#
|
|
|
# If you want to use the token-level output, use model.get_sequence_output()
|
|
|
# instead.
|
|
|
output_layer = model.get_pooled_output()
|
|
|
|
|
|
hidden_size = output_layer.shape[-1].value # 获取向量
|
|
|
|
|
|
output_weights = tf.get_variable( # 构造全连接层，二分类的权重参数
|
|
|
"output_weights", [num_labels, hidden_size],
|
|
|
initializer=tf.truncated_normal_initializer(stddev=0.02))
|
|
|
|
|
|
output_bias = tf.get_variable( # 构造偏值b
|
|
|
"output_bias", [num_labels], initializer=tf.zeros_initializer())
|
|
|
|
|
|
with tf.variable_scope("loss"): # 常规的loss function
|
|
|
if is_training:
|
|
|
# I.e., 0.1 dropout
|
|
|
output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)
|
|
|
|
|
|
logits = tf.matmul(output_layer, output_weights, transpose_b=True) # 结果乘上权重
|
|
|
logits = tf.nn.bias_add(logits, output_bias) # 再加上偏值项
|
|
|
probabilities = tf.nn.softmax(logits, axis=-1) # 加上softmax层
|
|
|
log_probs = tf.nn.log_softmax(logits, axis=-1) # 加上softmax层
|
|
|
|
|
|
one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)
|
|
|
|
|
|
per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1) # 计算得到损失
|
|
|
loss = tf.reduce_mean(per_example_loss) # 优化损失
|
|
|
|
|
|
return (loss, per_example_loss, logits, probabilities) # 返回结果
|
|
|
~~~
|
|
|
|
