|
|
@ -1,15 +1,22 @@
|
|
|
|
|
|
|
|
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
|
|
|
import argparse
|
|
|
|
#
|
|
|
|
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
|
|
|
|
#
|
|
|
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
|
|
|
#
|
|
|
|
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
|
|
|
|
# limitations under the License.
|
|
|
|
import copy
|
|
|
|
import copy
|
|
|
|
import json
|
|
|
|
|
|
|
|
import os
|
|
|
|
import os
|
|
|
|
import shutil
|
|
|
|
|
|
|
|
import tempfile
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
import numpy as np
|
|
|
|
|
|
|
|
|
|
|
|
from . import extension
|
|
|
|
from . import extension
|
|
|
|
from ..updaters.trainer import Trainer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class PlotAttentionReport(extension.Extension):
|
|
|
|
class PlotAttentionReport(extension.Extension):
|
|
|
@ -49,8 +56,7 @@ class PlotAttentionReport(extension.Extension):
|
|
|
|
iaxis=0,
|
|
|
|
iaxis=0,
|
|
|
|
okey="output",
|
|
|
|
okey="output",
|
|
|
|
oaxis=0,
|
|
|
|
oaxis=0,
|
|
|
|
subsampling_factor=1,
|
|
|
|
subsampling_factor=1, ):
|
|
|
|
):
|
|
|
|
|
|
|
|
self.att_vis_fn = att_vis_fn
|
|
|
|
self.att_vis_fn = att_vis_fn
|
|
|
|
self.data = copy.deepcopy(data)
|
|
|
|
self.data = copy.deepcopy(data)
|
|
|
|
self.data_dict = {k: v for k, v in copy.deepcopy(data)}
|
|
|
|
self.data_dict = {k: v for k, v in copy.deepcopy(data)}
|
|
|
@ -77,44 +83,30 @@ class PlotAttentionReport(extension.Extension):
|
|
|
|
for i in range(num_encs):
|
|
|
|
for i in range(num_encs):
|
|
|
|
for idx, att_w in enumerate(att_ws[i]):
|
|
|
|
for idx, att_w in enumerate(att_ws[i]):
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.att%d.png" % (
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.att%d.png" % (
|
|
|
|
self.outdir,
|
|
|
|
self.outdir, uttid_list[idx], i + 1, )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
i + 1,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.att%d.npy" % (
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.att%d.npy" % (
|
|
|
|
self.outdir,
|
|
|
|
self.outdir, uttid_list[idx], i + 1, )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
i + 1,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
np.save(np_filename.format(trainer), att_w)
|
|
|
|
np.save(np_filename.format(trainer), att_w)
|
|
|
|
self._plot_and_save_attention(att_w, filename.format(trainer))
|
|
|
|
self._plot_and_save_attention(att_w,
|
|
|
|
|
|
|
|
filename.format(trainer))
|
|
|
|
# han
|
|
|
|
# han
|
|
|
|
for idx, att_w in enumerate(att_ws[num_encs]):
|
|
|
|
for idx, att_w in enumerate(att_ws[num_encs]):
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.han.png" % (
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.han.png" % (
|
|
|
|
self.outdir,
|
|
|
|
self.outdir, uttid_list[idx], )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.han.npy" % (
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.han.npy" % (
|
|
|
|
self.outdir,
|
|
|
|
self.outdir, uttid_list[idx], )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
np.save(np_filename.format(trainer), att_w)
|
|
|
|
np.save(np_filename.format(trainer), att_w)
|
|
|
|
self._plot_and_save_attention(
|
|
|
|
self._plot_and_save_attention(
|
|
|
|
att_w, filename.format(trainer), han_mode=True
|
|
|
|
att_w, filename.format(trainer), han_mode=True)
|
|
|
|
)
|
|
|
|
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
for idx, att_w in enumerate(att_ws):
|
|
|
|
for idx, att_w in enumerate(att_ws):
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.png" % (
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.png" % (self.outdir,
|
|
|
|
self.outdir,
|
|
|
|
uttid_list[idx], )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.npy" % (
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.npy" % (
|
|
|
|
self.outdir,
|
|
|
|
self.outdir, uttid_list[idx], )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
np.save(np_filename.format(trainer), att_w)
|
|
|
|
np.save(np_filename.format(trainer), att_w)
|
|
|
|
self._plot_and_save_attention(att_w, filename.format(trainer))
|
|
|
|
self._plot_and_save_attention(att_w, filename.format(trainer))
|
|
|
|
|
|
|
|
|
|
|
@ -131,8 +123,7 @@ class PlotAttentionReport(extension.Extension):
|
|
|
|
logger.add_figure(
|
|
|
|
logger.add_figure(
|
|
|
|
"%s_att%d" % (uttid_list[idx], i + 1),
|
|
|
|
"%s_att%d" % (uttid_list[idx], i + 1),
|
|
|
|
plot.gcf(),
|
|
|
|
plot.gcf(),
|
|
|
|
step,
|
|
|
|
step, )
|
|
|
|
)
|
|
|
|
|
|
|
|
# han
|
|
|
|
# han
|
|
|
|
for idx, att_w in enumerate(att_ws[num_encs]):
|
|
|
|
for idx, att_w in enumerate(att_ws[num_encs]):
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
@ -140,8 +131,7 @@ class PlotAttentionReport(extension.Extension):
|
|
|
|
logger.add_figure(
|
|
|
|
logger.add_figure(
|
|
|
|
"%s_han" % (uttid_list[idx]),
|
|
|
|
"%s_han" % (uttid_list[idx]),
|
|
|
|
plot.gcf(),
|
|
|
|
plot.gcf(),
|
|
|
|
step,
|
|
|
|
step, )
|
|
|
|
)
|
|
|
|
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
for idx, att_w in enumerate(att_ws):
|
|
|
|
for idx, att_w in enumerate(att_ws):
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
|
att_w = self.trim_attention_weight(uttid_list[idx], att_w)
|
|
|
@ -298,8 +288,7 @@ class PlotCTCReport(extension.Extension):
|
|
|
|
iaxis=0,
|
|
|
|
iaxis=0,
|
|
|
|
okey="output",
|
|
|
|
okey="output",
|
|
|
|
oaxis=0,
|
|
|
|
oaxis=0,
|
|
|
|
subsampling_factor=1,
|
|
|
|
subsampling_factor=1, ):
|
|
|
|
):
|
|
|
|
|
|
|
|
self.ctc_vis_fn = ctc_vis_fn
|
|
|
|
self.ctc_vis_fn = ctc_vis_fn
|
|
|
|
self.data = copy.deepcopy(data)
|
|
|
|
self.data = copy.deepcopy(data)
|
|
|
|
self.data_dict = {k: v for k, v in copy.deepcopy(data)}
|
|
|
|
self.data_dict = {k: v for k, v in copy.deepcopy(data)}
|
|
|
@ -325,29 +314,19 @@ class PlotCTCReport(extension.Extension):
|
|
|
|
for i in range(num_encs):
|
|
|
|
for i in range(num_encs):
|
|
|
|
for idx, ctc_prob in enumerate(ctc_probs[i]):
|
|
|
|
for idx, ctc_prob in enumerate(ctc_probs[i]):
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.ctc%d.png" % (
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.ctc%d.png" % (
|
|
|
|
self.outdir,
|
|
|
|
self.outdir, uttid_list[idx], i + 1, )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
i + 1,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
ctc_prob = self.trim_ctc_prob(uttid_list[idx], ctc_prob)
|
|
|
|
ctc_prob = self.trim_ctc_prob(uttid_list[idx], ctc_prob)
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.ctc%d.npy" % (
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.ctc%d.npy" % (
|
|
|
|
self.outdir,
|
|
|
|
self.outdir, uttid_list[idx], i + 1, )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
i + 1,
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
np.save(np_filename.format(trainer), ctc_prob)
|
|
|
|
np.save(np_filename.format(trainer), ctc_prob)
|
|
|
|
self._plot_and_save_ctc(ctc_prob, filename.format(trainer))
|
|
|
|
self._plot_and_save_ctc(ctc_prob, filename.format(trainer))
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
for idx, ctc_prob in enumerate(ctc_probs):
|
|
|
|
for idx, ctc_prob in enumerate(ctc_probs):
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.png" % (
|
|
|
|
filename = "%s/%s.ep.{.updater.epoch}.png" % (self.outdir,
|
|
|
|
self.outdir,
|
|
|
|
uttid_list[idx], )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
ctc_prob = self.trim_ctc_prob(uttid_list[idx], ctc_prob)
|
|
|
|
ctc_prob = self.trim_ctc_prob(uttid_list[idx], ctc_prob)
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.npy" % (
|
|
|
|
np_filename = "%s/%s.ep.{.updater.epoch}.npy" % (
|
|
|
|
self.outdir,
|
|
|
|
self.outdir, uttid_list[idx], )
|
|
|
|
uttid_list[idx],
|
|
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
np.save(np_filename.format(trainer), ctc_prob)
|
|
|
|
np.save(np_filename.format(trainer), ctc_prob)
|
|
|
|
self._plot_and_save_ctc(ctc_prob, filename.format(trainer))
|
|
|
|
self._plot_and_save_ctc(ctc_prob, filename.format(trainer))
|
|
|
|
|
|
|
|
|
|
|
@ -363,8 +342,7 @@ class PlotCTCReport(extension.Extension):
|
|
|
|
logger.add_figure(
|
|
|
|
logger.add_figure(
|
|
|
|
"%s_ctc%d" % (uttid_list[idx], i + 1),
|
|
|
|
"%s_ctc%d" % (uttid_list[idx], i + 1),
|
|
|
|
plot.gcf(),
|
|
|
|
plot.gcf(),
|
|
|
|
step,
|
|
|
|
step, )
|
|
|
|
)
|
|
|
|
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
for idx, ctc_prob in enumerate(ctc_probs):
|
|
|
|
for idx, ctc_prob in enumerate(ctc_probs):
|
|
|
|
ctc_prob = self.trim_ctc_prob(uttid_list[idx], ctc_prob)
|
|
|
|
ctc_prob = self.trim_ctc_prob(uttid_list[idx], ctc_prob)
|
|
|
@ -420,8 +398,11 @@ class PlotCTCReport(extension.Extension):
|
|
|
|
for idx in set(topk_ids.reshape(-1).tolist()):
|
|
|
|
for idx in set(topk_ids.reshape(-1).tolist()):
|
|
|
|
if idx == 0:
|
|
|
|
if idx == 0:
|
|
|
|
plt.plot(
|
|
|
|
plt.plot(
|
|
|
|
times_probs, ctc_prob[:, 0], ":", label="<blank>", color="grey"
|
|
|
|
times_probs,
|
|
|
|
)
|
|
|
|
ctc_prob[:, 0],
|
|
|
|
|
|
|
|
":",
|
|
|
|
|
|
|
|
label="<blank>",
|
|
|
|
|
|
|
|
color="grey")
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
plt.plot(times_probs, ctc_prob[:, idx])
|
|
|
|
plt.plot(times_probs, ctc_prob[:, idx])
|
|
|
|
plt.xlabel(u"Input [frame]", fontsize=12)
|
|
|
|
plt.xlabel(u"Input [frame]", fontsize=12)
|
|
|
|