|
|
@ -61,7 +61,8 @@ class MixFrontend():
|
|
|
|
return False
|
|
|
|
return False
|
|
|
|
|
|
|
|
|
|
|
|
def is_end(self, before_char, after_char) -> bool:
|
|
|
|
def is_end(self, before_char, after_char) -> bool:
|
|
|
|
if ((self.is_alphabet(before_char) or before_char == " ") and (self.is_alphabet(after_char) or after_char == " ")):
|
|
|
|
if ((self.is_alphabet(before_char) or before_char == " ") and
|
|
|
|
|
|
|
|
(self.is_alphabet(after_char) or after_char == " ")):
|
|
|
|
return True
|
|
|
|
return True
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
return False
|
|
|
|
return False
|
|
|
@ -86,7 +87,8 @@ class MixFrontend():
|
|
|
|
if point_index == 0 or point_index == len(text) - 1:
|
|
|
|
if point_index == 0 or point_index == len(text) - 1:
|
|
|
|
new_text = text
|
|
|
|
new_text = text
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
if not self.is_end(text[point_index - 1], text[point_index + 1]):
|
|
|
|
if not self.is_end(text[point_index - 1], text[point_index +
|
|
|
|
|
|
|
|
1]):
|
|
|
|
new_text = text
|
|
|
|
new_text = text
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
new_text = text[:point_index] + "。" + text[point_index + 1:]
|
|
|
|
new_text = text[:point_index] + "。" + text[point_index + 1:]
|
|
|
@ -97,7 +99,8 @@ class MixFrontend():
|
|
|
|
|
|
|
|
|
|
|
|
# first
|
|
|
|
# first
|
|
|
|
if first_index != 0:
|
|
|
|
if first_index != 0:
|
|
|
|
if not self.is_end(text[first_index - 1], text[first_index + 1]):
|
|
|
|
if not self.is_end(text[first_index - 1], text[first_index +
|
|
|
|
|
|
|
|
1]):
|
|
|
|
new_text += (text[:first_index] + ".")
|
|
|
|
new_text += (text[:first_index] + ".")
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
new_text += (text[:first_index] + "。")
|
|
|
|
new_text += (text[:first_index] + "。")
|
|
|
@ -108,7 +111,8 @@ class MixFrontend():
|
|
|
|
if not self.is_end(text[end_index - 1], text[end_index + 1]):
|
|
|
|
if not self.is_end(text[end_index - 1], text[end_index + 1]):
|
|
|
|
new_text += text[point_indexs[-2] + 1:]
|
|
|
|
new_text += text[point_indexs[-2] + 1:]
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
new_text += (text[point_indexs[-2] + 1 : end_index] + "。" + text[end_index + 1 : ])
|
|
|
|
new_text += (text[point_indexs[-2] + 1:end_index] + "。" +
|
|
|
|
|
|
|
|
text[end_index + 1:])
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
new_text += "."
|
|
|
|
new_text += "."
|
|
|
|
|
|
|
|
|
|
|
@ -117,7 +121,8 @@ class MixFrontend():
|
|
|
|
end_index = point_indexs[-1]
|
|
|
|
end_index = point_indexs[-1]
|
|
|
|
# first
|
|
|
|
# first
|
|
|
|
if first_index != 0:
|
|
|
|
if first_index != 0:
|
|
|
|
if not self.is_end(text[first_index - 1], text[first_index + 1]):
|
|
|
|
if not self.is_end(text[first_index - 1], text[first_index +
|
|
|
|
|
|
|
|
1]):
|
|
|
|
new_text += (text[:first_index] + ".")
|
|
|
|
new_text += (text[:first_index] + ".")
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
new_text += (text[:first_index] + "。")
|
|
|
|
new_text += (text[:first_index] + "。")
|
|
|
@ -126,16 +131,20 @@ class MixFrontend():
|
|
|
|
# middle
|
|
|
|
# middle
|
|
|
|
for j in range(1, len(point_indexs) - 1):
|
|
|
|
for j in range(1, len(point_indexs) - 1):
|
|
|
|
point_index = point_indexs[j]
|
|
|
|
point_index = point_indexs[j]
|
|
|
|
if not self.is_end(text[point_index - 1], text[point_index + 1]):
|
|
|
|
if not self.is_end(text[point_index - 1], text[point_index +
|
|
|
|
new_text += (text[point_indexs[j-1] + 1 : point_index] + ".")
|
|
|
|
1]):
|
|
|
|
|
|
|
|
new_text += (
|
|
|
|
|
|
|
|
text[point_indexs[j - 1] + 1:point_index] + ".")
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
new_text += (text[point_indexs[j-1] + 1 : point_index] + "。")
|
|
|
|
new_text += (
|
|
|
|
|
|
|
|
text[point_indexs[j - 1] + 1:point_index] + "。")
|
|
|
|
# last
|
|
|
|
# last
|
|
|
|
if end_index != len(text) - 1:
|
|
|
|
if end_index != len(text) - 1:
|
|
|
|
if not self.is_end(text[end_index - 1], text[end_index + 1]):
|
|
|
|
if not self.is_end(text[end_index - 1], text[end_index + 1]):
|
|
|
|
new_text += text[point_indexs[-2] + 1:]
|
|
|
|
new_text += text[point_indexs[-2] + 1:]
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
new_text += (text[point_indexs[-2] + 1 : end_index] + "。" + text[end_index + 1 : ])
|
|
|
|
new_text += (text[point_indexs[-2] + 1:end_index] + "。" +
|
|
|
|
|
|
|
|
text[end_index + 1:])
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
new_text += "."
|
|
|
|
new_text += "."
|
|
|
|
|
|
|
|
|
|
|
@ -224,7 +233,7 @@ class MixFrontend():
|
|
|
|
|
|
|
|
|
|
|
|
def get_input_ids(self,
|
|
|
|
def get_input_ids(self,
|
|
|
|
sentence: str,
|
|
|
|
sentence: str,
|
|
|
|
merge_sentences: bool=True,
|
|
|
|
merge_sentences: bool=False,
|
|
|
|
get_tone_ids: bool=False,
|
|
|
|
get_tone_ids: bool=False,
|
|
|
|
add_sp: bool=True,
|
|
|
|
add_sp: bool=True,
|
|
|
|
to_tensor: bool=True) -> Dict[str, List[paddle.Tensor]]:
|
|
|
|
to_tensor: bool=True) -> Dict[str, List[paddle.Tensor]]:
|
|
|
@ -232,28 +241,29 @@ class MixFrontend():
|
|
|
|
sentences = self._split(sentence)
|
|
|
|
sentences = self._split(sentence)
|
|
|
|
phones_list = []
|
|
|
|
phones_list = []
|
|
|
|
result = {}
|
|
|
|
result = {}
|
|
|
|
|
|
|
|
|
|
|
|
for text in sentences:
|
|
|
|
for text in sentences:
|
|
|
|
phones_seg = []
|
|
|
|
phones_seg = []
|
|
|
|
segments = self._distinguish(text)
|
|
|
|
segments = self._distinguish(text)
|
|
|
|
for seg in segments:
|
|
|
|
for seg in segments:
|
|
|
|
content = seg[0]
|
|
|
|
content = seg[0]
|
|
|
|
lang = seg[1]
|
|
|
|
lang = seg[1]
|
|
|
|
if lang == "zh":
|
|
|
|
if content != '':
|
|
|
|
|
|
|
|
if lang == "en":
|
|
|
|
|
|
|
|
input_ids = self.en_frontend.get_input_ids(
|
|
|
|
|
|
|
|
content, merge_sentences=True, to_tensor=to_tensor)
|
|
|
|
|
|
|
|
else:
|
|
|
|
input_ids = self.zh_frontend.get_input_ids(
|
|
|
|
input_ids = self.zh_frontend.get_input_ids(
|
|
|
|
content,
|
|
|
|
content,
|
|
|
|
merge_sentences=True,
|
|
|
|
merge_sentences=True,
|
|
|
|
get_tone_ids=get_tone_ids,
|
|
|
|
get_tone_ids=get_tone_ids,
|
|
|
|
to_tensor=to_tensor)
|
|
|
|
to_tensor=to_tensor)
|
|
|
|
|
|
|
|
|
|
|
|
elif lang == "en":
|
|
|
|
|
|
|
|
input_ids = self.en_frontend.get_input_ids(
|
|
|
|
|
|
|
|
content, merge_sentences=True, to_tensor=to_tensor)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
phones_seg.append(input_ids["phone_ids"][0])
|
|
|
|
phones_seg.append(input_ids["phone_ids"][0])
|
|
|
|
if add_sp:
|
|
|
|
if add_sp:
|
|
|
|
phones_seg.append(self.sp_id_tensor)
|
|
|
|
phones_seg.append(self.sp_id_tensor)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if phones_seg == []:
|
|
|
|
|
|
|
|
phones_seg.append(self.sp_id_tensor)
|
|
|
|
phones = paddle.concat(phones_seg)
|
|
|
|
phones = paddle.concat(phones_seg)
|
|
|
|
phones_list.append(phones)
|
|
|
|
phones_list.append(phones)
|
|
|
|
|
|
|
|
|
|
|
|