fix the text_featurize

pull/856/head
huangyuxin 3 years ago
parent cb49aba64f
commit db831ae702

@ -118,8 +118,10 @@ class TextFeaturizer():
"""
text = text.strip()
if replace_space:
text = text.replace(" ", SPACE)
return list(text)
text_list = [SPACE if item == " " else item for item in list(text)]
else:
text_list = list(text)
return text_list
def char_detokenize(self, tokens):
"""Character detokenizer.

Loading…
Cancel
Save