|
|
@ -118,8 +118,10 @@ class TextFeaturizer():
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
text = text.strip()
|
|
|
|
text = text.strip()
|
|
|
|
if replace_space:
|
|
|
|
if replace_space:
|
|
|
|
text = text.replace(" ", SPACE)
|
|
|
|
text_list = [SPACE if item == " " else item for item in list(text)]
|
|
|
|
return list(text)
|
|
|
|
else:
|
|
|
|
|
|
|
|
text_list = list(text)
|
|
|
|
|
|
|
|
return text_list
|
|
|
|
|
|
|
|
|
|
|
|
def char_detokenize(self, tokens):
|
|
|
|
def char_detokenize(self, tokens):
|
|
|
|
"""Character detokenizer.
|
|
|
|
"""Character detokenizer.
|
|
|
|