PaddleSpeech/speechx/examples/ngram/zh/local/text_to_lexicon.py

#!/usr/bin/env python3
import argparse
from collections import Counter


def main(args):
    counter = Counter()
    with open(args.text, 'r') as fin, open(args.lexicon, 'w') as fout:
        for line in fin:
            line = line.strip()
            if args.has_key:
                utt, text = line.split(maxsplit=1)
                words = text.split()
            else:
                words = line.split()

            counter.update(words)

        for word in counter:
            val = " ".join(list(word))
            fout.write(f"{word}\t{val}\n")
            fout.flush()


if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description='text(line:utt1 中国 人) to lexicon（line:中国 中 国).')
    parser.add_argument(
        '--has_key', default=True, help='text path, with utt or not')
    parser.add_argument(
        '--text', required=True, help='text path. line: utt1 中国 人 or 中国 人')
    parser.add_argument(
        '--lexicon', required=True, help='lexicon path. line:中国 中 国')
    args = parser.parse_args()
    print(args)

    main(args)