|
|
@ -83,7 +83,7 @@ def main():
|
|
|
|
# train
|
|
|
|
# train
|
|
|
|
spm.SentencePieceTrainer.Train(
|
|
|
|
spm.SentencePieceTrainer.Train(
|
|
|
|
input=fp.name,
|
|
|
|
input=fp.name,
|
|
|
|
vocab_size=args.vocab_size,
|
|
|
|
vocab_size=args.spm_vocab_size,
|
|
|
|
model_type=args.spm_mode,
|
|
|
|
model_type=args.spm_mode,
|
|
|
|
model_prefix=args.spm_model_prefix,
|
|
|
|
model_prefix=args.spm_model_prefix,
|
|
|
|
input_sentence_size=100000000,
|
|
|
|
input_sentence_size=100000000,
|
|
|
|