install kenlm, zhon

.gitignore

@@ -1,6 +1,5 @@
 .DS_Store
 *.pyc
-tools/venv
 .vscode
 *.log
 *.pdmodel
@@ -10,3 +9,6 @@ tools/venv
 *.tar.gz
 .ipynb_checkpoints
 *.npz
+
+tools/venv
+tools/kenlm

setup.sh

@@ -57,11 +57,11 @@ if [ $? != 0 ]; then
 fi
 
 
-# install kaldi-comptiable feature 
-pushd third_party/python_kaldi_features/
-python setup.py install
+# install third_party
+pushd third_party
+bash install.sh
 if [ $? != 0 ]; then
-   error_msg "Please check why kaldi feature install error!"
+   error_msg "Please check why third_party install error!"
    exit -1
 fi
 popd

third_party/install.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# install kaldi-comptiable feature
+pushd python_kaldi_features
+python3 setup.py install
+if [ $? != 0 ]; then
+   error_msg "Please check why kaldi feature install error!"
+   exit -1
+fi
+popd
+
+# install zhon
+pushd zhon
+python3 setup.py install
+if [ $? != 0 ]; then
+   error_msg "Please check why zhon install error!"
+   exit -1
+fi
+popd
+
+# install pypinyin
+pushd python-pinyin
+python3 setup.py install
+if [ $? != 0 ]; then
+   error_msg "Please check why pypinyin install error!"
+   exit -1
+fi
+popd
+
+# install mmseg
+pushd pymmseg-cpp/
+python3 setup.py install
+if [ $? != 0 ]; then
+   error_msg "Please check why pymmseg install error!"
+   exit -1
+fi
+popd
+

third_party/zhon/.gitignore

@@ -0,0 +1,3 @@
+build
+dist
+*egg-info

tools/Makefile

@@ -1,12 +1,21 @@
 PYTHON:= python3.8
-.PHONY: all clean
+.PHONY: all clean kenlm
 
-all: virtualenv
+all: virtualenv kenlm
 
 virtualenv:
 	test -d venv || virtualenv -p $(PYTHON) venv
 	touch venv/bin/activate
 
+kenlm:
+	# Ubuntu 16.04 透過 apt 會安裝 boost 1.58.0
+	# it seems that boost (1.54.0) requires higher version. After I switched to g++-5 it compiles normally.
+	apt install -y build-essential cmake libboost-system-dev libboost-thread-dev libboost-program-options-dev libboost-test-dev libeigen3-dev zlib1g-dev libbz2-dev liblzma-dev
+	apt-get install -y gcc-5 g++-5 && update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 50  && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 50
+	test -d kenlm || wget -O - https://kheafield.com/code/kenlm.tar.gz | tar xz
+	mkdir -p kenlm/build && cd kenlm/build && cmake .. && make -j4 && make install
+	cd kenlm && python setup.py install
+
 clean:
 	rm -fr venv
 	find -iname "*.pyc" -delete