parallel data scripts; more mask test; need pybind11 repo

.gitignore

@@ -1,7 +1,7 @@
 .DS_Store
 *.pyc
 .vscode
-*.log
+*log
 *.pdmodel
 *.pdiparams*
 *.zip

deepspeech/exps/deepspeech2/model.py

@@ -170,7 +170,7 @@ class DeepSpeech2Trainer(Trainer):
             train_dataset,
             batch_sampler=batch_sampler,
             collate_fn=collate_fn,
-            num_workers=config.data.num_workers, )
+            num_workers=config.data.num_workers)
         self.valid_loader = DataLoader(
             dev_dataset,
             batch_size=config.data.batch_size,

examples/aishell/s0/local/data.sh

@@ -66,19 +66,22 @@ fi
 if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     # format manifest with tokenids, vocab size
     for dataset in train dev test; do
+    {
         python3 ${MAIN_ROOT}/utils/format_data.py \
-        --feat_type "raw" \
+                --feat_type "raw" \
-        --cmvn_path "data/mean_std.json" \
+                --cmvn_path "data/mean_std.json" \
-        --unit_type "char" \
+                --unit_type "char" \
-        --vocab_path="data/vocab.txt" \
+                --vocab_path="data/vocab.txt" \
-        --manifest_path="data/manifest.${dataset}.raw" \
+                --manifest_path="data/manifest.${dataset}.raw" \
-        --output_path="data/manifest.${dataset}"
+                --output_path="data/manifest.${dataset}"
-    done
 
-    if [ $? -ne 0 ]; then
+        if [ $? -ne 0 ]; then
-        echo "Formt mnaifest failed. Terminated."
+                echo "Formt mnaifest failed. Terminated."
-        exit 1
+                exit 1
-    fi
+        fi
+    } &
+    done
+    wait
 fi
 
 echo "Aishell data preparation done."

examples/aishell/s1/local/data.sh

@@ -67,19 +67,22 @@ fi
 if [ ${stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
     # format manifest with tokenids, vocab size
     for dataset in train dev test; do
+    {
         python3 ${MAIN_ROOT}/utils/format_data.py \
-        --feat_type "raw" \
+            --feat_type "raw" \
-        --cmvn_path "data/mean_std.json" \
+            --cmvn_path "data/mean_std.json" \
-        --unit_type "char" \
+            --unit_type "char" \
-        --vocab_path="data/vocab.txt" \
+            --vocab_path="data/vocab.txt" \
-        --manifest_path="data/manifest.${dataset}.raw" \
+            --manifest_path="data/manifest.${dataset}.raw" \
-        --output_path="data/manifest.${dataset}"
+            --output_path="data/manifest.${dataset}"
-    done
 
-    if [ $? -ne 0 ]; then
+        if [ $? -ne 0 ]; then
-        echo "Formt mnaifest failed. Terminated."
+            echo "Formt mnaifest failed. Terminated."
-        exit 1
+            exit 1
-    fi
+        fi
+    } &
+    done
+    wait
 fi
 
 echo "Aishell data preparation done."

requirements.txt

@@ -9,3 +9,4 @@ sox
 tensorboardX
 typeguard
 yacs
+pybind11

tests/mask_test.py

@@ -48,7 +48,9 @@ class TestU2Model(unittest.TestCase):
 
     def test_make_pad_mask(self):
         res = make_pad_mask(self.lengths)
+        res1 = make_non_pad_mask(self.lengths).logical_not()
         self.assertSequenceEqual(res.numpy().tolist(), self.pad_masks.tolist())
+        self.assertSequenceEqual(res.numpy().tolist(), res1.tolist())
 
 
 if __name__ == '__main__':