using jsonlines to read manifest and dump decode result

pull/852/head
Hui Zhang 4 years ago
parent 33b8790252
commit 83a5e0f8d6

@ -12,7 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Contains data helper functions."""
import codecs
import json
import math
from typing import List
@ -92,12 +91,8 @@ def read_manifest(
"""
manifest = []
for json_line in codecs.open(manifest_path, 'r', 'utf-8'):
try:
json_data = json.loads(json_line)
except Exception as e:
raise IOError("Error reading manifest: %s" % str(e))
with jsonlines.open(manifest_path, 'r') as reader:
for json_data in reader:
feat_len = json_data["feat_shape"][
0] if 'feat_shape' in json_data else 1.0
token_len = json_data["token_shape"][

Loading…
Cancel
Save