|
|
@ -12,7 +12,6 @@
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
# limitations under the License.
|
|
|
|
"""Contains data helper functions."""
|
|
|
|
"""Contains data helper functions."""
|
|
|
|
import codecs
|
|
|
|
|
|
|
|
import json
|
|
|
|
import json
|
|
|
|
import math
|
|
|
|
import math
|
|
|
|
from typing import List
|
|
|
|
from typing import List
|
|
|
@ -92,12 +91,8 @@ def read_manifest(
|
|
|
|
"""
|
|
|
|
"""
|
|
|
|
|
|
|
|
|
|
|
|
manifest = []
|
|
|
|
manifest = []
|
|
|
|
for json_line in codecs.open(manifest_path, 'r', 'utf-8'):
|
|
|
|
with jsonlines.open(manifest_path, 'r') as reader:
|
|
|
|
try:
|
|
|
|
for json_data in reader:
|
|
|
|
json_data = json.loads(json_line)
|
|
|
|
|
|
|
|
except Exception as e:
|
|
|
|
|
|
|
|
raise IOError("Error reading manifest: %s" % str(e))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
feat_len = json_data["feat_shape"][
|
|
|
|
feat_len = json_data["feat_shape"][
|
|
|
|
0] if 'feat_shape' in json_data else 1.0
|
|
|
|
0] if 'feat_shape' in json_data else 1.0
|
|
|
|
token_len = json_data["token_shape"][
|
|
|
|
token_len = json_data["token_shape"][
|
|
|
|