|
|
|
|
# Copyright (c) 2017-2019 NVIDIA CORPORATION. All rights reserved.
|
|
|
|
|
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
|
|
|
|
|
# See the LICENSE file for licensing terms (BSD-style).
|
|
|
|
|
# Modified from https://github.com/webdataset/webdataset
|
|
|
|
|
#
|
|
|
|
|
# flake8: noqa
|
|
|
|
|
from .cache import cached_tarfile_samples
|
|
|
|
|
from .cache import cached_tarfile_to_samples
|
|
|
|
|
from .cache import lru_cleanup
|
|
|
|
|
from .cache import pipe_cleaner
|
|
|
|
|
from .compat import FluidWrapper
|
|
|
|
|
from .compat import WebDataset
|
|
|
|
|
from .compat import WebLoader
|
|
|
|
|
from .extradatasets import MockDataset
|
|
|
|
|
from .extradatasets import with_epoch
|
|
|
|
|
from .extradatasets import with_length
|
|
|
|
|
from .filters import associate
|
|
|
|
|
from .filters import audio_cmvn
|
|
|
|
|
from .filters import audio_compute_fbank
|
|
|
|
|
from .filters import audio_data_filter
|
|
|
|
|
from .filters import audio_padding
|
|
|
|
|
from .filters import audio_resample
|
|
|
|
|
from .filters import audio_spec_aug
|
|
|
|
|
from .filters import audio_tokenize
|
|
|
|
|
from .filters import batched
|
|
|
|
|
from .filters import decode
|
|
|
|
|
from .filters import detshuffle
|
|
|
|
|
from .filters import extract_keys
|
|
|
|
|
from .filters import getfirst
|
|
|
|
|
from .filters import info
|
|
|
|
|
from .filters import map
|
|
|
|
|
from .filters import map_dict
|
|
|
|
|
from .filters import map_tuple
|
|
|
|
|
from .filters import pipelinefilter
|
|
|
|
|
from .filters import placeholder
|
|
|
|
|
from .filters import rename
|
|
|
|
|
from .filters import rename_keys
|
|
|
|
|
from .filters import select
|
|
|
|
|
from .filters import shuffle
|
|
|
|
|
from .filters import slice
|
|
|
|
|
from .filters import sort
|
|
|
|
|
from .filters import to_tuple
|
|
|
|
|
from .filters import transform_with
|
|
|
|
|
from .filters import unbatched
|
|
|
|
|
from .filters import xdecode
|
|
|
|
|
from .handlers import ignore_and_continue
|
|
|
|
|
from .handlers import ignore_and_stop
|
|
|
|
|
from .handlers import reraise_exception
|
|
|
|
|
from .handlers import warn_and_continue
|
|
|
|
|
from .handlers import warn_and_stop
|
|
|
|
|
from .mix import RandomMix
|
|
|
|
|
from .mix import RoundRobin
|
|
|
|
|
from .pipeline import DataPipeline
|
|
|
|
|
from .shardlists import MultiShardSample
|
|
|
|
|
from .shardlists import non_empty
|
|
|
|
|
from .shardlists import resampled
|
|
|
|
|
from .shardlists import ResampledShards
|
|
|
|
|
from .shardlists import shardspec
|
|
|
|
|
from .shardlists import SimpleShardList
|
|
|
|
|
from .shardlists import single_node_only
|
|
|
|
|
from .shardlists import split_by_node
|
|
|
|
|
from .shardlists import split_by_worker
|
|
|
|
|
from .tariterators import tarfile_samples
|
|
|
|
|
from .tariterators import tarfile_to_samples
|
|
|
|
|
from .utils import PipelineStage
|
|
|
|
|
from .utils import repeatedly
|
|
|
|
|
from .writer import numpy_dumps
|
|
|
|
|
from .writer import ShardWriter
|
|
|
|
|
from .writer import TarWriter
|
