Repository navigation
Expand file tree
/
Copy pathprocess_fisher_data.py
More file actions
407 lines (348 loc) · 13.1 KB
/
Copy pathprocess_fisher_data.py
File metadata and controls
407 lines (348 loc) · 13.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# USAGE:
# python process_fisher_data.py \
# --audio_root=<audio (.wav) directory>
# --transcript_root=<LDC Fisher dataset directory> \
# --dest_root=<destination directory> \
# --data_sets=LDC2004S13-Part1,LDC2005S13-Part2 \
# --remove_noises
#
# Matches Fisher dataset transcripts to the corresponding audio file (.wav),
# and slices them into min_slice_duration segments with one speaker.
# Also performs some other processing on transcripts.
#
# Heavily derived from Patter's Fisher processing script.
import argparse
import glob
import json
import os
import re
from math import ceil, floor
import numpy as np
import scipy.io.wavfile as wavfile
from tqdm import tqdm
parser = argparse.ArgumentParser(description="Fisher Data Processing")
parser.add_argument(
"--audio_root", default=None, type=str, required=True, help="The path to the root of the audio (wav) data folder.",
)
parser.add_argument(
"--transcript_root",
default=None,
type=str,
required=True,
help="The path to the root of the transcript data folder.",
)
parser.add_argument(
"--dest_root", default=None, type=str, required=True, help="Path to the destination root directory.",
)
# Optional arguments
parser.add_argument(
"--min_slice_duration", default=10.0, type=float, help="Minimum audio slice duration after processing.",
)
parser.add_argument(
"--keep_low_conf", action="store_true", help="Keep all utterances with low confidence transcripts",
)
parser.add_argument(
"--remove_noises", action="store_true", help="Removes transcripted noises such as [laughter].",
)
parser.add_argument(
"--noises_to_emoji", action="store_true", help="Converts transcripts for noises to an emoji character.",
)
args = parser.parse_args()
# Total number of files before segmenting, and train/val/test splits
NUM_FILES = 5850 + 5849
TRAIN_END_IDX = int(NUM_FILES * 0.8)
VAL_END_IDX = int(NUM_FILES * 0.9)
# Known transcription errors and their fixes (from Mozilla)
TRANSCRIPT_BUGS = {
"fe_03_00265-B-3353-3381": "correct",
"fe_03_00991-B-52739-52829": "that's one of those",
"fe_03_10282-A-34442-34484.wav": "they don't want",
"fe_03_10677-B-10104-10641": "uh my mine yeah the german shepherd "
+ "pitbull mix he snores almost as loud "
+ "as i do",
"fe_03_00027-B-39380-39405": None,
"fe_03_11487-B-3109-23406": None,
"fe_03_01326-A-30742-30793": None,
}
TRANSCRIPT_NUMBERS = {
"401k": "four o one k",
"f16": "f sixteen",
"m16": "m sixteen",
"ak47": "a k forty seven",
"v8": "v eight",
"y2k": "y two k",
"mp3": "m p three",
"vh1": "v h one",
"90210": "nine o two one o",
"espn2": "e s p n two",
"u2": "u two",
"dc3s": "d c threes",
"book 2": "book two",
"s2b": "s two b",
"3d": "three d",
}
TAG_MAP = {
"[laughter]": "🤣",
"[laugh]": "🤣",
"[noise]": "😕",
"[sigh]": "😕",
"[cough]": "😕",
"[mn]": "😕",
"[breath]": "😕",
"[lipsmack]": "😕",
"[[skip]]": "",
"[pause]": "",
"[sneeze]": "😕",
}
def __write_sample(dest, file_id, count, file_count, sample_rate, audio, duration, transcript):
"""
Writes one slice to the given target directory.
Args:
dest: the destination directory
file_id: name of the transcript/audio file for this block
count: the count of segments in the file so far
file_count: the total number of filse processed so far
sample rate: sample rate of the audio data
audio: audio data of the current sample
duration: audio duration of the current sample
transcript: transcript of the current sample
"""
partition = __partition_name(file_count)
audio_path = os.path.join(dest, partition, f"{file_id}_{count:03}.wav")
# Write audio
wavfile.write(audio_path, sample_rate, audio)
# Write transcript info
transcript = {
"audio_filepath": audio_path,
"duration": duration,
"text": transcript,
}
# Append to manifest
manifest_path = os.path.join(dest, f"manifest_{partition}.json")
with open(manifest_path, 'a') as f:
json.dump(transcript, f)
f.write('\n')
def __normalize(utt):
replace_table = str.maketrans(dict.fromkeys('()*;:"!&{},.-?'))
utt = (
utt.lower()
.replace('[uh]', 'uh')
.replace('[um]', 'um')
.replace('<noise>', '[noise]')
.replace('<spoken_noise>', '[vocalized-noise]')
.replace('.period', 'period')
.replace('.dot', 'dot')
.replace('-hyphen', 'hyphen')
.replace('._', ' ')
.translate(replace_table)
)
utt = re.sub(r"'([a-z]+)'", r'\1', utt) # Unquote quoted words
return utt
def __process_utterance(file_id, trans_path, line, keep_low_conf, rem_noises, emojify):
"""
Processes one utterance (one line of a transcript).
Args:
file_id: the ID of the transcript file
trans_path: transcript path
line: one line in the transcript file
keep_low_conf: whether to keep low confidence lines
rem_noises: whether to remove noise symbols
emojify: whether to convert noise symbols to emoji, lower precedence
"""
# Check for lines to skip (comments, empty, low confidence)
if line.startswith('#') or not line.strip() or (not keep_low_conf and '((' in line):
return None, None, None, None
# Data and sanity checks
line = line.split()
t_start, t_end = float(line[0]), float(line[1])
if (t_start < 0) or (t_end < t_start):
print(f"Invalid time: {t_start} to {t_end} in {trans_path}")
return None, None, None, None
channel = line[2]
idx = 0 if line[2] == 'A:' else 1
if channel not in ('A:', 'B:'):
print(f"Could not read channel info ({channel}) in {trans_path}")
return None, None, None, None
# Replacements as necessary
line_id = '-'.join([file_id, channel[0], str(t_start * 10), str(t_end * 10)])
content = TRANSCRIPT_BUGS.get(line_id, ' '.join(line[3:]))
if content is None:
return None, None, None, None
for tag, newtag in TRANSCRIPT_NUMBERS.items():
content = content.replace(tag, newtag)
content = __normalize(content)
if rem_noises:
for k, _ in TAG_MAP.items():
content = content.replace(k, '')
elif emojify:
for k, v in TAG_MAP.items():
content = content.replace(k, v)
return t_start, t_end, idx, content
def __process_one_file(
trans_path,
sample_rate,
audio_data,
file_id,
dst_root,
min_slice_duration,
file_count,
keep_low_conf,
rem_noises,
emojify,
):
"""
Creates one block of audio slices and their corresponding transcripts.
Args:
trans_path: filepath to transcript
sample_rate: sample rate of the audio
audio_data: numpy array of shape [samples, channels]
file_id: identifying label, e.g. 'fe_03_01102'
dst_root: path to destination directory
min_slice_duration: min number of seconds for an audio slice
file_count: total number of files processed so far
keep_low_conf: keep utterances with low-confidence transcripts
rem_noises: remove noise symbols
emojify: convert noise symbols into emoji characters
"""
count = 0
with open(trans_path, encoding="utf-8") as fin:
fin.readline() # Comment w/ corresponding sph filename
fin.readline() # Comment about transcriber
transcript_buffers = ['', ''] # [A buffer, B buffer]
audio_buffers = [[], []]
buffer_durations = [0.0, 0.0]
for line in fin:
t_start, t_end, idx, content = __process_utterance(
file_id, trans_path, line, keep_low_conf, rem_noises, emojify
)
if content is None or not content:
continue
duration = t_end - t_start
# Append utterance to buffer
transcript_buffers[idx] += content
audio_buffers[idx].append(
audio_data[floor(t_start * sample_rate) : ceil(t_end * sample_rate), idx,]
)
buffer_durations[idx] += duration
if buffer_durations[idx] < min_slice_duration:
transcript_buffers[idx] += ' '
else:
# Write out segment and transcript
count += 1
__write_sample(
dst_root,
file_id,
count,
file_count,
sample_rate,
np.concatenate(audio_buffers[idx], axis=0),
buffer_durations[idx],
transcript_buffers[idx],
)
# Clear buffers
transcript_buffers[idx] = ''
audio_buffers[idx] = []
buffer_durations[idx] = 0.0
# Note: We drop any shorter "scraps" at the end of the file, if
# they end up shorter than min_slice_duration.
def __partition_name(file_count):
if file_count >= VAL_END_IDX:
return "test"
elif file_count >= TRAIN_END_IDX:
return "val"
else:
return "train"
def __process_data(
audio_root, transcript_root, dst_root, min_slice_duration, file_count, keep_low_conf, rem_noises, emojify,
):
"""
Converts Fisher wav files to numpy arrays, segments audio and transcripts.
Args:
audio_root: source directory with the wav files
transcript_root: source directory with the transcript files
(can be the same as audio_root)
dst_root: where the processed and segmented files will be stored
min_slice_duration: minimum number of seconds for a slice of output
file_count: total number of files processed so far
keep_low_conf: whether or not to keep low confidence transcriptions
rem_noises: whether to remove noise symbols
emojify: whether to convert noise symbols to emoji, lower precedence
Assumes:
1. There is exactly one transcripts directory in data_folder
2. Audio files are all: <audio_root>/audio-wav/fe_03_xxxxx.wav
"""
transcript_list = glob.glob(os.path.join(transcript_root, "fe_03_p*_tran*", "data", "trans", "*", "*.txt"))
print("Found {} transcripts.".format(len(transcript_list)))
count = file_count
# Grab audio file associated with each transcript, and slice
for trans_path in tqdm(transcript_list, desc="Matching and segmenting"):
file_id, _ = os.path.splitext(os.path.basename(trans_path))
audio_path = os.path.join(audio_root, "audio_wav", file_id + ".wav")
sample_rate, audio_data = wavfile.read(audio_path)
# Create a set of segments (a block) for each file
__process_one_file(
trans_path,
sample_rate,
audio_data,
file_id,
dst_root,
min_slice_duration,
count,
keep_low_conf,
rem_noises,
emojify,
)
count += 1
return count
def main():
# Arguments to the script
audio_root = args.audio_root
transcript_root = args.transcript_root
dest_root = args.dest_root
min_slice_duration = args.min_slice_duration
keep_low_conf = args.keep_low_conf
rem_noises = args.remove_noises
emojify = args.noises_to_emoji
print(f"Expected number of files to segment: {NUM_FILES}")
print("With a 80/10/10 split:")
print(f"Number of training files: {TRAIN_END_IDX}")
print(f"Number of validation files: {VAL_END_IDX - TRAIN_END_IDX}")
print(f"Number of test files: {NUM_FILES - VAL_END_IDX}")
if not os.path.exists(os.path.join(dest_root, 'train/')):
os.makedirs(os.path.join(dest_root, 'train/'))
os.makedirs(os.path.join(dest_root, 'val/'))
os.makedirs(os.path.join(dest_root, 'test/'))
else:
# Wipe manifest contents first
open(os.path.join(dest_root, "manifest_train.json"), 'w').close()
open(os.path.join(dest_root, "manifest_val.json"), 'w').close()
open(os.path.join(dest_root, "manifest_test.json"), 'w').close()
file_count = 0
for data_set in ['LDC2004S13-Part1', 'LDC2005S13-Part2']:
print(f"\n\nWorking on dataset: {data_set}")
file_count = __process_data(
os.path.join(audio_root, data_set),
os.path.join(transcript_root, data_set),
dest_root,
min_slice_duration,
file_count,
keep_low_conf,
rem_noises,
emojify,
)
print(f"Total file count so far: {file_count}")
if __name__ == "__main__":
main()