forked from mozilla/DeepSpeech
/
tc-benchmark-tests.sh
executable file
·105 lines (82 loc) · 3.31 KB
/
tc-benchmark-tests.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
#!/bin/bash
set -xe
source $(dirname "$0")/tc-tests-utils.sh
exec_benchmark()
{
model_file="$1"
run_postfix=$2
mkdir -p /tmp/bench-ds/ || true
mkdir -p /tmp/bench-ds-nolm/ || true
csv=${TASKCLUSTER_ARTIFACTS}/benchmark-${run_postfix}.csv
csv_nolm=${TASKCLUSTER_ARTIFACTS}/benchmark-nolm-${run_postfix}.csv
png=${TASKCLUSTER_ARTIFACTS}/benchmark-${run_postfix}.png
svg=${TASKCLUSTER_ARTIFACTS}/benchmark-${run_postfix}.svg
python ${DS_ROOT_TASK}/DeepSpeech/ds/bin/benchmark_nc.py \
--dir /tmp/bench-ds/ \
--models ${model_file} \
--wav /tmp/LDC93S1.wav \
--alphabet /tmp/alphabet.txt \
--lm_binary /tmp/lm.binary \
--trie /tmp/trie \
--csv ${csv}
python ${DS_ROOT_TASK}/DeepSpeech/ds/bin/benchmark_nc.py \
--dir /tmp/bench-ds-nolm/ \
--models ${model_file} \
--wav /tmp/LDC93S1.wav \
--alphabet /tmp/alphabet.txt \
--csv ${csv_nolm}
python ${DS_ROOT_TASK}/DeepSpeech/ds/bin/benchmark_plotter.py \
--dataset "TaskCluster model" ${csv} \
--dataset "TaskCluster model (no LM)" ${csv_nolm} \
--title "TaskCluster model benchmark" \
--wav /tmp/LDC93S1.wav \
--plot ${png} \
--size 1280x720
python ${DS_ROOT_TASK}/DeepSpeech/ds/bin/benchmark_plotter.py \
--dataset "TaskCluster model" ${csv} \
--dataset "TaskCluster model (no LM)" ${csv_nolm} \
--title "TaskCluster model benchmark" \
--wav /tmp/LDC93S1.wav \
--plot ${svg} \
--size 1280x720
}
pyver=3.5.6
unset PYTHON_BIN_PATH
unset PYTHONPATH
export PYENV_ROOT="${HOME}/ds-test/.pyenv"
export PATH="${PYENV_ROOT}/bin:$PATH"
mkdir -p ${TASKCLUSTER_ARTIFACTS} || true
mkdir -p ${PYENV_ROOT} || true
# We still need to get model, wav and alphabet
download_data
# Follow benchmark naming from parameters in bin/run-tc-ldc93s1.sh
# Okay, it's not really the real LSTM sizes, just a way to verify how things
# actually behave.
for size in 100 200 300 400 500 600 700 800 900;
do
cp /tmp/${model_name} /tmp/test.frozen.e75.lstm${size}.ldc93s1.pb
done;
# Let's make it a ZIP file. We don't want the directory structure.
zip --junk-paths -r9 /tmp/test.frozen.e75.lstm100-900.ldc93s1.zip /tmp/test.frozen.e75.lstm*.ldc93s1.pb && rm /tmp/test.frozen.e75.lstm*.ldc93s1.pb
# And prepare for multiple files on the CLI
model_list=""
for size in 10 20 30 40 50 60 70 80 90;
do
cp /tmp/${model_name} /tmp/test.frozen.e75.lstm${size}.ldc93s1.pb
model_list="${model_list} /tmp/test.frozen.e75.lstm${size}.ldc93s1.pb"
done;
# Let's prepare another model for single-model codepath
mv /tmp/${model_name} /tmp/test.frozen.e75.lstm494.ldc93s1.pb
export TASKCLUSTER_SCHEME=${DEEPSPEECH_ARTIFACTS_ROOT}/native_client.tar.xz
install_pyenv "${PYENV_ROOT}"
install_pyenv_virtualenv "$(pyenv root)/plugins/pyenv-virtualenv"
PYENV_NAME=deepspeech-test
pyenv install ${pyver}
pyenv virtualenv ${pyver} ${PYENV_NAME}
source ${PYENV_ROOT}/versions/${pyver}/envs/${PYENV_NAME}/bin/activate
pip install -r ${DS_ROOT_TASK}/DeepSpeech/ds/requirements.txt | cat
exec_benchmark "/tmp/test.frozen.e75.lstm494.ldc93s1.pb" "single-model"
exec_benchmark "/tmp/test.frozen.e75.lstm100-900.ldc93s1.zip" "zipfile-model"
exec_benchmark "${model_list}" "multi-model"
deactivate
pyenv uninstall --force ${PYENV_NAME}