Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix vague language codes caused wrong recognition result #136

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Fix vague language codes caused wrong recognition result #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2432a8d
Add cloud speech-to-text and translation language code to constants.py
BingLingGroup Feb 14, 2019
1290ce6
Add cloud speech-to-text and translation language codes to __init__.py
BingLingGroup Feb 14, 2019
41303bb
Fix pylint code format issues
BingLingGroup Feb 14, 2019
7714456
Reverse ffmpeg dependency check issue
BingLingGroup Feb 14, 2019
c3643a2
Add help message to readme.md
BingLingGroup Feb 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 6 additions & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,9 @@ dist/
.DS_Store
MANIFEST
*#*
.vscode
.vscode
/.vs/autosub/v15/.suo
/.vs
/autosub.sln
/autosub.pyproj
/.idea
25 changes: 19 additions & 6 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,7 @@ Autosub is a utility for automatic speech recognition and subtitle generation. I
```
$ autosub -h
usage: autosub [-h] [-C CONCURRENCY] [-o OUTPUT] [-F FORMAT] [-S SRC_LANGUAGE]
[-D DST_LANGUAGE] [-K API_KEY] [--list-formats]
[--list-languages]
[-D DST_LANGUAGE] [-K API_KEY] [-lf] [-lsc] [-ltc]
[source_path]

positional arguments:
Expand All @@ -36,10 +35,24 @@ optional arguments:
-D DST_LANGUAGE, --dst-language DST_LANGUAGE
Desired language for the subtitles
-K API_KEY, --api-key API_KEY
The Google Translate API key to be used. (Required for
subtitle translation)
--list-formats List all available subtitle formats
--list-languages List all available source/destination languages
The Google Translation API key to be used. (Required
for subtitle translation)
-lf, --list-formats List all available subtitle formats
-lsc, --list-speech-to-text-codes
List all available source language codes, which mean
the speech-to-text available language codes.
[WARNING]: Its name format is different from the
destination language codes. And it's Google who make
that difference not the developers of the autosub.
Reference: https://cloud.google.com/speech-to-
text/docs/languages
-ltc, --list-translation-codes
List all available destination language codes, which
mean the translation language codes. [WARNING]: Its
name format is different from the source language
codes. And it's Google who make that difference not
the developers of the autosub. Reference:
https://cloud.google.com/translate/docs/languages
```

### License
Expand Down
67 changes: 53 additions & 14 deletions autosub/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@
from progressbar import ProgressBar, Percentage, Bar, ETA

from autosub.constants import (
LANGUAGE_CODES, GOOGLE_SPEECH_API_KEY, GOOGLE_SPEECH_API_URL,
SPEECH_TO_TEXT_LANGUAGE_CODES, TRANSLATION_LANGUAGE_CODES,
GOOGLE_SPEECH_API_KEY, GOOGLE_SPEECH_API_URL,
)
from autosub.formatters import FORMATTERS

Expand Down Expand Up @@ -116,7 +117,7 @@ def __call__(self, data):
return None


class Translator(object): # pylint: disable=too-few-public-methods
class Translator(object): # pylint: disable=too-few-public-methods
"""
Class for translating a sentence from a one language to another.
"""
Expand Down Expand Up @@ -329,17 +330,32 @@ def validate(args):
)
return False

if args.src_language not in LANGUAGE_CODES.keys():
if args.src_language not in SPEECH_TO_TEXT_LANGUAGE_CODES.keys():
print(
"Source language not supported. "
"Run with --list-languages to see all supported languages."
"Run with -lsc or --list-speech-to-text-codes "
"to see all supported languages."
)
return False

if args.dst_language not in LANGUAGE_CODES.keys():
if args.dst_language is None:
print(
"Destination language not provided. "
"Only performing speech recognition."
)
args.dst_language = args.src_language

elif args.dst_language == args.src_language:
print(
"Source language is the same as the Destination language. "
"Only performing speech recognition."
)

elif args.dst_language not in TRANSLATION_LANGUAGE_CODES.keys():
print(
"Destination language not supported. "
"Run with --list-languages to see all supported languages."
"Run with -ltc or --list-translation-codes "
"to see all supported languages."
)
return False

Expand All @@ -366,14 +382,31 @@ def main():
default=DEFAULT_SUBTITLE_FORMAT)
parser.add_argument('-S', '--src-language', help="Language spoken in source file",
default=DEFAULT_SRC_LANGUAGE)
parser.add_argument('-D', '--dst-language', help="Desired language for the subtitles",
default=DEFAULT_DST_LANGUAGE)
parser.add_argument('-D', '--dst-language', help="Desired language for the subtitles")
parser.add_argument('-K', '--api-key',
help="The Google Translate API key to be used. \
help="The Google Translation API key to be used. \
(Required for subtitle translation)")
parser.add_argument('--list-formats', help="List all available subtitle formats",
parser.add_argument('-lf', '--list-formats', help="List all available subtitle formats",
action='store_true')
parser.add_argument('-lsc', '--list-speech-to-text-codes',
help="""List all available source language codes,
which mean the speech-to-text
available language codes.
[WARNING]: Its name format is different from
the destination language codes.
And it's Google who make that difference
not the developers of the autosub.
Reference: https://cloud.google.com/speech-to-text/docs/languages""",
action='store_true')
parser.add_argument('--list-languages', help="List all available source/destination languages",
parser.add_argument('-ltc', '--list-translation-codes',
help="""List all available destination language codes,
which mean the translation
language codes.
[WARNING]: Its name format is different from
the source language codes.
And it's Google who make that difference
not the developers of the autosub.
Reference: https://cloud.google.com/translate/docs/languages""",
action='store_true')

args = parser.parse_args()
Expand All @@ -384,9 +417,15 @@ def main():
print("{format}".format(format=subtitle_format))
return 0

if args.list_languages:
print("List of all languages:")
for code, language in sorted(LANGUAGE_CODES.items()):
if args.list_speech_to_text_codes:
print("List of all source language codes:")
for code, language in sorted(SPEECH_TO_TEXT_LANGUAGE_CODES.items()):
print("{code}\t{language}".format(code=code, language=language))
return 0

if args.list_translation_codes:
print("List of all destination language codes:")
for code, language in sorted(TRANSLATION_LANGUAGE_CODES.items()):
print("{code}\t{language}".format(code=code, language=language))
return 0

Expand Down
154 changes: 146 additions & 8 deletions autosub/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,132 @@
GOOGLE_SPEECH_API_KEY = "AIzaSyBOti4mM-6x9WDnZIjIeyEU21OpBXqWBgw"
GOOGLE_SPEECH_API_URL = "http://www.google.com/speech-api/v2/recognize?client=chromium&lang={lang}&key={key}" # pylint: disable=line-too-long

LANGUAGE_CODES = {
SPEECH_TO_TEXT_LANGUAGE_CODES = {
'af-ZA': 'Afrikaans (South Africa)',
'am-ET': 'Amharic (Ethiopia)',
'ar-AE': 'Arabic (United Arab Emirates)',
'ar-BH': 'Arabic (Bahrain)',
'ar-DZ': 'Arabic (Algeria)',
'ar-EG': 'Arabic (Egypt)',
'ar-IL': 'Arabic (Israel)',
'ar-IQ': 'Arabic (Iraq)',
'ar-JO': 'Arabic (Jordan)',
'ar-KW': 'Arabic (Kuwait)',
'ar-LB': 'Arabic (Lebanon)',
'ar-MA': 'Arabic (Morocco)',
'ar-OM': 'Arabic (Oman)',
'ar-PS': 'Arabic (State of Palestine)',
'ar-QA': 'Arabic (Qatar)',
'ar-SA': 'Arabic (Saudi Arabia)',
'ar-TN': 'Arabic (Tunisia)',
'az-AZ': 'Azerbaijani (Azerbaijan)',
'bg-BG': 'Bulgarian (Bulgaria)',
'bn-BD': 'Bengali (Bangladesh)',
'bn-IN': 'Bengali (India)',
'ca-ES': 'Catalan (Spain)',
'cmn-Hans-CN': 'Chinese, Mandarin (Simplified, China)',
'cmn-Hans-HK': 'Chinese, Mandarin (Simplified, Hong Kong)',
'cmn-Hant-TW': 'Chinese, Mandarin (Traditional, Taiwan)',
'cs-CZ': 'Czech (Czech Republic)',
'da-DK': 'Danish (Denmark)',
'de-DE': 'German (Germany)',
'el-GR': 'Greek (Greece)',
'en-AU': 'English (Australia)',
'en-CA': 'English (Canada)',
'en-GB': 'English (United Kingdom)',
'en-GH': 'English (Ghana)',
'en-IE': 'English (Ireland)',
'en-IN': 'English (India)',
'en-KE': 'English (Kenya)',
'en-NG': 'English (Nigeria)',
'en-NZ': 'English (New Zealand)',
'en-PH': 'English (Philippines)',
'en-SG': 'English (Singapore)',
'en-TZ': 'English (Tanzania)',
'en-US': 'English (United States)',
'en-ZA': 'English (South Africa)',
'es-AR': 'Spanish (Argentina)',
'es-BO': 'Spanish (Bolivia)',
'es-CL': 'Spanish (Chile)',
'es-CO': 'Spanish (Colombia)',
'es-CR': 'Spanish (Costa Rica)',
'es-DO': 'Spanish (Dominican Republic)',
'es-EC': 'Spanish (Ecuador)',
'es-ES': 'Spanish (Spain)',
'es-GT': 'Spanish (Guatemala)',
'es-HN': 'Spanish (Honduras)',
'es-MX': 'Spanish (Mexico)',
'es-NI': 'Spanish (Nicaragua)',
'es-PA': 'Spanish (Panama)',
'es-PE': 'Spanish (Peru)',
'es-PR': 'Spanish (Puerto Rico)',
'es-PY': 'Spanish (Paraguay)',
'es-SV': 'Spanish (El Salvador)',
'es-US': 'Spanish (United States)',
'es-UY': 'Spanish (Uruguay)',
'es-VE': 'Spanish (Venezuela)',
'eu-ES': 'Basque (Spain)',
'fa-IR': 'Persian (Iran)',
'fi-FI': 'Finnish (Finland)',
'fil-PH ': 'Filipino (Philippines)',
'fr-CA': 'French (Canada)',
'fr-FR': 'French (France)',
'gl-ES': 'Galician (Spain)',
'gu-IN': 'Gujarati (India)',
'he-IL': 'Hebrew (Israel)',
'hi-IN': 'Hindi (India)',
'hr-HR': 'Croatian (Croatia)',
'hu-HU': 'Hungarian (Hungary)',
'hy-AM': 'Armenian (Armenia)',
'id-ID': 'Indonesian (Indonesia)',
'is-IS': 'Icelandic (Iceland)',
'it-IT': 'Italian (Italy)',
'ja-JP': 'Japanese (Japan)',
'jv-ID': 'Javanese (Indonesia)',
'ka-GE': 'Georgian (Georgia)',
'km-KH': 'Khmer (Cambodia)',
'kn-IN': 'Kannada (India)',
'ko-KR': 'Korean (South Korea)',
'lo-LA': 'Lao (Laos)',
'lt-LT': 'Lithuanian (Lithuania)',
'lv-LV': 'Latvian (Latvia)',
'ml-IN': 'Malayalam (India)',
'mr-IN': 'Marathi (India)',
'ms-MY': 'Malay (Malaysia)',
'nb-NO': 'Norwegian Bokmal (Norway)',
'ne-NP': 'Nepali (Nepal)',
'nl-NL': 'Dutch (Netherlands)',
'pl-PL': 'Polish (Poland)',
'pt-BR': 'Portuguese (Brazil)',
'pt-PT': 'Portuguese (Portugal)',
'ro-RO': 'Romanian (Romania)',
'ru-RU': 'Russian (Russia)',
'si-LK': 'Sinhala (Sri Lanka)',
'sk-SK': 'Slovak (Slovakia)',
'sl-SI': 'Slovenian (Slovenia)',
'sr-RS': 'Serbian (Serbia)',
'su-ID': 'Sundanese (Indonesia)',
'sv-SE': 'Swedish (Sweden)',
'sw-KE': 'Swahili (Kenya)',
'sw-TZ': 'Swahili (Tanzania)',
'ta-IN': 'Tamil (India)',
'ta-LK': 'Tamil (Sri Lanka)',
'ta-MY': 'Tamil (Malaysia)',
'ta-SG': 'Tamil (Singapore)',
'te-IN': 'Telugu (India)',
'th-TH': 'Thai (Thailand)',
'tr-TR': 'Turkish (Turkey)',
'uk-UA': 'Ukrainian (Ukraine)',
'ur-IN': 'Urdu (India)',
'ur-PK': 'Urdu (Pakistan)',
'vi-VN': 'Vietnamese (Vietnam)',
'yue-Hant-HK': 'Chinese, Cantonese (Traditional, Hong Kong)',
'zu-ZA': 'Zulu (South Africa)'
}

TRANSLATION_LANGUAGE_CODES = {
'af': 'Afrikaans',
'am': 'Amharic',
'ar': 'Arabic',
'az': 'Azerbaijani',
'be': 'Belarusian',
Expand All @@ -17,6 +141,7 @@
'bs': 'Bosnian',
'ca': 'Catalan',
'ceb': 'Cebuano',
'co': 'Corsican',
'cs': 'Czech',
'cy': 'Welsh',
'da': 'Danish',
Expand All @@ -30,10 +155,14 @@
'fa': 'Persian',
'fi': 'Finnish',
'fr': 'French',
'fy': 'Frisian',
'ga': 'Irish',
'gd': 'Scots Gaelic',
'gl': 'Galician',
'gu': 'Gujarati',
'ha': 'Hausa',
'haw': 'Hawaiian',
'he': 'Hebrew',
'hi': 'Hindi',
'hmn': 'Hmong',
'hr': 'Croatian',
Expand All @@ -52,7 +181,10 @@
'km': 'Khmer',
'kn': 'Kannada',
'ko': 'Korean',
'ku': 'Kurdish',
'ky': 'Kyrgyz',
'la': 'Latin',
'lb': 'Luxembourgish',
'lo': 'Lao',
'lt': 'Lithuanian',
'lv': 'Latvian',
Expand All @@ -64,39 +196,45 @@
'mr': 'Marathi',
'ms': 'Malay',
'mt': 'Maltese',
'my': 'Myanmar (Burmese)',
'my': 'Myanmar(Burmese)',
'ne': 'Nepali',
'nl': 'Dutch',
'no': 'Norwegian',
'ny': 'Chichewa',
'ny': 'Nyanja(Chichewa)',
'pa': 'Punjabi',
'pl': 'Polish',
'pt': 'Portuguese',
'ps': 'Pashto',
'pt': 'Portuguese(Portugal,Brazil)',
'ro': 'Romanian',
'ru': 'Russian',
'si': 'Sinhala',
'sd': 'Sindhi',
'si': 'Sinhala(Sinhalese)',
'sk': 'Slovak',
'sl': 'Slovenian',
'sm': 'Samoan',
'sn': 'Shona',
'so': 'Somali',
'sq': 'Albanian',
'sr': 'Serbian',
'st': 'Sesotho',
'su': 'Sudanese',
'su': 'Sundanese',
'sv': 'Swedish',
'sw': 'Swahili',
'ta': 'Tamil',
'te': 'Telugu',
'tg': 'Tajik',
'th': 'Thai',
'tl': 'Filipino',
'tl': 'Tagalog(Filipino)',
'tr': 'Turkish',
'uk': 'Ukrainian',
'ur': 'Urdu',
'uz': 'Uzbek',
'vi': 'Vietnamese',
'xh': 'Xhosa',
'yi': 'Yiddish',
'yo': 'Yoruba',
'zh': 'Chinese (Simplified)',
'zh-CN': 'Chinese (Simplified)',
'zh-TW': 'Chinese (Traditional)',
'zu': 'Zulu',
'zu': 'Zulu'
}