From a3a664b7f1714605b8d59fce2992bbe1ca620515 Mon Sep 17 00:00:00 2001 From: terrtia Date: Wed, 27 Mar 2024 11:15:22 +0100 Subject: [PATCH] fix: [languages] fix items language min probability --- bin/lib/Language.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/bin/lib/Language.py b/bin/lib/Language.py index 03293bed..4a840f02 100755 --- a/bin/lib/Language.py +++ b/bin/lib/Language.py @@ -434,7 +434,7 @@ def delete_obj_translation(obj_global_id, language, field=''): class LanguagesDetector: - def __init__(self, nb_langs=3, min_proportion=0.2, min_probability=0.7, min_len=0): + def __init__(self, nb_langs=3, min_proportion=0.2, min_probability=-1, min_len=0): self.lt = LibreTranslateAPI(get_translator_instance()) try: self.lt.languages() @@ -461,9 +461,11 @@ def detect_gcld3(self, content): languages.append(lang.language) return languages - def detect_lexilang(self, content): # TODO clean text ??? - TODO REMOVE SEPARATOR + def detect_lexilang(self, content): language, prob = lexilang_detect(content) - if prob > 0: + if prob > 0 and self.min_probability == -1: + return [language] + elif prob > 0.4: return [language] else: return [] @@ -506,6 +508,8 @@ def detect(self, content, force_gcld3=False): # TODO detect length between 20-2 # libretranslate # else: # languages = self.detect_libretranslate(content) + if not languages: + languages = [] return languages class LanguageTranslator: