fix: 修复上传文档数据集时出现重复问题 (#465)

1Panel-dev · May 15, 2024 · e74ba87 · e74ba87
1 parent cad10bb
commit e74ba87
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 1 deletion.
diff --git a/apps/dataset/serializers/dataset_serializers.py b/apps/dataset/serializers/dataset_serializers.py
@@ -316,7 +316,8 @@ def save(self, instance: Dict, with_valid=True):
                     problem_model_list.append(problem)
                 for problem_paragraph_mapping in document_paragraph_dict_model.get('problem_paragraph_mapping_list'):
                     problem_paragraph_mapping_list.append(problem_paragraph_mapping)
-
+            problem_model_list, problem_paragraph_mapping_list = DocumentSerializers.Create.reset_problem_model(
+                problem_model_list, problem_paragraph_mapping_list)
             # 插入知识库
             dataset.save()
             # 插入文档

diff --git a/apps/dataset/serializers/document_serializers.py b/apps/dataset/serializers/document_serializers.py
@@ -549,10 +549,26 @@ def get_paragraph_model(document_model, paragraph_list: List):
                     problem_paragraph_mapping_list.append(problem_paragraph_mapping)
                 paragraph_model_list.append(paragraph)
 
+            problem_model_list, problem_paragraph_mapping_list = DocumentSerializers.Create.reset_problem_model(
+                problem_model_list, problem_paragraph_mapping_list)
+
             return {'document': document_model, 'paragraph_model_list': paragraph_model_list,
                     'problem_model_list': problem_model_list,
                     'problem_paragraph_mapping_list': problem_paragraph_mapping_list}
 
+        @staticmethod
+        def reset_problem_model(problem_model_list, problem_paragraph_mapping_list):
+            new_problem_model_list = [x for i, x in enumerate(problem_model_list) if
+                                      len([item for item in problem_model_list[:i] if item.content == x.content]) <= 0]
+
+            for new_problem_model in new_problem_model_list:
+                old_model_list = [problem.id for problem in problem_model_list if
+                                  problem.content == new_problem_model.content]
+                for problem_paragraph_mapping in problem_paragraph_mapping_list:
+                    if old_model_list.__contains__(problem_paragraph_mapping.problem_id):
+                        problem_paragraph_mapping.problem_id = new_problem_model.id
+            return new_problem_model_list, problem_paragraph_mapping_list
+
         @staticmethod
         def get_document_paragraph_model(dataset_id, instance: Dict):
             document_model = Document(