Refactor sort/search/pagination for ClinicalData table endpoint

cBioPortal · Sep 28, 2023 · 0009e1e · 0009e1e
1 parent a62e81e
commit 0009e1e
Show file tree

Hide file tree

Showing 40 changed files with 769 additions and 401 deletions.
diff --git a/db-scripts/src/main/resources/cgds.sql b/db-scripts/src/main/resources/cgds.sql
@@ -517,7 +517,7 @@ CREATE TABLE `clinical_patient` (
   `ATTR_ID` varchar(255) NOT NULL,
   `ATTR_VALUE` varchar(255) NOT NULL,
   PRIMARY KEY (`INTERNAL_ID`, `ATTR_ID`),
-  FOREIGN KEY (`INTERNAL_ID`) REFERENCES `patient` (`INTERNAL_ID`) ON DELETE CASCADE
+  FOREIGN KEY (`INTERNAL_ID`) REFERENCES `patient` (`INTERNAL_ID`) ON DELETE CASCADE,
 );
 
 -- --------------------------------------------------------
@@ -526,7 +526,7 @@ CREATE TABLE `clinical_sample` (
   `ATTR_ID` varchar(255) NOT NULL,
   `ATTR_VALUE` varchar(255) NOT NULL,
   PRIMARY KEY (`INTERNAL_ID`,`ATTR_ID`),
-  FOREIGN KEY (`INTERNAL_ID`) REFERENCES `sample` (`INTERNAL_ID`) ON DELETE CASCADE
+  FOREIGN KEY (`INTERNAL_ID`) REFERENCES `sample` (`INTERNAL_ID`) ON DELETE CASCADE,
 );
 
 -- --------------------------------------------------------
@@ -756,4 +756,3 @@ CREATE TABLE `resource_study` (
 
 -- THIS MUST BE KEPT IN SYNC WITH db.version PROPERTY IN pom.xml
 INSERT INTO info VALUES ('2.13.1', NULL);
-
diff --git a/db-scripts/src/main/resources/migration.sql b/db-scripts/src/main/resources/migration.sql
@@ -1027,4 +1027,4 @@ ALTER TABLE `clinical_event_data` MODIFY COLUMN `VALUE` varchar(3000) NOT NULL;
 CREATE INDEX idx_clinical_event_key ON clinical_event_data (`KEY`);
 CREATE INDEX idx_clinical_event_value ON clinical_event_data (`VALUE`);
 CREATE INDEX idx_sample_stable_id ON sample (`STABLE_ID`);
-UPDATE `info` SET `DB_SCHEMA_VERSION`="2.13.1";
+UPDATE `info` SET `DB_SCHEMA_VERSION`="2.13.1";
diff --git a/model/src/main/java/org/cbioportal/model/ClinicalDataCollection.java b/model/src/main/java/org/cbioportal/model/ClinicalDataCollection.java
diff --git a/model/src/main/java/org/cbioportal/model/SampleClinicalDataCollection.java b/model/src/main/java/org/cbioportal/model/SampleClinicalDataCollection.java
@@ -0,0 +1,19 @@
+package org.cbioportal.model;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class SampleClinicalDataCollection {
+
+    private Map<String, List<ClinicalData>> byUniqueSampleKey = new HashMap<>(); 
+
+    public Map<String, List<ClinicalData>> getByUniqueSampleKey() {
+        return byUniqueSampleKey;
+    }
+
+    public void setByUniqueSampleKey(Map<String, List<ClinicalData>> byUniqueSampleKey) {
+        this.byUniqueSampleKey = byUniqueSampleKey;
+    }
+
+}
diff --git a/...ence/persistence-api/src/main/java/org/cbioportal/persistence/ClinicalDataRepository.java b/...ence/persistence-api/src/main/java/org/cbioportal/persistence/ClinicalDataRepository.java
@@ -47,14 +47,6 @@ BaseMeta fetchMetaClinicalDataInStudy(String studyId, List<String> ids, List<Str
     List<ClinicalData> fetchClinicalData(List<String> studyIds, List<String> ids, List<String> attributeIds,
                                          String clinicalDataType, String projection);
 
-    List<ClinicalData> fetchSampleClinicalTable(List<String> studyIds, List<String> ids,
-                                                Integer pageSize, Integer pageNumber, String searchTerm,
-                                                String sortBy, String direction);
-
-    @Cacheable(cacheResolver = "generalRepositoryCacheResolver", condition = "@cacheEnabledConfig.getEnabled()")
-    Integer fetchSampleClinicalTableCount(List<String> studyIds, List<String> ids, String searchTerm,
-                                          String sortBy, String direction);
-
     @Cacheable(cacheResolver = "generalRepositoryCacheResolver", condition = "@cacheEnabledConfig.getEnabled()")
     BaseMeta fetchMetaClinicalData(List<String> studyIds, List<String> ids, List<String> attributeIds,
                                    String clinicalDataType);
@@ -65,5 +57,13 @@ List<ClinicalDataCount> fetchClinicalDataCounts(List<String> studyIds, List<Stri
 
     @Cacheable(cacheResolver = "generalRepositoryCacheResolver", condition = "@cacheEnabledConfig.getEnabled()")
     List<ClinicalData> getPatientClinicalDataDetailedToSample(List<String> studyIds, List<String> patientIds,
-            List<String> attributeIds);
+                                                              List<String> attributeIds);
+
+    List<Integer> getVisibleSampleInternalIdsForClinicalTable(List<String> studyIds, List<String> sampleIds,
+                                                              Integer pageSize, Integer pageNumber, String searchTerm,
+                                                              String sortBy, String direction);
+
+    List<ClinicalData> getSampleClinicalDataBySampleInternalIds(List<Integer> visibleSampleInternalIds);
+
+    List<ClinicalData> getPatientClinicalDataBySampleInternalIds(List<Integer> visibleSampleInternalIds);
 }
diff --git a/persistence/persistence-mybatis/pom.xml b/persistence/persistence-mybatis/pom.xml
@@ -7,6 +7,18 @@
              PROJECT_VERSION env variable (see version.sh) -->
         <version>0-unknown-version-SNAPSHOT</version>
     </parent>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>9</source>
+                    <target>9</target>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>persistence-mybatis</artifactId>

diff --git a/...mybatis/src/main/java/org/cbioportal/persistence/mybatis/CancerTypeMyBatisRepository.java b/...mybatis/src/main/java/org/cbioportal/persistence/mybatis/CancerTypeMyBatisRepository.java
@@ -4,7 +4,7 @@
 import org.cbioportal.model.meta.BaseMeta;
 import org.cbioportal.persistence.CancerTypeRepository;
 import org.cbioportal.persistence.PersistenceConstants;
-import org.cbioportal.persistence.mybatis.util.OffsetCalculator;
+import org.cbioportal.persistence.mybatis.util.PaginationCalculator;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
@@ -16,14 +16,14 @@ public class CancerTypeMyBatisRepository implements CancerTypeRepository {
     @Autowired
     private CancerTypeMapper cancerTypeMapper;
     @Autowired
-    private OffsetCalculator offsetCalculator;
+    private PaginationCalculator paginationCalculator;
 
     @Override
     public List<TypeOfCancer> getAllCancerTypes(String projection, Integer pageSize, Integer pageNumber, String sortBy,
                                                 String direction) {
 
         return cancerTypeMapper.getAllCancerTypes(projection, pageSize,
-                offsetCalculator.calculate(pageSize, pageNumber), sortBy, direction);
+                paginationCalculator.offset(pageSize, pageNumber), sortBy, direction);
     }
 
     @Override

diff --git a/.../src/main/java/org/cbioportal/persistence/mybatis/ClinicalAttributeMyBatisRepository.java b/.../src/main/java/org/cbioportal/persistence/mybatis/ClinicalAttributeMyBatisRepository.java
@@ -5,7 +5,7 @@
 import org.cbioportal.model.meta.BaseMeta;
 import org.cbioportal.persistence.ClinicalAttributeRepository;
 import org.cbioportal.persistence.PersistenceConstants;
-import org.cbioportal.persistence.mybatis.util.OffsetCalculator;
+import org.cbioportal.persistence.mybatis.util.PaginationCalculator;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
@@ -18,14 +18,14 @@ public class ClinicalAttributeMyBatisRepository implements ClinicalAttributeRepo
     @Autowired
     private ClinicalAttributeMapper clinicalAttributeMapper;
     @Autowired
-    private OffsetCalculator offsetCalculator;
+    private PaginationCalculator paginationCalculator;
 
     @Override
     public List<ClinicalAttribute> getAllClinicalAttributes(String projection, Integer pageSize, Integer pageNumber,
                                                             String sortBy, String direction) {
 
         return clinicalAttributeMapper.getClinicalAttributes(null, projection, pageSize,
-                offsetCalculator.calculate(pageSize, pageNumber), sortBy, direction);
+                paginationCalculator.offset(pageSize, pageNumber), sortBy, direction);
     }
 
     @Override
@@ -47,7 +47,7 @@ public List<ClinicalAttribute> getAllClinicalAttributesInStudy(String studyId, S
                                                                    String direction) {
 
         return clinicalAttributeMapper.getClinicalAttributes(Arrays.asList(studyId), projection, pageSize,
-                offsetCalculator.calculate(pageSize, pageNumber), sortBy, direction);
+                paginationCalculator.offset(pageSize, pageNumber), sortBy, direction);
     }
 
     @Override

diff --git a/...sistence-mybatis/src/main/java/org/cbioportal/persistence/mybatis/ClinicalDataMapper.java b/...sistence-mybatis/src/main/java/org/cbioportal/persistence/mybatis/ClinicalDataMapper.java
@@ -20,7 +20,8 @@ List<ClinicalData> getPatientClinicalData(List<String> studyIds, List<String> pa
 
     List<ClinicalData> getSampleClinicalTable(List<String> studyIds, List<String> sampleIds, String projection,
                                               Integer limit, Integer offset, String searchTerm,
-                                              String sortBy, String direction);
+                                              String sortByAttrId, Boolean sortAttrIsNumber, Boolean sortIsPatientAttr,
+                                              String direction);
 
     Integer getSampleClinicalTableCount(List<String> studyIds, List<String> sampleIds, String projection,
                                         String searchTerm, String sortBy, String direction);
@@ -36,4 +37,19 @@ List<ClinicalDataCount> fetchPatientClinicalDataCounts(List<String> studyIds, Li
     List<ClinicalData> getPatientClinicalDataDetailedToSample(List<String> studyIds, List<String> patientIds,
                                                               List<String> attributeIds, String projection, Integer limit,
                                                               Integer offset, String sortBy, String direction);
+
+    List<Integer> getVisibleSampleInternalIdsForClinicalTable(List<String> studyIds,
+                                                              List<String> sampleIds,
+                                                              String projection,
+                                                              Integer limit,
+                                                              Integer offset,
+                                                              String searchTerm,
+                                                              String sortAttrId,
+                                                              Boolean sortAttrIsNumber,
+                                                              Boolean sortIsPatientAttr,
+                                                              String direction);
+
+    List<ClinicalData> getSampleClinicalDataBySampleInternalIds(List<Integer> sampleInternalIds);
+
+    List<ClinicalData> getPatientClinicalDataBySampleInternalIds(List<Integer> sampleInternalIds);
 }
diff --git a/...batis/src/main/java/org/cbioportal/persistence/mybatis/ClinicalDataMyBatisRepository.java b/...batis/src/main/java/org/cbioportal/persistence/mybatis/ClinicalDataMyBatisRepository.java
@@ -1,20 +1,22 @@
 package org.cbioportal.persistence.mybatis;
 
+import org.cbioportal.model.ClinicalAttribute;
 import org.cbioportal.model.ClinicalData;
 import org.cbioportal.model.ClinicalDataCount;
 import org.cbioportal.model.Patient;
 import org.cbioportal.model.meta.BaseMeta;
 import org.cbioportal.persistence.ClinicalDataRepository;
+import org.cbioportal.persistence.ClinicalAttributeRepository;
 import org.cbioportal.persistence.PatientRepository;
 import org.cbioportal.persistence.PersistenceConstants;
-import org.cbioportal.persistence.mybatis.util.OffsetCalculator;
+import org.cbioportal.persistence.mybatis.util.PaginationCalculator;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
+import org.springframework.util.Assert;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
+import java.util.*;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 @Repository
 public class ClinicalDataMyBatisRepository implements ClinicalDataRepository {
@@ -24,7 +26,9 @@ public class ClinicalDataMyBatisRepository implements ClinicalDataRepository {
     @Autowired
     private PatientRepository patientRepository;
     @Autowired
-    private OffsetCalculator offsetCalculator;
+    private PaginationCalculator paginationCalculator;
+    @Autowired
+    private ClinicalAttributeRepository clinicalAttributeRepository;
 
     @Override
     public List<ClinicalData> getAllClinicalDataOfSampleInStudy(String studyId, String sampleId,
@@ -34,7 +38,7 @@ public List<ClinicalData> getAllClinicalDataOfSampleInStudy(String studyId, Stri
 
         return clinicalDataMapper.getSampleClinicalData(Arrays.asList(studyId), Arrays.asList(sampleId),
             attributeId != null ? Arrays.asList(attributeId) : null, projection, pageSize, 
-            offsetCalculator.calculate(pageSize, pageNumber), sortBy, direction);
+            paginationCalculator.offset(pageSize, pageNumber), sortBy, direction);
     }
 
     @Override
@@ -51,7 +55,7 @@ public List<ClinicalData> getAllClinicalDataOfPatientInStudy(String studyId, Str
 
         return clinicalDataMapper.getPatientClinicalData(Arrays.asList(studyId), Arrays.asList(patientId),
             attributeId != null ? Arrays.asList(attributeId) : null, projection, pageSize, 
-            offsetCalculator.calculate(pageSize, pageNumber), sortBy, direction);
+            paginationCalculator.offset(pageSize, pageNumber), sortBy, direction);
     }
 
     @Override
@@ -69,11 +73,11 @@ public List<ClinicalData> getAllClinicalDataInStudy(String studyId, String attri
         if (clinicalDataType.equals(PersistenceConstants.SAMPLE_CLINICAL_DATA_TYPE)) {
             return clinicalDataMapper.getSampleClinicalData(Arrays.asList(studyId), null, 
                 attributeId != null ? Arrays.asList(attributeId) : null, projection, pageSize, 
-                offsetCalculator.calculate(pageSize, pageNumber), sortBy, direction);
+                paginationCalculator.offset(pageSize, pageNumber), sortBy, direction);
         } else {
             return clinicalDataMapper.getPatientClinicalData(Arrays.asList(studyId), null, 
                 attributeId != null ? Arrays.asList(attributeId) : null, projection, pageSize, 
-                offsetCalculator.calculate(pageSize, pageNumber), sortBy, direction);
+                paginationCalculator.offset(pageSize, pageNumber), sortBy, direction);
         }
     }
 
@@ -137,23 +141,28 @@ public List<ClinicalData> fetchClinicalData(List<String> studyIds, List<String>
         }
     }
 
-    @Override
-    public List<ClinicalData> fetchSampleClinicalTable(List<String> studyIds, List<String> ids,
-                                                       Integer pageSize, Integer pageNumber, String searchTerm,
-                                                       String sortBy, String direction) {
-        if (ids.isEmpty()) {
+    public List<Integer> getVisibleSampleInternalIdsForClinicalTable(List<String> studyIds, List<String> sampleIds,
+                                                                       Integer pageSize, Integer pageNumber, String searchTerm,
+                                                                       String sortBy, String direction) {
+        if (sampleIds.isEmpty()) {
             return new ArrayList<>();
         }
-        int offset = offsetCalculator.calculate(pageSize, pageNumber);
-        return clinicalDataMapper.getSampleClinicalTable(studyIds, ids,"SUMMARY", pageSize,
-            offset, searchTerm, sortBy, direction);
-    }
-
-    @Override
-    public Integer fetchSampleClinicalTableCount(List<String> studyIds, List<String> sampleIds,
-                                                 String searchTerm, String sortBy, String direction) {
-        return clinicalDataMapper.getSampleClinicalTableCount(studyIds, sampleIds,"SUMMARY", 
-            searchTerm, sortBy, direction);
+        Integer offset = paginationCalculator.offset(pageSize, pageNumber);
+        String sortAttrId = sortBy;
+        Boolean sortAttrIsNumber = false;
+        Boolean sortIsPatientAttr = false;
+        if (sortBy != null && ! sortBy.isEmpty()) {
+            Stream<String> uniqueStudyIds = studyIds.stream().distinct();
+            Optional<ClinicalAttribute> clinicalAttributeMeta = uniqueStudyIds
+                .map(studyId -> clinicalAttributeRepository.getClinicalAttribute(studyId, sortBy))
+                .filter(Objects::nonNull)
+                .findFirst();
+            Assert.isTrue(clinicalAttributeMeta.isPresent(), "Attribute was not found");
+            sortAttrIsNumber = clinicalAttributeMeta.get().getDatatype().equals("NUMBER");
+            sortIsPatientAttr = clinicalAttributeMeta.get().getPatientAttribute();
+        }
+        return clinicalDataMapper.getVisibleSampleInternalIdsForClinicalTable(studyIds, sampleIds,"SUMMARY", pageSize, 
+            offset, searchTerm, sortAttrId, sortAttrIsNumber, sortIsPatientAttr, direction);
     }
 
     @Override
@@ -195,4 +204,20 @@ public List<ClinicalData> getPatientClinicalDataDetailedToSample(List<String> st
         return clinicalDataMapper.getPatientClinicalDataDetailedToSample(studyIds, patientIds, attributeIds, "SUMMARY",
                 0, 0, null, null);
     }
+
+    @Override
+    public List<ClinicalData> getSampleClinicalDataBySampleInternalIds(List<Integer> sampleInternalIds) {
+        if (sampleInternalIds == null || sampleInternalIds.isEmpty()) {
+            return new ArrayList<>();
+        }
+        return clinicalDataMapper.getSampleClinicalDataBySampleInternalIds(sampleInternalIds);
+    }
+
+    @Override
+    public List<ClinicalData> getPatientClinicalDataBySampleInternalIds(List<Integer> sampleInternalIds) {
+        if (sampleInternalIds == null || sampleInternalIds.isEmpty()) {
+            return new ArrayList<>();
+        }
+        return clinicalDataMapper.getPatientClinicalDataBySampleInternalIds(sampleInternalIds);
+    }
 }
diff --git a/...atis/src/main/java/org/cbioportal/persistence/mybatis/ClinicalEventMyBatisRepository.java b/...atis/src/main/java/org/cbioportal/persistence/mybatis/ClinicalEventMyBatisRepository.java
@@ -4,7 +4,7 @@
 import org.cbioportal.model.ClinicalEventData;
 import org.cbioportal.model.meta.BaseMeta;
 import org.cbioportal.persistence.ClinicalEventRepository;
-import org.cbioportal.persistence.mybatis.util.OffsetCalculator;
+import org.cbioportal.persistence.mybatis.util.PaginationCalculator;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Repository;
 
@@ -21,14 +21,14 @@ public class ClinicalEventMyBatisRepository implements ClinicalEventRepository {
     @Autowired
     private ClinicalEventMapper clinicalEventMapper;
     @Autowired
-    private OffsetCalculator offsetCalculator;
+    private PaginationCalculator paginationCalculator;
 
     @Override
     public List<ClinicalEvent> getAllClinicalEventsOfPatientInStudy(String studyId, String patientId, String projection, 
                                                                     Integer pageSize, Integer pageNumber, String sortBy, 
                                                                     String direction) {
         return clinicalEventMapper.getPatientClinicalEvent(studyId, patientId, projection, pageSize, 
-            offsetCalculator.calculate(pageSize, pageNumber), sortBy, direction);
+            paginationCalculator.offset(pageSize, pageNumber), sortBy, direction);
     }
 
     @Override
@@ -47,7 +47,7 @@ public List<ClinicalEventData> getDataOfClinicalEvents(List<Integer> clinicalEve
     public List<ClinicalEvent> getAllClinicalEventsInStudy(String studyId, String projection, Integer pageSize,
                                                            Integer pageNumber, String sortBy, String direction) {
         return clinicalEventMapper.getStudyClinicalEvent(studyId, projection, pageSize, 
-            offsetCalculator.calculate(pageSize, pageNumber), sortBy, direction);
+            paginationCalculator.offset(pageSize, pageNumber), sortBy, direction);
     }
 
     @Override