Merge pull request #76 from zktuong/devel

Ability to handle TCR data

.github/workflows/tests.yml

@@ -25,9 +25,9 @@ jobs:
           - { python-version: 3.7, os: ubuntu-latest }
           - { python-version: 3.8, os: ubuntu-latest }
           - { python-version: 3.9, os: ubuntu-latest }
-          # 3.8 fails on macosx likely because of issue noted here:
-          # https://github.com/scipy/scipy/issues/11835
           - { python-version: 3.7, os: macos-latest }
+          - { python-version: 3.8, os: macos-latest }
+          - { python-version: 3.9, os: macos-latest }
     runs-on: ${{ matrix.config.os }}
     env:
       R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
@@ -43,21 +43,38 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
 
+    - id: R
+      name: Check R version
+      run: |
+        R --version > VERSION        
+        echo "##[set-output name=version;]$(head -1 VERSION | awk '{print $3}')"
+        echo "##[set-output name=mainbiocversion;]$(Rscript -e 'cat(unlist(tools:::.BioC_version_associated_with_R_version()))' | awk '{print $1}')"
+        echo "##[set-output name=subbiocversion;]$(Rscript -e 'cat(unlist(tools:::.BioC_version_associated_with_R_version()))' | awk '{print $2}')"
+        echo "##[set-output name=biocversion;]$(Rscript -e 'cat(as.character(tools:::.BioC_version_associated_with_R_version()))' | awk '{print $1}')"
+      shell: bash -l {0}
+
     - name: Set up R
       uses: r-lib/actions/setup-r@v1
       with:
-        r-version: '4.0.3'
-    - name: Setup r-lib/remotes
-      run:
+        r-version: ${{ steps.R.outputs.version}}
+
+    - name: Setup r-lib/remotes      
+      run: |
         install.packages('remotes')
       shell: Rscript {0}
-    - name: Install system dependencies
+
+    - name: Install ubuntu system dependencies
+      if: matrix.config.os == 'ubuntu-latest'
       run: |
         sudo apt-get install libcurl4-openssl-dev
-        sysreqs=$(Rscript -e 'cat("apt-get update -y && apt-get install -y", paste(gsub("apt-get install -y ", "", remotes::system_requirements("ubuntu", "20.04")), collapse = " "))')
+        sysreqs=$(Rscript -e 'cat("apt-get update -y && apt-get install -y", paste(gsub("apt-get install -y ", "", remotes::system_requirements("ubuntu", "20.04", package = c("shazam","alakazam","tigger","airr","optparse","Biostrings","GenomicAlignments","IRanges","BiocManager","RCurl","XML"))), collapse = " "))')
         echo $sysreqs
         sudo -s eval "$sysreqs"
-        
+    - name: Install macOS system dependencies
+      if: matrix.config.os == 'macos-latest'
+      run: |
+        brew install cairo pkg-config autoconf automake libtool
+
     - name: Cache conda
       uses: actions/cache@v1
       env:
@@ -68,23 +85,23 @@ jobs:
         key:
           ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
           hashFiles('environment.yml') }}
-
-    - name: Cache R packages
-      if: "!contains(github.event.head_commit.message, '/nocache') && matrix.config.os != 'ubuntu-latest'"
+    
+    - name: Cache ubuntu R packages
+      if: "!contains(github.event.head_commit.message, '/nocache') && matrix.config.os == 'ubuntu-latest'"
       uses: actions/cache@v2
       with:
-        path: ${{ env.R_LIBS_USER }}
-        key: ${{ env.cache-version }}-${{ matrix.config.os }}-biocversion-RELEASE_3_12-r-4.0-${{ hashFiles('.github/depends.Rds') }}
-        restore-keys: ${{ env.cache-version }}-${{ matrix.config.os }}-biocversion-RELEASE_3_12-r-4.0-
+        path: /home/runner/work/_temp/Library
+        key: ${{ env.cache-version }}-${{ matrix.config.os }}-biocversion-RELEASE_${{ steps.R.outputs.mainbiocversion}}_${{ steps.R.outputs.subbiocversion}}-r-${{ steps.R.outputs.version}}-${{ hashFiles('.github/depends.Rds') }}
+        restore-keys: ${{ env.cache-version }}-${{ matrix.config.os }}-biocversion-RELEASE_${{ steps.R.outputs.mainbiocversion}}_${{ steps.R.outputs.subbiocversion}}-r-${{ steps.R.outputs.version}}-
 
-    - name: Cache R packages on ubuntu-latest
-      if: "!contains(github.event.head_commit.message, '/nocache') && matrix.config.os == 'ubuntu-latest' "
+    - name: Cache macOS R packages
+      if: "!contains(github.event.head_commit.message, '/nocache') && matrix.config.os != 'ubuntu-latest'"
       uses: actions/cache@v2
       with:
-        path: /home/runner/work/_temp/Library
-        key: ${{ env.cache-version }}-${{ matrix.config.os }}-biocversion-RELEASE_3_12-r-4.0-${{ hashFiles('.github/depends.Rds') }}
-        restore-keys: ${{ env.cache-version }}-${{ matrix.config.os }}-biocversion-RELEASE_3_12-r-4.0-
-            
+        path: ${{ env.R_LIBS_USER }}
+        key: ${{ env.cache-version }}-${{ matrix.config.os }}-biocversion-RELEASE_${{ steps.R.outputs.mainbiocversion}}_${{ steps.R.outputs.subbiocversion}}-r-${{ steps.R.outputs.version}}-${{ hashFiles('.github/depends.Rds') }}
+        restore-keys: ${{ env.cache-version }}-${{ matrix.config.os }}-biocversion-RELEASE_${{ steps.R.outputs.mainbiocversion}}_${{ steps.R.outputs.subbiocversion}}-r-${{ steps.R.outputs.version}}-
+
     - name: Setup Miniconda
       uses: conda-incubator/setup-miniconda@v2.0.1
       with:
@@ -99,37 +116,47 @@ jobs:
 
     - name: Install Dandelion
       run: |
-        python setup.py install
-
-    - name: Force installation of rpy2
-      uses: nick-invision/retry@v2
-      with:
-        timeout_seconds: 15
-        max_attempts: 3
-        command: |
-          pip install "rpy2>=3.4.0"
-        on_retry_command: |
-          pip install "rpy2>=3.4.0" -U
+        python setup.py install    
               
-    - name: install R dependencies
+    - name: Install ubuntu R dependencies
+      if: matrix.config.os == 'ubuntu-latest'
       run: |
         install.packages(c('RCurl','XML'))
         remotes::install_cran("BiocManager")
-        BiocManager::install(version = "3.12", ask = FALSE)
+        BiocManager::install(version = ${{ steps.R.outputs.biocversion}}, ask = FALSE)
         BiocManager::install(c('Biostrings', 'GenomicAlignments', 'IRanges'))
         install.packages(c('shazam', 'alakazam', 'tigger', 'airr', 'optparse'))
       shell: Rscript {0}
 
-    - name: Test if R dependencies are installed properly
-      uses: nick-invision/retry@v2
-      with:
-        timeout_seconds: 15
-        max_attempts: 3
-        command: |
-          Rscript tests/r_dependencies.R
-        on_retry_command: |
-          Rscript tests/r_install.R
+    - name: Install macOS R dependencies
+      if: matrix.config.os != 'ubuntu-latest'
+      run: |
+        options(install.packages.compile.from.source = "never")
+        install.packages(c('RCurl','XML'))
+        remotes::install_cran("BiocManager")
+        BiocManager::install(version = ${{ steps.R.outputs.biocversion}}, ask = FALSE)
+        BiocManager::install(c('Biostrings', 'GenomicAlignments', 'IRanges'))
+        install.packages(c('shazam', 'alakazam', 'tigger', 'airr', 'optparse'))
+      shell: Rscript {0}
+
+    - name: Test if R dependencies are installed properly      
+      run: |
+        Rscript tests/r_dependencies.R
+      shell: bash -l {0}
 
     - name: Test with pytest
       run: |
-        pytest -rP tests -W ignore::DeprecationWarning -W ignore::PendingDeprecationWarning -W ignore::FutureWarning
+        pytest --cov=./ --cov-report=xml -rP tests -W ignore::DeprecationWarning -W ignore::PendingDeprecationWarning -W ignore::FutureWarning
+    
+    - name: Upload coverage to Codecov
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: ./coverage.xml
+        directory: ./coverage/reports/
+        flags: unittests
+        env_vars: OS,PYTHON
+        name: codecov-umbrella
+        fail_ci_if_error: true
+        path_to_write_report: ./coverage/codecov_report.txt
+        verbose: true

.gitignore

@@ -5,7 +5,14 @@
 *-checkpoint.ipynb
 */dist/
 *__pycache__
+*.pytest_cache
 *scmtag.py
-dist/*
-sc_dandelion.egg-info/*
+*dist
+*sc_dandelion.egg-info
+*.coverage
+*.coverage*
 dandelion/logging/version.py
+tests/test_local.py
+tests/reassigned_all/
+tests/reassigned_filtered/
+tests/test_local_tr.py

DESCRIPTION.md

@@ -4,6 +4,7 @@
 [![master](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=master)]((https://github.com/zktuong/dandelion/actions?query=workflow%3Atests))
 [![](https://byob.yarr.is/zktuong/dandelion/devel-version)](https://github.com/zktuong/dandelion/tree/devel)
 [![tests](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=devel)](https://github.com/zktuong/dandelion/actions?query=workflow%3Atests)
+[![codecov](https://codecov.io/gh/zktuong/dandelion/branch/master/graph/badge.svg?token=661BMU1FBO)](https://codecov.io/gh/zktuong/dandelion)
 
 ![](docs/notebooks/img/dandelion_logo_illustration.png)
 
@@ -16,7 +17,7 @@ Hi there! I have put together a python package for analyzing single cell BCR/V(D
 
 Original preprint:
 
-*Emily Stephenson, Gary Reynolds, Rachel A Botting, Fernando J Calero-Nieto, Michael Morgan, Zewen Kelvin Tuong, Karsten Bach, Waradon Sungnak, Kaylee B Worlock, Masahiro Yoshida, Natsuhiko Kumasaka, Katarzyna Kania, Justin Engelbert, Bayanne Olabi, Jarmila Stremenova Spegarova, Nicola K Wilson, Nicole Mende, Laura Jardine, Louis CS Gardner, Issac Goh, Dave Horsfall, Jim McGrath, Simone Webb, Michael W Mather, Rik GH Lindeboom, Emma Dann, Ni Huang, Krzysztof Polanski, Elena Prigmore, Florian Gothe, Jonathan Scott, Rebecca P Payne, Kenneth F Baker, Aidan T Hanrath, Ina CD Schim van der Loeff, Andrew S Barr, Amada Sanchez-Gonzalez, Laura Bergamaschi, Federica Mescia, Josephine L Barnes, Eliz Kilich, Angus de Wilton, Anita Saigal, Aarash Saleh, Sam M Janes, Claire M Smith, Nusayhah Gopee, Caroline Wilson, Paul Coupland, Jonathan M Coxhead, Vladimir Y Kiselev, Stijn van Dongen, Jaume Bacardit, Hamish W King, Anthony J Rostron, A John Simpson, Sophie Hambleton, Elisa Laurenti, Paul A Lyons, Kerstin B Meyer, Marko Z Nikolic, Christopher JA Duncan, Ken Smith, Sarah A Teichmann, Menna R Clatworthy, John C Marioni, Berthold Gottgens, Muzlifah Haniffa.* ***Single-cell multi-omics analysis of the immune response in COVID-19***. *Nature Medicine 2021.04.20; doi: https://dx.doi.org/10.1038/s41591-021-01329-2*
+*Emily Stephenson, Gary Reynolds, Rachel A Botting, Fernando J Calero-Nieto, Michael Morgan, Zewen Kelvin Tuong, Karsten Bach, Waradon Sungnak, Kaylee B Worlock, Masahiro Yoshida, Natsuhiko Kumasaka, Katarzyna Kania, Justin Engelbert, Bayanne Olabi, Jarmila Stremenova Spegarova, Nicola K Wilson, Nicole Mende, Laura Jardine, Louis CS Gardner, Issac Goh, Dave Horsfall, Jim McGrath, Simone Webb, Michael W Mather, Rik GH Lindeboom, Emma Dann, Ni Huang, Krzysztof Polanski, Elena Prigmore, Florian Gothe, Jonathan Scott, Rebecca P Payne, Kenneth F Baker, Aidan T Hanrath, Ina CD Schim van der Loeff, Andrew S Barr, Amada Sanchez-Gonzalez, Laura Bergamaschi, Federica Mescia, Josephine L Barnes, Eliz Kilich, Angus de Wilton, Anita Saigal, Aarash Saleh, Sam M Janes, Claire M Smith, Nusayhah Gopee, Caroline Wilson, Paul Coupland, Jonathan M Coxhead, Vladimir Y Kiselev, Stijn van Dongen, Jaume Bacardit, Hamish W King, Anthony J Rostron, A John Simpson, Sophie Hambleton, Elisa Laurenti, Paul A Lyons, Kerstin B Meyer, Marko Z Nikolic, Christopher JA Duncan, Ken Smith, Sarah A Teichmann, Menna R Clatworthy, John C Marioni, Berthold Gottgens, Muzlifah Haniffa.* ***The cellular immune response to COVID-19 deciphered by single cell multi-omics across three UK centres***. *medRxiv 2021.01.13.21249725; doi: https://doi.org/10.1101/2021.01.13.21249725*
 
 ## Overview
 

README.md

@@ -4,6 +4,7 @@
 [![master](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=master)]((https://github.com/zktuong/dandelion/actions?query=workflow%3Atests))
 [![](https://byob.yarr.is/zktuong/dandelion/devel-version)](https://github.com/zktuong/dandelion/tree/devel)
 [![tests](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=devel)](https://github.com/zktuong/dandelion/actions?query=workflow%3Atests)
+[![codecov](https://codecov.io/gh/zktuong/dandelion/branch/master/graph/badge.svg?token=661BMU1FBO)](https://codecov.io/gh/zktuong/dandelion)
 
 ![](docs/notebooks/img/dandelion_logo_illustration.png)
 
@@ -16,7 +17,7 @@ Hi there! I have put together a python package for analyzing single cell BCR/V(D
 
 Original preprint:
 
-*Emily Stephenson, Gary Reynolds, Rachel A Botting, Fernando J Calero-Nieto, Michael Morgan, Zewen Kelvin Tuong, Karsten Bach, Waradon Sungnak, Kaylee B Worlock, Masahiro Yoshida, Natsuhiko Kumasaka, Katarzyna Kania, Justin Engelbert, Bayanne Olabi, Jarmila Stremenova Spegarova, Nicola K Wilson, Nicole Mende, Laura Jardine, Louis CS Gardner, Issac Goh, Dave Horsfall, Jim McGrath, Simone Webb, Michael W Mather, Rik GH Lindeboom, Emma Dann, Ni Huang, Krzysztof Polanski, Elena Prigmore, Florian Gothe, Jonathan Scott, Rebecca P Payne, Kenneth F Baker, Aidan T Hanrath, Ina CD Schim van der Loeff, Andrew S Barr, Amada Sanchez-Gonzalez, Laura Bergamaschi, Federica Mescia, Josephine L Barnes, Eliz Kilich, Angus de Wilton, Anita Saigal, Aarash Saleh, Sam M Janes, Claire M Smith, Nusayhah Gopee, Caroline Wilson, Paul Coupland, Jonathan M Coxhead, Vladimir Y Kiselev, Stijn van Dongen, Jaume Bacardit, Hamish W King, Anthony J Rostron, A John Simpson, Sophie Hambleton, Elisa Laurenti, Paul A Lyons, Kerstin B Meyer, Marko Z Nikolic, Christopher JA Duncan, Ken Smith, Sarah A Teichmann, Menna R Clatworthy, John C Marioni, Berthold Gottgens, Muzlifah Haniffa.* ***Single-cell multi-omics analysis of the immune response in COVID-19***. *Nature Medicine 2021.04.20; doi: https://dx.doi.org/10.1038/s41591-021-01329-2*
+*Emily Stephenson, Gary Reynolds, Rachel A Botting, Fernando J Calero-Nieto, Michael Morgan, Zewen Kelvin Tuong, Karsten Bach, Waradon Sungnak, Kaylee B Worlock, Masahiro Yoshida, Natsuhiko Kumasaka, Katarzyna Kania, Justin Engelbert, Bayanne Olabi, Jarmila Stremenova Spegarova, Nicola K Wilson, Nicole Mende, Laura Jardine, Louis CS Gardner, Issac Goh, Dave Horsfall, Jim McGrath, Simone Webb, Michael W Mather, Rik GH Lindeboom, Emma Dann, Ni Huang, Krzysztof Polanski, Elena Prigmore, Florian Gothe, Jonathan Scott, Rebecca P Payne, Kenneth F Baker, Aidan T Hanrath, Ina CD Schim van der Loeff, Andrew S Barr, Amada Sanchez-Gonzalez, Laura Bergamaschi, Federica Mescia, Josephine L Barnes, Eliz Kilich, Angus de Wilton, Anita Saigal, Aarash Saleh, Sam M Janes, Claire M Smith, Nusayhah Gopee, Caroline Wilson, Paul Coupland, Jonathan M Coxhead, Vladimir Y Kiselev, Stijn van Dongen, Jaume Bacardit, Hamish W King, Anthony J Rostron, A John Simpson, Sophie Hambleton, Elisa Laurenti, Paul A Lyons, Kerstin B Meyer, Marko Z Nikolic, Christopher JA Duncan, Ken Smith, Sarah A Teichmann, Menna R Clatworthy, John C Marioni, Berthold Gottgens, Muzlifah Haniffa.* ***The cellular immune response to COVID-19 deciphered by single cell multi-omics across three UK centres***. *medRxiv 2021.01.13.21249725; doi: https://doi.org/10.1101/2021.01.13.21249725*
 
 ## Overview
 

codecov.yaml

@@ -0,0 +1,30 @@
+codecov:
+  require_ci_to_pass: no
+
+coverage:
+  status:
+    project:
+      default:
+        # Require 1% coverage, i.e., always succeed
+        target: 1
+    patch: false
+    changes: false
+
+comment:
+  layout: "diff, flags, files"
+  behavior: once
+  require_base: no
+
+ignore:
+  - "bin"
+  - "database"
+  - "docs"
+  - "*.ipynb"
+  - "*.md"
+  - "*.txt"
+  - "*.yml"
+  - "*.in"
+  - "*.yaml"
+  - "*.cfg"
+  - "LICENSE"
+  - ".git*"