Merge pull request #49 from zktuong/devel

v0.1.0

.github/workflows/badge.yml

@@ -21,7 +21,7 @@ jobs:
         with:
           python-version: '3.7'
       - id: version
-        run: echo "##[set-output name=version;]$(python dandelion/version.py)"
+        run: echo "##[set-output name=version;]$(python dandelion/logging/_version.py)"
       - name: version badge
         uses: RubbaBoy/BYOB@v1.2.0
         with:
@@ -44,7 +44,7 @@ jobs:
         with:
           python-version: '3.7'
       - id: version
-        run: echo "##[set-output name=version;]$(python dandelion/version.py)"
+        run: echo "##[set-output name=version;]$(python dandelion/logging/_version.py)"
       - name: version badge
         uses: RubbaBoy/BYOB@v1.2.0
         with:

.github/workflows/tests.yml

@@ -8,9 +8,13 @@ on:
     branches:
       - "master"
 
+env:
+  cache-version: 'cache-v1'
+
 jobs:
   tests:
     runs-on: ubuntu-latest
+    timeout-minutes: 720
     defaults:
       run:
         shell: bash -l {0}
@@ -19,14 +23,35 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: [3.7, 3.8]
+    env:
+      R_REMOTES_NO_ERRORS_FROM_WARNINGS: true
+      NOT_CRAN: true
+      TZ: UTC
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+
     steps:
     - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+
     - name: Set up Python
       uses: actions/setup-python@v2
-    - name: Install macOS system dependencies
-      if: matrix.os == 'macos-latest'
+
+    - name: Set up R
+      uses: r-lib/actions/setup-r@v1
+      with:
+        r-version: '4.0.3'
+    - name: Setup r-lib/remotes
+      run:
+        install.packages('remotes')
+      shell: Rscript {0}
+    - name: Install system dependencies
       run: |
-        brew install cairo pkg-config autoconf automake libtool
+        sudo apt-get install libcurl4-openssl-dev
+        sysreqs=$(Rscript -e 'cat("apt-get update -y && apt-get install -y", paste(gsub("apt-get install -y ", "", remotes::system_requirements("ubuntu", "20.04")), collapse = " "))')
+        echo $sysreqs
+        sudo -s eval "$sysreqs"
+        
     - name: Cache conda
       uses: actions/cache@v1
       env:
@@ -37,6 +62,23 @@ jobs:
         key:
           ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
           hashFiles('environment.yml') }}
+
+    - name: Cache R packages
+      if: "!contains(github.event.head_commit.message, '/nocache') && matrix.os != 'ubuntu-latest'"
+      uses: actions/cache@v2
+      with:
+        path: ${{ env.R_LIBS_USER }}
+        key: ${{ env.cache-version }}-${{ matrix.os }}-biocversion-RELEASE_3_12-r-4.0-${{ hashFiles('.github/depends.Rds') }}
+        restore-keys: ${{ env.cache-version }}-${{ matrix.os }}-biocversion-RELEASE_3_12-r-4.0-
+
+    - name: Cache R packages on ubuntu-latest
+      if: "!contains(github.event.head_commit.message, '/nocache') && matrix.os == 'ubuntu-latest' "
+      uses: actions/cache@v2
+      with:
+        path: /home/runner/work/_temp/Library
+        key: ${{ env.cache-version }}-${{ matrix.os }}-biocversion-RELEASE_3_12-r-4.0-${{ hashFiles('.github/depends.Rds') }}
+        restore-keys: ${{ env.cache-version }}-${{ matrix.os }}-biocversion-RELEASE_3_12-r-4.0-
+
     - name: Setup Miniconda
       uses: conda-incubator/setup-miniconda@v2.0.1
       with:
@@ -48,9 +90,40 @@ jobs:
         python-version: ${{ matrix.python-version }}
         environment-file: environment.yml
         use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
+
     - name: Install Dandelion
       run: |
         python setup.py install
+
+    - name: Force installation of rpy2
+      uses: nick-invision/retry@v2
+      with:
+        timeout_seconds: 15
+        max_attempts: 3
+        command: |
+          pip install "rpy2>=3.4.0"
+        on_retry_command: |
+          pip install "rpy2>=3.4.0" -U
+              
+    - name: install R dependencies
+      run: |
+        install.packages(c('RCurl','XML'))
+        remotes::install_cran("BiocManager")
+        BiocManager::install(version = "3.12", ask = FALSE)
+        BiocManager::install(c('Biostrings', 'GenomicAlignments', 'IRanges'))
+        install.packages(c('shazam', 'alakazam', 'tigger', 'airr', 'optparse'))
+      shell: Rscript {0}
+
+    - name: Test if R dependencies are installed properly
+      uses: nick-invision/retry@v2
+      with:
+        timeout_seconds: 15
+        max_attempts: 3
+        command: |
+          Rscript tests/r_dependencies.R
+        on_retry_command: |
+          Rscript tests/r_install.R
+
     - name: Test with pytest
       run: |
         pytest tests -W ignore::DeprecationWarning -W ignore::PendingDeprecationWarning -W ignore::FutureWarning

.gitignore

@@ -6,5 +6,6 @@
 */*/*/__pycache__/
 */*/__pycache__/
 */__pycache__/
+__pycache__
 dist/*
 sc_dandelion.egg-info/*

DESCRIPTION.md

@@ -1,8 +1,11 @@
 [![](https://readthedocs.org/projects/sc-dandelion/badge/?version=latest)](https://sc-dandelion.readthedocs.io/en/latest/?badge=latest)
 [![](https://badge.fury.io/py/sc-dandelion.svg)](https://badge.fury.io/py/sc-dandelion)
-![](https://github.com/zktuong/dandelion/tree/master/notebooks/img/dandelion_logo_illustration.png)
+[![](https://byob.yarr.is/zktuong/dandelion/master-version)](https://github.com/zktuong/dandelion/tree/master)
+[![master](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=master)]((https://github.com/zktuong/dandelion/actions?query=workflow%3Atests))
+[![](https://byob.yarr.is/zktuong/dandelion/devel-version)](https://github.com/zktuong/dandelion/tree/devel)
+[![tests](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=devel)](https://github.com/zktuong/dandelion/actions?query=workflow%3Atests)
 
-Version = 0.0.27.post1
+![](https://github.com/zktuong/dandelion/tree/master/notebooks/img/dandelion_logo_illustration.png)
 
 Hi there! I have put together a python package for analyzing single cell BCR/V(D)J data from 10x Genomics 5' solution! It streamlines the pre-processing, leveraging some tools from immcantation suite, and integrates with scanpy/anndata for single-cell BCR analysis. It also includes a couple of functions for visualization. 
 

README.md

@@ -1,28 +1,25 @@
 [![](https://readthedocs.org/projects/sc-dandelion/badge/?version=latest)](https://sc-dandelion.readthedocs.io/en/latest/?badge=latest)
-[![](https://badge.fury.io/py/sc-dandelion.svg)](https://badge.fury.io/py/sc-dandelion)
-![master](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=master)
-![tests](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=devel)
-
-![](notebooks/img/dandelion_logo_illustration.png)
-
+[![](https://img.shields.io/pypi/v/sc-dandelion?logo=PyPI)](https://pypi.org/project/sc-dandelion/)
 [![](https://byob.yarr.is/zktuong/dandelion/master-version)](https://github.com/zktuong/dandelion/tree/master)
+[![master](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=master)]((https://github.com/zktuong/dandelion/actions?query=workflow%3Atests))
 [![](https://byob.yarr.is/zktuong/dandelion/devel-version)](https://github.com/zktuong/dandelion/tree/devel)
+[![tests](https://github.com/zktuong/dandelion/workflows/tests/badge.svg?branch=devel)](https://github.com/zktuong/dandelion/actions?query=workflow%3Atests)
 
-***There is something wrong with v0.0.28's find_clones and generate_network functions. Please either revert to the last release [v0.0.27.post2](https://github.com/clatworthylab/dandelion) or update to the devel ([v0.0.29](https://github.com/zktuong/dandelion/tree/devel)) versions.***
+![](docs/notebooks/img/dandelion_logo_illustration.png)
 
 Hi there! I have put together a python package for analyzing single cell BCR/V(D)J data from 10x Genomics 5' solution! It streamlines the pre-processing, leveraging some tools from immcantation suite, and integrates with scanpy/anndata for single-cell BCR analysis. It also includes a couple of functions for visualization. 
 
-Dandelion is now included in a preprint:
+`dandelion` is now included in a preprint:
 
 *Emily Stephenson, Gary Reynolds, Rachel A Botting, Fernando J Calero-Nieto, Michael Morgan, Zewen Kelvin Tuong, Karsten Bach, Waradon Sungnak, Kaylee B Worlock, Masahiro Yoshida, Natsuhiko Kumasaka, Katarzyna Kania, Justin Engelbert, Bayanne Olabi, Jarmila Stremenova Spegarova, Nicola K Wilson, Nicole Mende, Laura Jardine, Louis CS Gardner, Issac Goh, Dave Horsfall, Jim McGrath, Simone Webb, Michael W Mather, Rik GH Lindeboom, Emma Dann, Ni Huang, Krzysztof Polanski, Elena Prigmore, Florian Gothe, Jonathan Scott, Rebecca P Payne, Kenneth F Baker, Aidan T Hanrath, Ina CD Schim van der Loeff, Andrew S Barr, Amada Sanchez-Gonzalez, Laura Bergamaschi, Federica Mescia, Josephine L Barnes, Eliz Kilich, Angus de Wilton, Anita Saigal, Aarash Saleh, Sam M Janes, Claire M Smith, Nusayhah Gopee, Caroline Wilson, Paul Coupland, Jonathan M Coxhead, Vladimir Y Kiselev, Stijn van Dongen, Jaume Bacardit, Hamish W King, Anthony J Rostron, A John Simpson, Sophie Hambleton, Elisa Laurenti, Paul A Lyons, Kerstin B Meyer, Marko Z Nikolic, Christopher JA Duncan, Ken Smith, Sarah A Teichmann, Menna R Clatworthy, John C Marioni, Berthold Gottgens, Muzlifah Haniffa.* ***The cellular immune response to COVID-19 deciphered by single cell multi-omics across three UK centres***. *medRxiv 2021.01.13.21249725; doi: https://doi.org/10.1101/2021.01.13.21249725*
 
 ## Overview
 
-![](notebooks/img/dandelion_overview.png)
+![](docs/notebooks/img/dandelion_overview.png)
 
 Illustration of the `Dandelion` class slots
 
-![](notebooks/img/dandelion_class.png)
+![](docs/notebooks/img/dandelion_class.png)
 
 Please refer to the [documentation](https://sc-dandelion.readthedocs.io/) or the notebooks [here](https://nbviewer.jupyter.org/github/zktuong/dandelion/tree/master/docs/notebooks/):
 
@@ -34,7 +31,7 @@ I would reccomend installing this in order:
 ```bash
 # in bash/zsh terminal
 # create a conda environment with specific modules
-conda create --name dandelion python=3.7 
+conda create --name dandelion python=3.7 # or 3.8
 conda activate dandelion
 ```
 
@@ -44,11 +41,8 @@ First, install [scanpy](https://scanpy.readthedocs.io/en/latest/installation.htm
 # these are required by dandelion
 conda install -c conda-forge distance joblib plotnine adjustText
 conda install -c bioconda igblast blast # if this doesn't work, download them manually (see below)
-conda install -c conda-forge "rpy2>=3.3.2,<3.3.5" # to make compatible for R version 4
-# or pip install "rpy2>=3.3.2,<3.3.5"
-# If it fails because it's compiling using clang, first, work out where the path is to your gcc compiler (use brew to install gcc if needed):
-# then run
-# env CC=/path/to/location/of/bin/gcc-9 pip install "rpy2>=3.3.2,<3.3.5"
+conda install -c conda-forge "rpy2>=3.4" # to make compatible for R version 4
+# or pip install rpy2>=3.4
 
 # Use pip to install the following with --no-cache-dir --upgrade if necessary
 # and then lastly install this
@@ -103,22 +97,22 @@ source ~/.bash_profile # or ~/.zshenv
 Python packages
 ```python
 # conda
-python>=3.7,<3.8 (conda-forge)
+python>=3.7,<=3.8 (conda-forge)
 numpy>=1.18.4 (conda-forge)
 pandas>=1.0.3 (conda-forge)
 distance>=0.1.3 (conda-forge)
-joblib==0.14.1 (conda-forge)
-jupyter==1.0.0 (conda-forge)
+joblib>=0.14.1 (conda-forge)
+jupyter (conda-forge)
 scikit-learn>=0.23.0 (conda-forge)
 numba>=0.48.0 (conda-forge)
-pytables==3.6.1 (conda-forge)
+pytables>=3.6.1 (conda-forge)
 seaborn>=0.10.1 (conda-forge)
 leidenalg>=0.8.0 (conda-forge)
 plotnine>=0.6.0 (conda-forge)
 
 # Other executables (through conda)
 blast>=2.10.0 (bioconda) # depends on the database version as well
-igblast==1.15.0 (bioconda)
+igblast>=1.15.0 (bioconda)
 
 # pip
 anndata>=0.7.1
@@ -129,7 +123,7 @@ changeo>=1.0.0
 presto>=0.6.0
 polyleven>=0.5
 networkx>=2.4
-rpy2>=3.3.2,<3.3.5
+rpy2>=3.4 or # rpy2>=3.3.2,<3.3.5
 ```
 
 R packages

dandelion/__init__.py

@@ -2,11 +2,12 @@
 # @Author: kt16
 # @Date:   2020-05-12 18:11:20
 # @Last Modified by:   Kelvin
-# @Last Modified time: 2020-11-23 15:43:34
+# @Last Modified time: 2021-03-30 13:51:02
 
 from . import preprocessing as pp
 from . import utilities as utl
 from . import tools as tl
 from . import plotting as pl
-from .utilities import read_pkl, read_h5, Dandelion, update_metadata, concat, load_data
+from .utilities import read_pkl, read_h5, read_10x_airr, from_scirpy, to_scirpy, Dandelion, update_metadata, concat, load_data
 from .version import __version__
+from . import logging

dandelion/logging/__init__.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python
+# @Author: kt16
+# @Date:   2020-05-12 18:11:20
+# @Last Modified by:   Kelvin
+# @Last Modified time: 2021-02-16 15:59:05
+
+from ._logging import print_versions, print_header

dandelion/logging/_logging.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+# @Author: Kelvin
+# @Date:   2021-02-06 13:18:58
+# @Last Modified by:   Kelvin
+# @Last Modified time: 2021-02-20 09:07:55
+from typing import Union, Sequence, Tuple
+
+modules = ['dandelion', 'pandas', 'numpy', 'matplotlib',
+           'networkx', 'scipy', 'skbio', 'distance', 'polyleven']
+
+
+# borrowed from scanpy's logging module
+def _versions_dependencies(dependencies: Sequence):
+    for mod in dependencies:
+        mod_name, dist_name = mod if isinstance(mod, tuple) else (mod, mod)
+        try:
+            imp = __import__(mod_name)
+            yield dist_name, imp.__version__
+        except (ImportError, AttributeError):
+            pass
+
+
+def print_versions(dependencies: Sequence = modules):
+    '''
+    Versions that are essential for dandelion's operation.
+    '''
+    print(' '.join(
+        f'{mod}=={ver}'
+        for mod, ver in _versions_dependencies(dependencies)
+    ))
+
+
+def print_header(dependencies: Sequence = modules):
+    '''
+    Versions that are essential for dandelion's operation.
+    '''
+    print(' '.join(
+        f'{mod}=={ver}'
+        for mod, ver in _versions_dependencies(dependencies)
+    ))

dandelion/logging/_version.py

@@ -0,0 +1,2 @@
+exec(open('dandelion/version.py').read())
+print(__version__)