Merge pull request #161 from Mesnage-Org/ns-rse/pyyaml

Switching to run JupyterLite Notebooks

.pre-commit-config.yaml

@@ -36,3 +36,10 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
+
+ci:
+  autofix_prs: true
+  autofix_commit_msg: '[pre-commit.ci] Fixing issues with pre-commit'
+  autoupdate_schedule: weekly
+  autoupdate_commit_msg: '[pre-commit.ci] pre-commit automatically updated.'
+  # skip: [] # Optionally list ids of hooks to skip on CI

README.md

@@ -6,26 +6,23 @@
 [![Docs](https://img.shields.io/badge/github.io-docs-green)](https://mesnage-org.github.io/pgfinder/)
 [![](https://img.shields.io/badge/ORDA--DOI-10.15131%2Fshef.data.20101751.v1-lightgrey)](https://doi.org/10.15131/shef.data.20101751.v1)
 
-Interactive notebooks are available for different versions from the links below. For descriptions of the features of each version
-please refer to the [Releases](https://github.com/Mesnage-Org/pgfinder/releases) page. If you wish to use the latest changes
-and improvements select the Notebooks from the `current` version which reflects changes made to the `master` branch that
-have not yet been released.
-
-| Version  | Interactive | Example |
-|----------|-------------|---------|
-| `current` | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Mesnage-Org/PGFinder/master?urlpath=tree/pgfinder_interactive.ipynb) | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Mesnage-Org/PGFinder/master?urlpath=tree/pgfinder.ipynb) |
-| `v0.1.0` | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Mesnage-Org/PGFinder/v0.1.0?urlpath=tree/pgfinder_interactive.ipynb) | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Mesnage-Org/PGFinder/v0.1.0?urlpath=tree/pgfinder.ipynb) |
-
-  * [Introduction](https://mesnage-org.github.io/pgfinder/introduction.html)
-  * [Installation](https://mesnage-org.github.io/pgfinder/installation.html)
-  * [Usage](https://mesnage-org.github.io/pgfinder/usage.html)
-  * [Contributing](https://mesnage-org.github.io/pgfinder/contributing.html)
-  * [Copying](https://mesnage-org.github.io/pgfinder/copying.html)
+Interactive notebooks are available for use at
+[pgfinder-jupyterlite](https://mesnage-org.github.io/pgfinder-jupyterlite/lab?path=pgfinder_interactive.ipynb). For
+descriptions of the features of each version please refer to the
+[Releases](https://github.com/Mesnage-Org/pgfinder/releases) page. If you wish to use the candidate releases please
+refer to the [Usage](https://mesnage-org.github.io/pgfinder/master/usage.html).
+
+
+* [Introduction](https://mesnage-org.github.io/pgfinder/master/introduction.html)
+* [Installation](https://mesnage-org.github.io/pgfinder/master/installation.html)
+* [Usage](https://mesnage-org.github.io/pgfinder/master/usage.html)
+* [Contributing](https://mesnage-org.github.io/pgfinder/master/contributing.html)
+* [Copying](https://mesnage-org.github.io/pgfinder/master/copying.html)
 
 ## Links
 
-  * [Mesnage Lab](https://mesnagelab.weebly.com/)
+* [Mesnage Lab](https://mesnagelab.weebly.com/)
 
 ## References
 
-  * [PGFinder, a novel analysis pipeline for the consistent, reproducible, and high-resolution structural analysis of bacterial peptidoglycans | eLife](https://elifesciences.org/articles/70597)
+* [PGFinder, a novel analysis pipeline for the consistent, reproducible, and high-resolution structural analysis of bacterial peptidoglycans | eLife](https://elifesciences.org/articles/70597)

data/masses/c_diff_monomer_masses.csv

@@ -1,4 +1,4 @@
-Structure,Monoisotopicmass
+Structure,Monoisotopicmass
 gm(-Ac) |0,456.1956
 gm(-Ac) (x2) |0,934.3755
 gm(-Ac) (x3) |0,1412.5554

data/masses/e_coli_monomer_masses.csv

@@ -1,4 +1,4 @@
-Structure,Monoisotopicmass
+Structure,Monoisotopicmass
 gm|0,498.2061
 gm (x2)|0,976.386
 gm (x3)|0,1454.5659

demo_ftrs.py

@@ -11,7 +11,19 @@
 theo_masses = pgio.theo_masses_reader(csv_filepath)
 validation.validate_theo_masses_df(theo_masses)
 
-mod_test = ['Sodium','Potassium','Anh','DeAc','DeAc_Anh','Nude','Decay','Amidation','Amidase','Double_Anh','multimers_Glyco']
+mod_test = [
+    "Sodium",
+    "Potassium",
+    "Anh",
+    "DeAc",
+    "DeAc_Anh",
+    "Nude",
+    "Decay",
+    "Amidation",
+    "Amidase",
+    "Double_Anh",
+    "multimers_Glyco",
+]
 
 results = matching.data_analysis(masses, theo_masses, 0.5, mod_test, 10)
 
@@ -22,4 +34,4 @@
 print(f"ppm             : {results.attrs['ppm']}")
 print(results)
 
-pgio.dataframe_to_csv_metadata(save_filepath='./', output_dataframe=results)
+pgio.dataframe_to_csv_metadata(save_filepath="./", output_dataframe=results)

docs/contributing.md

@@ -4,42 +4,72 @@ There are a few ways to contribute:
 
 - [Raise an issue](https://github.com/Mesnage-Org/pgfinder/issues) to identify a bug or suggest a new feature.
 - Fork the repository and make a pull request to suggest changes to the code.
-- If you'd like to contribute a mass database, please do this by [raising an issue](https://github.com/Mesnage-Org/pgfinder/issues).
+- If you'd like to contribute a mass database, please do this by [raising an
+  issue](https://github.com/Mesnage-Org/pgfinder/issues).
 
 ## Development Installation
 
 The current version when installed from GitHub is a combination of the most recent Git tag combined with the hash of the
 current `HEAD` of the branch and how many commits away from the last tag it is. For further details on versioning please
 refer to [versioneer documentation](https://github.com/python-versioneer/python-versioneer).
 
-If you wish to contribute to the development of PGFinder you should clone (your fork of) this repository and install it in editable
-mode (`pip install -e`) with the following commands.
+If you wish to contribute to the development of PGFinder you should clone (your fork of) this repository and install it
+in editable mode (`pip install -e`) with the following commands which install additional dependencies for tests,
+documentation and linting.
 
 ```bash
 # Clone the repository
 git clone https://github.com/Mesnage-Org/PGFinder.git
 cd pgfinder
 # Install in editable mode
 pip install -e .
-# Install test dependencies
-pip install -e .[tests]
+# Install extra dependencies
+pip install -e ".[dev,docs,tests]"
 ```
 
 ## Testing
 
-To run unit tests:
+To run unit tests suite use [pytest](https://pytest.org)
 
 ```bash
 pytest
 ```
 
-The tests check output against an expected baseline for [Maxquant](data/baseline_output.csv) or
-[FTRS](data/baseline_output_ftrs.csv). To recreate this (e.g. in response to improvements to the scientific
-"correctness" of the code ouput), use:
+## Linting
 
+PGFinder uses [pre-commit](https://pre-commit.com) hooks to ensure code conforms to the [PEP8 Python Style
+Guide](https://pep8.org/) using the [ruff](https://duckduckgo.com/?q=ruff+linter&t=opera&ia=images) linter, applies
+[black](https://black.readthedocs.io/en/stable/index.html) formatting and ensures Notebooks are clean on
+submission. These hooks are run on GitHub when Pull Requests are made using [pre-commit.ci](https://pre-commit.ci) and
+if you have not run them locally then your pull request will fail the tests it needs to pass.
 
-```bash
-python make_baseline.py
+`pre-commit` will have been installed as part of the extra dependencies above (they are part of `dev`), but you need to
+install `pre-commit` and the hooks locally in your virtual environment. This can be done with the following commands.
+
+``` bash
+pre-commit install
+pre-commit install-hooks
 ```
 
-...and replace the existing file.
+Now when you make a `git commit` the hooks will run first and report any errors. Sometimes if the errors can be
+corrected automatically they will be and the files will be modified in place, but you will have to then `git stage` the
+modified files again before completing the commit (this gives you an opportunity to review the changes that have been
+made but typically they are ok to accept, they will be `black` formatting or Notebook cleaning).
+
+
+## Releasing to PyPI
+
+Release to the [Python Package Index (PyPI)](https://pypi.org) are automated and occur when a new release is made on
+GitHub with a tag that begins with `v#.#.#'`. PGFinder uses [semantic verisoning](https://semver.org/).
+
+### Pre-release candidates
+
+If you have new features you wish to test using the [PGFinder JupyterLite
+Notebook](https://github.com/Mesnage-Org/pgfinder-jupyterlite) then the tag should indicate that this is a "pre-release"
+and as well as a semantic release number it should be followed with one of the [PEP0440 pre-release
+spellings](https://peps.python.org/pep-0440/#pre-release-spelling) (using a [pre-release
+separator](https://peps.python.org/pep-0440/#pre-release-separators)). For example if the most recent release is
+`v0.0.4` and you have a new feature you wish to test in the Jupyter Lite Notebook the tag should take the form
+`v0.0.4-a1` to indicate this is the first alpha revision of the new feature. If you find errors and correct them bump
+the number following `a` i.e. `v0.0.4-a2`. Once you are happy that the new feature is working as desired you can proceed
+with making a new release which in this example, following semantic versioning, would be `v0.0.5`.

docs/installation.md

@@ -2,7 +2,7 @@
 
 ## Online Notebooks
 
-There is no need to install `pgfinder` - you can [run it online using myBinder](usage.md).
+There is no need to install `pgfinder` - you can [run it online](usage.md).
 
 ## Virtual Environment
 
@@ -12,7 +12,7 @@ run the following commands to create the virtual environment and activate it:
 
 
 ```bash
-conda create --force -n pgfinder python=3.7
+conda create --force -n pgfinder python=3.10
 conda activate pgfinder
 ```
 

docs/usage.md

@@ -5,32 +5,92 @@ describes software inputs and outputs.
 
 ## Online Notebooks
 
-Interactive notebooks are available for different versions from the links below. For descriptions of the features of each version
-please refer to the [Releases](https://github.com/Mesnage-Org/pgfinder/releases) page. If you wish to use the latest changes
-and improvements select the Notebooks from the `current` version which reflects changes made to the `master` branch that
-have not yet been released.
+Interactive Notebooks are implemented via
+[pgfinder-jupyterlite](https://github.com/Mesnage-Org/pgfinder-jupyterlite). To use these go to
+[mesnage-org.github.io/pgfinder-jupyterlite](https://mesnage-org.github.io/pgfinder-jupyterlite/lab?path=pgfinder_interactive.ipynb).
 
-| Version  | Interactive | Example |
-|----------|-------------|---------|
-| `current` | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Mesnage-Org/PGFinder/master?urlpath=tree/pgfinder_interactive.ipynb) | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Mesnage-Org/PGFinder/master?urlpath=tree/pgfinder.ipynb) |
-| `v0.1.0` | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Mesnage-Org/PGFinder/v0.1.0?urlpath=tree/pgfinder_interactive.ipynb) | [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/Mesnage-Org/PGFinder/v0.1.0?urlpath=tree/pgfinder.ipynb) |
+### IMPORTANT - Clearing Cache
 
-**NB** It is essential that you wait for file uploads to complete before attempting to run your analysis. This is most
-relevant when uploading `.ftrs` files during **Step 1**. You can tell that your uploads (whether that is `.ftrs` or
-`.csv` files at **Step 1** or **Step 3**) have completed because the filename will be listed beside the upload button in
-the Notebook as highlighted in the image below.
+This implementation of Notebooks is relatively new and it is important that your browser has loaded the
+page anew **each and every time you wish to run analyses** so that it does not use a cached version from a previous
+visit you have made to the site. How you do this will depend on the browser you are using.
+
+#### Firefox
+
+Instructions on clearing the cache for a single site for [Firefox](https://superuser.com/a/733154). Alternatively
+click on the padlock to the left of the address bar and select `Clear cookies and site data...`. Then refresh the page
+by pressing `F5`.
+
+
+#### Chrome/Chromium
+
+Instructions on clearing the cache for a single site for
+[Chrome/Chromium](https://www.guidingtech.com/clear-chrome-cookies-cache-one-site-only/) areavaiable, but you can use
+the shortcut of pressing `Shift + F5`
+
+#### Opera
+
+Instructions on clearing the cache for a single site for [Opera](https://forums.opera.com/post/229032). Alternatively
+clock on the padlock to the left of the address bar and select `Site settings` then clock on the `Clear data`
+button. Then refresh the page by pressing `F5`.
+
+
+### Running Analyses
+
+Once the page is loaded you will see the following [JupyterLab](https://jupyter.org/) layout.
+
+![JupyterLite Notebook Launched](img/jupyter_lite_pgfinder_notebook_launch.png)
+
+To run the analysis follow the instructions and select _Kernel > Restart Kernel and Run All Cells_. This will take a
+little while to run (30-60 seconds) because `pgfinder` and its dependencies are being installed. Once completed you
+should see some output indicating that parameters have been loaded and buttons and menus should appear under each of the
+**Step N** headings as shown in the image below.
+
+![JupyterLite Notebook Ready](img/jupyter_lite_pgfinder_notebook_ready.png)
+
+You can now work through each of the steps uploading Deconvoluted Data, Selecting Modifications, Choosing or uploading a
+Mass Library, setting the PPM tolerance and the Time Window for in-source decay and salt adduct clean up.
+
+**NB** In versions of the Notebook that ran under myBinder it was essential that you wait for file uploads to complete
+before attempting to run your analysis. This is most relevant when uploading `.ftrs` files during **Step 1**. You can
+tell that your uploads (whether that is `.ftrs` or `.csv` files at **Step 1** or **Step 3**) have completed because the
+filename will be listed beside the upload button in the Notebook as highlighted in the image below. However, because
+this Notebook is running in your browser the upload is only going from your local harddrive into the memory your browser
+is using and should happen very quickly.
 
 ![](img/binder_upload.png)
 
+Once all files have uploaded you are ready to run your analysis by clicking on the **Run Analysis** at the bottom. On
+completion this button will change to show **Results**, click on this button to download your results.
+
+![JupyterLite Notebook Results](img/jupyter_lite_pgfinder_notebook_results.png)
+
+### Testing pre-release candidates
+
+If a development version with new features is available for testing you can test it in the Notebook by explicitly
+stating the version of `pgfinder` that should be used. To do this a pre-release candidate will need to be released to
+PyPI (see [Contributing](contributing.md)) and you will need to know the full version including the pre-release
+suffix (see the [Release History](https://pypi.org/project/pgfinder/#history)). You then append this version to the
+`%pip install pgfinder` command in the first code cell.
+
+Clear the cache for the site and reload it (see **IMPORTANT** note above) and start a fresh instance of the Notebook,
+but before restarting the kernel you need to modify the version of PGFinder that is installed explicitly under **Step
+0 : Install and Setup PGFinder**.
+
+Code cells are hidden by default in the Notebooks, to expand them click on the **first** set of three dots and you will
+see a Notebook cell with the line `%pip install pgfinder` in it. If the release candidate is `v0.0.4-a5` then append
+(without the leading `v`) to the install command separating with `==`, i.e. `%pip install pgfinder==0.0.4-a5`.
+
+![JupyterLite Explicit Version](img/jupyter_lite_pgfinder_notebook_explicit_version.png)
+
+You can now restart the kernel (_Kernel > Restart Kernel and Run All Cells_) and run your analysis and the features in
+the candidate release will be run.
+
 
 ## Command Line
 
 If you wish to use the command line version you will have to follow the [installation](installation.md) instructions to
-install PGFinder on your computer. Once you have done so a demo is available with...
-
-```bash
-python demo.py
-```
+install PGFinder on your computer.
 
 ## `find_pg`
 

make_baseline.py

@@ -5,18 +5,30 @@
 # Set mass database and modifications
 csv_filepath = "data/masses/e_coli_monomer_masses.csv"
 theo_masses = pgio.theo_masses_reader(csv_filepath)
-mod_test = ['Sodium','Potassium','Anh','DeAc','DeAc_Anh','Nude','Decay','Amidation','Amidase','Double_Anh','multimers_Glyco']
+mod_test = [
+    "Sodium",
+    "Potassium",
+    "Anh",
+    "DeAc",
+    "DeAc_Anh",
+    "Nude",
+    "Decay",
+    "Amidation",
+    "Amidase",
+    "Double_Anh",
+    "multimers_Glyco",
+]
 
 # Generate maxquant baseline
 mq_filepath = "data/maxquant_test_data.txt"
 masses_mq = pgio.ms_file_reader(mq_filepath)
 validation.validate_raw_data_df(masses_mq)
 results = matching.data_analysis(masses_mq, theo_masses, 0.5, mod_test, 10)
-pgio.dataframe_to_csv_metadata(save_filepath='./data/', output_dataframe=results, filename='baseline_output_mq.csv')
+pgio.dataframe_to_csv_metadata(save_filepath="./data/", output_dataframe=results, filename="baseline_output_mq.csv")
 
 # Generate ftrs baseline
 ftrs_filepath = "data/ftrs_test_data.ftrs"
 masses_ftrs = pgio.ms_file_reader(ftrs_filepath)
 validation.validate_raw_data_df(masses_ftrs)
 results = matching.data_analysis(masses_ftrs, theo_masses, 0.5, mod_test, 10)
-pgio.dataframe_to_csv_metadata(save_filepath='./data/', output_dataframe=results, filename='baseline_output_ftrs.csv')
+pgio.dataframe_to_csv_metadata(save_filepath="./data/", output_dataframe=results, filename="baseline_output_ftrs.csv")

pgfinder.ipynb

@@ -90,7 +90,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "mod_test = ['Sodium','Nude', 'DeAc']\n",
+    "mod_test = [\"Sodium\", \"Nude\", \"DeAc\"]\n",
     "results = matching.data_analysis(raw_data, theo_masses, 0.5, mod_test, 10)"
    ]
   },
@@ -104,9 +104,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "metadata": {
-    "scrolled": true
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "display(from_dataframe(results.head(10)))"

pgfinder/__init__.py

@@ -1,21 +1,13 @@
 """Package initialisation"""
 from importlib.metadata import version
-import logging
 import yaml
 from pkgutil import get_data
-from pgfinder.logs.logs import setup_logger, LOGGER_NAME
 from pgfinder.utils import dict_to_decimal
 
-LOGGER = setup_logger()
-LOGGER = logging.getLogger(LOGGER_NAME)
-
-
 PARAMETERS_FILE = "config/parameters.yaml"
 PARAMETERS = get_data(__package__, PARAMETERS_FILE)
 PARAMETERS = yaml.safe_load(PARAMETERS)
-LOGGER.info(f"Loaded parameters from file : {PARAMETERS_FILE}")
 PARAMETERS = dict_to_decimal(PARAMETERS)
-LOGGER.info("All parameters converted to decimal")
 MULTIMERS = PARAMETERS["multimer"]
 MOD_TYPE = PARAMETERS["mod_type"]
 MASS_TO_CLEAN = PARAMETERS["mass_to_clean"]

pgfinder/find_pg.py

@@ -15,10 +15,11 @@
     dataframe_to_csv,
     dataframe_to_csv_metadata,
 )
-from pgfinder.logs.logs import LOGGER_NAME
+from pgfinder.logs.logs import LOGGER_NAME, setup_logger
 from pgfinder.utils import update_config
 from pgfinder.pgio import read_yaml, default_filename
 
+LOGGER = setup_logger()
 LOGGER = logging.getLogger(LOGGER_NAME)